sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/conf.h>
  36 #include <sys/priv.h>
  37 #include <sys/kernel.h>
  38 #include <sys/bus.h>
  39 #include <sys/module.h>
  40 #include <sys/malloc.h>
  41 #include <sys/queue.h>
  42 #include <sys/taskqueue.h>
  43 #include <sys/pciio.h>
  44 #include <dev/pci/pcireg.h>
  45 #include <dev/pci/pcivar.h>
  46 #include <dev/pci/pci_private.h>
  47 #include <sys/firmware.h>
  48 #include <sys/sbuf.h>
  49 #include <sys/smp.h>
  50 #include <sys/socket.h>
  51 #include <sys/sockio.h>
  52 #include <sys/sysctl.h>
  53 #include <net/ethernet.h>
  54 #include <net/if.h>
  55 #include <net/if_types.h>
  56 #include <net/if_dl.h>
  57 #include <net/if_vlan_var.h>
  58 #if defined(__i386__) || defined(__amd64__)
  59 #include <vm/vm.h>
  60 #include <vm/pmap.h>
  61 #endif
  62
  63 #include "common/common.h"
  64 #include "common/t4_msg.h"
  65 #include "common/t4_regs.h"
  66 #include "common/t4_regs_values.h"
  67 #include "t4_ioctl.h"
  68 #include "t4_l2t.h"
  69
  70 /* T4 bus driver interface */
  71 static int t4_probe(device_t);
  72 static int t4_attach(device_t);
  73 static int t4_detach(device_t);
  74 static device_method_t t4_methods[] = {
  75         DEVMETHOD(device_probe,         t4_probe),
  76         DEVMETHOD(device_attach,        t4_attach),
  77         DEVMETHOD(device_detach,        t4_detach),
  78
  79         DEVMETHOD_END
  80 };
  81 static driver_t t4_driver = {
  82         "t4nex",
  83         t4_methods,
  84         sizeof(struct adapter)
  85 };
  86
  87
  88 /* T4 port (cxgbe) interface */
  89 static int cxgbe_probe(device_t);
  90 static int cxgbe_attach(device_t);
  91 static int cxgbe_detach(device_t);
  92 static device_method_t cxgbe_methods[] = {
  93         DEVMETHOD(device_probe,         cxgbe_probe),
  94         DEVMETHOD(device_attach,        cxgbe_attach),
  95         DEVMETHOD(device_detach,        cxgbe_detach),
  96         { 0, 0 }
  97 };
  98 static driver_t cxgbe_driver = {
  99         "cxgbe",
 100         cxgbe_methods,
 101         sizeof(struct port_info)
 102 };
 103
 104 static d_ioctl_t t4_ioctl;
 105 static d_open_t t4_open;
 106 static d_close_t t4_close;
 107
 108 static struct cdevsw t4_cdevsw = {
 109        .d_version = D_VERSION,
 110        .d_flags = 0,
 111        .d_open = t4_open,
 112        .d_close = t4_close,
 113        .d_ioctl = t4_ioctl,
 114        .d_name = "t4nex",
 115 };
 116
 117 /* T5 bus driver interface */
 118 static int t5_probe(device_t);
 119 static device_method_t t5_methods[] = {
 120         DEVMETHOD(device_probe,         t5_probe),
 121         DEVMETHOD(device_attach,        t4_attach),
 122         DEVMETHOD(device_detach,        t4_detach),
 123
 124         DEVMETHOD_END
 125 };
 126 static driver_t t5_driver = {
 127         "t5nex",
 128         t5_methods,
 129         sizeof(struct adapter)
 130 };
 131
 132
 133 /* T5 port (cxl) interface */
 134 static driver_t cxl_driver = {
 135         "cxl",
 136         cxgbe_methods,
 137         sizeof(struct port_info)
 138 };
 139
 140 static struct cdevsw t5_cdevsw = {
 141        .d_version = D_VERSION,
 142        .d_flags = 0,
 143        .d_open = t4_open,
 144        .d_close = t4_close,
 145        .d_ioctl = t4_ioctl,
 146        .d_name = "t5nex",
 147 };
 148
 149 /* ifnet + media interface */
 150 static void cxgbe_init(void *);
 151 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 152 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 153 static void cxgbe_qflush(struct ifnet *);
 154 static int cxgbe_media_change(struct ifnet *);
 155 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 156
 157 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
 158
 159 /*
 160  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 161  * then ADAPTER_LOCK, then t4_uld_list_lock.
 162  */
 163 static struct sx t4_list_lock;
 164 SLIST_HEAD(, adapter) t4_list;
 165 #ifdef TCP_OFFLOAD
 166 static struct sx t4_uld_list_lock;
 167 SLIST_HEAD(, uld_info) t4_uld_list;
 168 #endif
 169
 170 /*
 171  * Tunables.  See tweak_tunables() too.
 172  *
 173  * Each tunable is set to a default value here if it's known at compile-time.
 174  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
 175  * provide a reasonable default when the driver is loaded.
 176  *
 177  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
 178  * T5 are under hw.cxl.
 179  */
 180
 181 /*
 182  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 183  */
 184 #define NTXQ_10G 16
 185 static int t4_ntxq10g = -1;
 186 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 187
 188 #define NRXQ_10G 8
 189 static int t4_nrxq10g = -1;
 190 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 191
 192 #define NTXQ_1G 4
 193 static int t4_ntxq1g = -1;
 194 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 195
 196 #define NRXQ_1G 2
 197 static int t4_nrxq1g = -1;
 198 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 199
 200 static int t4_rsrv_noflowq = 0;
 201 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
 202
 203 #ifdef TCP_OFFLOAD
 204 #define NOFLDTXQ_10G 8
 205 static int t4_nofldtxq10g = -1;
 206 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 207
 208 #define NOFLDRXQ_10G 2
 209 static int t4_nofldrxq10g = -1;
 210 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 211
 212 #define NOFLDTXQ_1G 2
 213 static int t4_nofldtxq1g = -1;
 214 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 215
 216 #define NOFLDRXQ_1G 1
 217 static int t4_nofldrxq1g = -1;
 218 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 219 #endif
 220
 221 #ifdef DEV_NETMAP
 222 #define NNMTXQ_10G 2
 223 static int t4_nnmtxq10g = -1;
 224 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g);
 225
 226 #define NNMRXQ_10G 2
 227 static int t4_nnmrxq10g = -1;
 228 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g);
 229
 230 #define NNMTXQ_1G 1
 231 static int t4_nnmtxq1g = -1;
 232 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g);
 233
 234 #define NNMRXQ_1G 1
 235 static int t4_nnmrxq1g = -1;
 236 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g);
 237 #endif
 238
 239 /*
 240  * Holdoff parameters for 10G and 1G ports.
 241  */
 242 #define TMR_IDX_10G 1
 243 static int t4_tmr_idx_10g = TMR_IDX_10G;
 244 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 245
 246 #define PKTC_IDX_10G (-1)
 247 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 248 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 249
 250 #define TMR_IDX_1G 1
 251 static int t4_tmr_idx_1g = TMR_IDX_1G;
 252 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 253
 254 #define PKTC_IDX_1G (-1)
 255 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 256 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 257
 258 /*
 259  * Size (# of entries) of each tx and rx queue.
 260  */
 261 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 262 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 263
 264 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 265 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 266
 267 /*
 268  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 269  */
 270 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 271 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 272
 273 /*
 274  * Configuration file.
 275  */
 276 #define DEFAULT_CF      "default"
 277 #define FLASH_CF        "flash"
 278 #define UWIRE_CF        "uwire"
 279 #define FPGA_CF         "fpga"
 280 static char t4_cfg_file[32] = DEFAULT_CF;
 281 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 282
 283 /*
 284  * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively).
 285  * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
 286  * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
 287  *            mark or when signalled to do so, 0 to never emit PAUSE.
 288  */
 289 static int t4_pause_settings = PAUSE_TX | PAUSE_RX;
 290 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings);
 291
 292 /*
 293  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
 294  * encouraged respectively).
 295  */
 296 static unsigned int t4_fw_install = 1;
 297 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 298
 299 /*
 300  * ASIC features that will be used.  Disable the ones you don't want so that the
 301  * chip resources aren't wasted on features that will not be used.
 302  */
 303 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 304 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 305
 306 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 307 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 308
 309 static int t4_toecaps_allowed = -1;
 310 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 311
 312 static int t4_rdmacaps_allowed = 0;
 313 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 314
 315 static int t4_iscsicaps_allowed = 0;
 316 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 317
 318 static int t4_fcoecaps_allowed = 0;
 319 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 320
 321 static int t5_write_combine = 0;
 322 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 323
 324 struct intrs_and_queues {
 325         uint16_t intr_type;     /* INTx, MSI, or MSI-X */
 326         uint16_t nirq;          /* Total # of vectors */
 327         uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
 328         uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */
 329         uint16_t ntxq10g;       /* # of NIC txq's for each 10G port */
 330         uint16_t nrxq10g;       /* # of NIC rxq's for each 10G port */
 331         uint16_t ntxq1g;        /* # of NIC txq's for each 1G port */
 332         uint16_t nrxq1g;        /* # of NIC rxq's for each 1G port */
 333         uint16_t rsrv_noflowq;  /* Flag whether to reserve queue 0 */
 334 #ifdef TCP_OFFLOAD
 335         uint16_t nofldtxq10g;   /* # of TOE txq's for each 10G port */
 336         uint16_t nofldrxq10g;   /* # of TOE rxq's for each 10G port */
 337         uint16_t nofldtxq1g;    /* # of TOE txq's for each 1G port */
 338         uint16_t nofldrxq1g;    /* # of TOE rxq's for each 1G port */
 339 #endif
 340 #ifdef DEV_NETMAP
 341         uint16_t nnmtxq10g;     /* # of netmap txq's for each 10G port */
 342         uint16_t nnmrxq10g;     /* # of netmap rxq's for each 10G port */
 343         uint16_t nnmtxq1g;      /* # of netmap txq's for each 1G port */
 344         uint16_t nnmrxq1g;      /* # of netmap rxq's for each 1G port */
 345 #endif
 346 };
 347
 348 struct filter_entry {
 349         uint32_t valid:1;       /* filter allocated and valid */
 350         uint32_t locked:1;      /* filter is administratively locked */
 351         uint32_t pending:1;     /* filter action is pending firmware reply */
 352         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 353         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 354
 355         struct t4_filter_specification fs;
 356 };
 357
 358 static int map_bars_0_and_4(struct adapter *);
 359 static int map_bar_2(struct adapter *);
 360 static void setup_memwin(struct adapter *);
 361 static int validate_mem_range(struct adapter *, uint32_t, int);
 362 static int fwmtype_to_hwmtype(int);
 363 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
 364     uint32_t *);
 365 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
 366 static uint32_t position_memwin(struct adapter *, int, uint32_t);
 367 static int cfg_itype_and_nqueues(struct adapter *, int, int,
 368     struct intrs_and_queues *);
 369 static int prep_firmware(struct adapter *);
 370 static int partition_resources(struct adapter *, const struct firmware *,
 371     const char *);
 372 static int get_params__pre_init(struct adapter *);
 373 static int get_params__post_init(struct adapter *);
 374 static int set_params__post_init(struct adapter *);
 375 static void t4_set_desc(struct adapter *);
 376 static void build_medialist(struct port_info *, struct ifmedia *);
 377 static int cxgbe_init_synchronized(struct port_info *);
 378 static int cxgbe_uninit_synchronized(struct port_info *);
 379 static int setup_intr_handlers(struct adapter *);
 380 static void quiesce_eq(struct adapter *, struct sge_eq *);
 381 static void quiesce_iq(struct adapter *, struct sge_iq *);
 382 static void quiesce_fl(struct adapter *, struct sge_fl *);
 383 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 384     driver_intr_t *, void *, char *);
 385 static int t4_free_irq(struct adapter *, struct irq *);
 386 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
 387     unsigned int);
 388 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 389 static void cxgbe_tick(void *);
 390 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 391 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
 392     struct mbuf *);
 393 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 394 static int fw_msg_not_handled(struct adapter *, const __be64 *);
 395 static int t4_sysctls(struct adapter *);
 396 static int cxgbe_sysctls(struct port_info *);
 397 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 398 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 399 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
 400 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 401 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 402 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 403 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 404 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 405 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
 406 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 407 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 408 #ifdef SBUF_DRAIN
 409 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 410 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 411 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 412 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
 413 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
 414 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 415 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 416 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 417 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 418 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 419 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 420 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 421 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
 422 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 423 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
 424 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 425 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 426 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 427 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 428 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 429 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 430 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
 431 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 432 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 433 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 434 #endif
 435 static inline void txq_start(struct ifnet *, struct sge_txq *);
 436 static uint32_t fconf_to_mode(uint32_t);
 437 static uint32_t mode_to_fconf(uint32_t);
 438 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 439 static int get_filter_mode(struct adapter *, uint32_t *);
 440 static int set_filter_mode(struct adapter *, uint32_t);
 441 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 442 static int get_filter(struct adapter *, struct t4_filter *);
 443 static int set_filter(struct adapter *, struct t4_filter *);
 444 static int del_filter(struct adapter *, struct t4_filter *);
 445 static void clear_filter(struct filter_entry *);
 446 static int set_filter_wr(struct adapter *, int);
 447 static int del_filter_wr(struct adapter *, int);
 448 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 449 static int load_fw(struct adapter *, struct t4_data *);
 450 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 451 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 452 static int set_sched_class(struct adapter *, struct t4_sched_params *);
 453 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
 454 #ifdef TCP_OFFLOAD
 455 static int toe_capability(struct port_info *, int);
 456 #endif
 457 static int mod_event(module_t, int, void *);
 458
 459 struct {
 460         uint16_t device;
 461         char *desc;
 462 } t4_pciids[] = {
 463         {0xa000, "Chelsio Terminator 4 FPGA"},
 464         {0x4400, "Chelsio T440-dbg"},
 465         {0x4401, "Chelsio T420-CR"},
 466         {0x4402, "Chelsio T422-CR"},
 467         {0x4403, "Chelsio T440-CR"},
 468         {0x4404, "Chelsio T420-BCH"},
 469         {0x4405, "Chelsio T440-BCH"},
 470         {0x4406, "Chelsio T440-CH"},
 471         {0x4407, "Chelsio T420-SO"},
 472         {0x4408, "Chelsio T420-CX"},
 473         {0x4409, "Chelsio T420-BT"},
 474         {0x440a, "Chelsio T404-BT"},
 475         {0x440e, "Chelsio T440-LP-CR"},
 476 }, t5_pciids[] = {
 477         {0xb000, "Chelsio Terminator 5 FPGA"},
 478         {0x5400, "Chelsio T580-dbg"},
 479         {0x5401,  "Chelsio T520-CR"},           /* 2 x 10G */
 480         {0x5402,  "Chelsio T522-CR"},           /* 2 x 10G, 2 X 1G */
 481         {0x5403,  "Chelsio T540-CR"},           /* 4 x 10G */
 482         {0x5407,  "Chelsio T520-SO"},           /* 2 x 10G, nomem */
 483         {0x5409,  "Chelsio T520-BT"},           /* 2 x 10GBaseT */
 484         {0x540a,  "Chelsio T504-BT"},           /* 4 x 1G */
 485         {0x540d,  "Chelsio T580-CR"},           /* 2 x 40G */
 486         {0x540e,  "Chelsio T540-LP-CR"},        /* 4 x 10G */
 487         {0x5410,  "Chelsio T580-LP-CR"},        /* 2 x 40G */
 488         {0x5411,  "Chelsio T520-LL-CR"},        /* 2 x 10G */
 489         {0x5412,  "Chelsio T560-CR"},           /* 1 x 40G, 2 x 10G */
 490         {0x5414,  "Chelsio T580-LP-SO-CR"},     /* 2 x 40G, nomem */
 491 #ifdef notyet
 492         {0x5404,  "Chelsio T520-BCH"},
 493         {0x5405,  "Chelsio T540-BCH"},
 494         {0x5406,  "Chelsio T540-CH"},
 495         {0x5408,  "Chelsio T520-CX"},
 496         {0x540b,  "Chelsio B520-SR"},
 497         {0x540c,  "Chelsio B504-BT"},
 498         {0x540f,  "Chelsio Amsterdam"},
 499         {0x5413,  "Chelsio T580-CHR"},
 500 #endif
 501 };
 502
 503 #ifdef TCP_OFFLOAD
 504 /*
 505  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 506  * exactly the same for both rxq and ofld_rxq.
 507  */
 508 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 509 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 510 #endif
 511
 512 /* No easy way to include t4_msg.h before adapter.h so we check this way */
 513 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
 514 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
 515
 516 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
 517
 518 static int
 519 t4_probe(device_t dev)
 520 {
 521         int i;
 522         uint16_t v = pci_get_vendor(dev);
 523         uint16_t d = pci_get_device(dev);
 524         uint8_t f = pci_get_function(dev);
 525
 526         if (v != PCI_VENDOR_ID_CHELSIO)
 527                 return (ENXIO);
 528
 529         /* Attach only to PF0 of the FPGA */
 530         if (d == 0xa000 && f != 0)
 531                 return (ENXIO);
 532
 533         for (i = 0; i < nitems(t4_pciids); i++) {
 534                 if (d == t4_pciids[i].device) {
 535                         device_set_desc(dev, t4_pciids[i].desc);
 536                         return (BUS_PROBE_DEFAULT);
 537                 }
 538         }
 539
 540         return (ENXIO);
 541 }
 542
 543 static int
 544 t5_probe(device_t dev)
 545 {
 546         int i;
 547         uint16_t v = pci_get_vendor(dev);
 548         uint16_t d = pci_get_device(dev);
 549         uint8_t f = pci_get_function(dev);
 550
 551         if (v != PCI_VENDOR_ID_CHELSIO)
 552                 return (ENXIO);
 553
 554         /* Attach only to PF0 of the FPGA */
 555         if (d == 0xb000 && f != 0)
 556                 return (ENXIO);
 557
 558         for (i = 0; i < nitems(t5_pciids); i++) {
 559                 if (d == t5_pciids[i].device) {
 560                         device_set_desc(dev, t5_pciids[i].desc);
 561                         return (BUS_PROBE_DEFAULT);
 562                 }
 563         }
 564
 565         return (ENXIO);
 566 }
 567
 568 static int
 569 t4_attach(device_t dev)
 570 {
 571         struct adapter *sc;
 572         int rc = 0, i, n10g, n1g, rqidx, tqidx;
 573         struct intrs_and_queues iaq;
 574         struct sge *s;
 575 #ifdef TCP_OFFLOAD
 576         int ofld_rqidx, ofld_tqidx;
 577 #endif
 578 #ifdef DEV_NETMAP
 579         int nm_rqidx, nm_tqidx;
 580 #endif
 581
 582         sc = device_get_softc(dev);
 583         sc->dev = dev;
 584
 585         pci_enable_busmaster(dev);
 586         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 587                 uint32_t v;
 588
 589                 pci_set_max_read_req(dev, 4096);
 590                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 591                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 592                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 593         }
 594
 595         sc->traceq = -1;
 596         mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
 597         snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
 598             device_get_nameunit(dev));
 599
 600         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 601             device_get_nameunit(dev));
 602         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 603         sx_xlock(&t4_list_lock);
 604         SLIST_INSERT_HEAD(&t4_list, sc, link);
 605         sx_xunlock(&t4_list_lock);
 606
 607         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 608         TAILQ_INIT(&sc->sfl);
 609         callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
 610
 611         rc = map_bars_0_and_4(sc);
 612         if (rc != 0)
 613                 goto done; /* error message displayed already */
 614
 615         /*
 616          * This is the real PF# to which we're attaching.  Works from within PCI
 617          * passthrough environments too, where pci_get_function() could return a
 618          * different PF# depending on the passthrough configuration.  We need to
 619          * use the real PF# in all our communication with the firmware.
 620          */
 621         sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 622         sc->mbox = sc->pf;
 623
 624         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 625         sc->an_handler = an_not_handled;
 626         for (i = 0; i < nitems(sc->cpl_handler); i++)
 627                 sc->cpl_handler[i] = cpl_not_handled;
 628         for (i = 0; i < nitems(sc->fw_msg_handler); i++)
 629                 sc->fw_msg_handler[i] = fw_msg_not_handled;
 630         t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 631         t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
 632         t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt);
 633         t4_init_sge_cpl_handlers(sc);
 634
 635         /* Prepare the adapter for operation */
 636         rc = -t4_prep_adapter(sc);
 637         if (rc != 0) {
 638                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 639                 goto done;
 640         }
 641
 642         /*
 643          * Do this really early, with the memory windows set up even before the
 644          * character device.  The userland tool's register i/o and mem read
 645          * will work even in "recovery mode".
 646          */
 647         setup_memwin(sc);
 648         sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
 649             device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
 650             device_get_nameunit(dev));
 651         if (sc->cdev == NULL)
 652                 device_printf(dev, "failed to create nexus char device.\n");
 653         else
 654                 sc->cdev->si_drv1 = sc;
 655
 656         /* Go no further if recovery mode has been requested. */
 657         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 658                 device_printf(dev, "recovery mode.\n");
 659                 goto done;
 660         }
 661
 662         /* Prepare the firmware for operation */
 663         rc = prep_firmware(sc);
 664         if (rc != 0)
 665                 goto done; /* error message displayed already */
 666
 667         rc = get_params__post_init(sc);
 668         if (rc != 0)
 669                 goto done; /* error message displayed already */
 670
 671         rc = set_params__post_init(sc);
 672         if (rc != 0)
 673                 goto done; /* error message displayed already */
 674
 675         rc = map_bar_2(sc);
 676         if (rc != 0)
 677                 goto done; /* error message displayed already */
 678
 679         rc = t4_create_dma_tag(sc);
 680         if (rc != 0)
 681                 goto done; /* error message displayed already */
 682
 683         /*
 684          * First pass over all the ports - allocate VIs and initialize some
 685          * basic parameters like mac address, port type, etc.  We also figure
 686          * out whether a port is 10G or 1G and use that information when
 687          * calculating how many interrupts to attempt to allocate.
 688          */
 689         n10g = n1g = 0;
 690         for_each_port(sc, i) {
 691                 struct port_info *pi;
 692
 693                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 694                 sc->port[i] = pi;
 695
 696                 /* These must be set before t4_port_init */
 697                 pi->adapter = sc;
 698                 pi->port_id = i;
 699
 700                 /* Allocate the vi and initialize parameters like mac addr */
 701                 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 702                 if (rc != 0) {
 703                         device_printf(dev, "unable to initialize port %d: %d\n",
 704                             i, rc);
 705                         free(pi, M_CXGBE);
 706                         sc->port[i] = NULL;
 707                         goto done;
 708                 }
 709
 710                 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX);
 711                 pi->link_cfg.requested_fc |= t4_pause_settings;
 712                 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX);
 713                 pi->link_cfg.fc |= t4_pause_settings;
 714
 715                 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
 716                 if (rc != 0) {
 717                         device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
 718                         free(pi, M_CXGBE);
 719                         sc->port[i] = NULL;
 720                         goto done;
 721                 }
 722
 723                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 724                     device_get_nameunit(dev), i);
 725                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 726                 sc->chan_map[pi->tx_chan] = i;
 727
 728                 if (is_10G_port(pi) || is_40G_port(pi)) {
 729                         n10g++;
 730                         pi->tmr_idx = t4_tmr_idx_10g;
 731                         pi->pktc_idx = t4_pktc_idx_10g;
 732                 } else {
 733                         n1g++;
 734                         pi->tmr_idx = t4_tmr_idx_1g;
 735                         pi->pktc_idx = t4_pktc_idx_1g;
 736                 }
 737
 738                 pi->xact_addr_filt = -1;
 739                 pi->linkdnrc = -1;
 740
 741                 pi->qsize_rxq = t4_qsize_rxq;
 742                 pi->qsize_txq = t4_qsize_txq;
 743
 744                 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
 745                 if (pi->dev == NULL) {
 746                         device_printf(dev,
 747                             "failed to add device for port %d.\n", i);
 748                         rc = ENXIO;
 749                         goto done;
 750                 }
 751                 device_set_softc(pi->dev, pi);
 752         }
 753
 754         /*
 755          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 756          */
 757         rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
 758         if (rc != 0)
 759                 goto done; /* error message displayed already */
 760
 761         sc->intr_type = iaq.intr_type;
 762         sc->intr_count = iaq.nirq;
 763
 764         s = &sc->sge;
 765         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 766         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 767         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
 768         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 769         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
 770 #ifdef TCP_OFFLOAD
 771         if (is_offload(sc)) {
 772                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 773                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 774                 s->neq += s->nofldtxq + s->nofldrxq;
 775                 s->niq += s->nofldrxq;
 776
 777                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 778                     M_CXGBE, M_ZERO | M_WAITOK);
 779                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 780                     M_CXGBE, M_ZERO | M_WAITOK);
 781         }
 782 #endif
 783 #ifdef DEV_NETMAP
 784         s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g;
 785         s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g;
 786         s->neq += s->nnmtxq + s->nnmrxq;
 787         s->niq += s->nnmrxq;
 788
 789         s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
 790             M_CXGBE, M_ZERO | M_WAITOK);
 791         s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
 792             M_CXGBE, M_ZERO | M_WAITOK);
 793 #endif
 794
 795         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 796             M_ZERO | M_WAITOK);
 797         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 798             M_ZERO | M_WAITOK);
 799         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 800             M_ZERO | M_WAITOK);
 801         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 802             M_ZERO | M_WAITOK);
 803         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 804             M_ZERO | M_WAITOK);
 805
 806         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 807             M_ZERO | M_WAITOK);
 808
 809         t4_init_l2t(sc, M_WAITOK);
 810
 811         /*
 812          * Second pass over the ports.  This time we know the number of rx and
 813          * tx queues that each port should get.
 814          */
 815         rqidx = tqidx = 0;
 816 #ifdef TCP_OFFLOAD
 817         ofld_rqidx = ofld_tqidx = 0;
 818 #endif
 819 #ifdef DEV_NETMAP
 820         nm_rqidx = nm_tqidx = 0;
 821 #endif
 822         for_each_port(sc, i) {
 823                 struct port_info *pi = sc->port[i];
 824
 825                 if (pi == NULL)
 826                         continue;
 827
 828                 pi->first_rxq = rqidx;
 829                 pi->first_txq = tqidx;
 830                 if (is_10G_port(pi) || is_40G_port(pi)) {
 831                         pi->flags |= iaq.intr_flags_10g;
 832                         pi->nrxq = iaq.nrxq10g;
 833                         pi->ntxq = iaq.ntxq10g;
 834                 } else {
 835                         pi->flags |= iaq.intr_flags_1g;
 836                         pi->nrxq = iaq.nrxq1g;
 837                         pi->ntxq = iaq.ntxq1g;
 838                 }
 839
 840                 if (pi->ntxq > 1)
 841                         pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
 842                 else
 843                         pi->rsrv_noflowq = 0;
 844
 845                 rqidx += pi->nrxq;
 846                 tqidx += pi->ntxq;
 847 #ifdef TCP_OFFLOAD
 848                 if (is_offload(sc)) {
 849                         pi->first_ofld_rxq = ofld_rqidx;
 850                         pi->first_ofld_txq = ofld_tqidx;
 851                         if (is_10G_port(pi) || is_40G_port(pi)) {
 852                                 pi->nofldrxq = iaq.nofldrxq10g;
 853                                 pi->nofldtxq = iaq.nofldtxq10g;
 854                         } else {
 855                                 pi->nofldrxq = iaq.nofldrxq1g;
 856                                 pi->nofldtxq = iaq.nofldtxq1g;
 857                         }
 858                         ofld_rqidx += pi->nofldrxq;
 859                         ofld_tqidx += pi->nofldtxq;
 860                 }
 861 #endif
 862 #ifdef DEV_NETMAP
 863                 pi->first_nm_rxq = nm_rqidx;
 864                 pi->first_nm_txq = nm_tqidx;
 865                 if (is_10G_port(pi) || is_40G_port(pi)) {
 866                         pi->nnmrxq = iaq.nnmrxq10g;
 867                         pi->nnmtxq = iaq.nnmtxq10g;
 868                 } else {
 869                         pi->nnmrxq = iaq.nnmrxq1g;
 870                         pi->nnmtxq = iaq.nnmtxq1g;
 871                 }
 872                 nm_rqidx += pi->nnmrxq;
 873                 nm_tqidx += pi->nnmtxq;
 874 #endif
 875         }
 876
 877         rc = setup_intr_handlers(sc);
 878         if (rc != 0) {
 879                 device_printf(dev,
 880                     "failed to setup interrupt handlers: %d\n", rc);
 881                 goto done;
 882         }
 883
 884         rc = bus_generic_attach(dev);
 885         if (rc != 0) {
 886                 device_printf(dev,
 887                     "failed to attach all child ports: %d\n", rc);
 888                 goto done;
 889         }
 890
 891         device_printf(dev,
 892             "PCIe x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
 893             sc->params.pci.width, sc->params.nports, sc->intr_count,
 894             sc->intr_type == INTR_MSIX ? "MSI-X" :
 895             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
 896             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
 897
 898         t4_set_desc(sc);
 899
 900 done:
 901         if (rc != 0 && sc->cdev) {
 902                 /* cdev was created and so cxgbetool works; recover that way. */
 903                 device_printf(dev,
 904                     "error during attach, adapter is now in recovery mode.\n");
 905                 rc = 0;
 906         }
 907
 908         if (rc != 0)
 909                 t4_detach(dev);
 910         else
 911                 t4_sysctls(sc);
 912
 913         return (rc);
 914 }
 915
 916 /*
 917  * Idempotent
 918  */
 919 static int
 920 t4_detach(device_t dev)
 921 {
 922         struct adapter *sc;
 923         struct port_info *pi;
 924         int i, rc;
 925
 926         sc = device_get_softc(dev);
 927
 928         if (sc->flags & FULL_INIT_DONE)
 929                 t4_intr_disable(sc);
 930
 931         if (sc->cdev) {
 932                 destroy_dev(sc->cdev);
 933                 sc->cdev = NULL;
 934         }
 935
 936         rc = bus_generic_detach(dev);
 937         if (rc) {
 938                 device_printf(dev,
 939                     "failed to detach child devices: %d\n", rc);
 940                 return (rc);
 941         }
 942
 943         for (i = 0; i < sc->intr_count; i++)
 944                 t4_free_irq(sc, &sc->irq[i]);
 945
 946         for (i = 0; i < MAX_NPORTS; i++) {
 947                 pi = sc->port[i];
 948                 if (pi) {
 949                         t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid);
 950                         if (pi->dev)
 951                                 device_delete_child(dev, pi->dev);
 952
 953                         mtx_destroy(&pi->pi_lock);
 954                         free(pi, M_CXGBE);
 955                 }
 956         }
 957
 958         if (sc->flags & FULL_INIT_DONE)
 959                 adapter_full_uninit(sc);
 960
 961         if (sc->flags & FW_OK)
 962                 t4_fw_bye(sc, sc->mbox);
 963
 964         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
 965                 pci_release_msi(dev);
 966
 967         if (sc->regs_res)
 968                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
 969                     sc->regs_res);
 970
 971         if (sc->udbs_res)
 972                 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
 973                     sc->udbs_res);
 974
 975         if (sc->msix_res)
 976                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
 977                     sc->msix_res);
 978
 979         if (sc->l2t)
 980                 t4_free_l2t(sc->l2t);
 981
 982 #ifdef TCP_OFFLOAD
 983         free(sc->sge.ofld_rxq, M_CXGBE);
 984         free(sc->sge.ofld_txq, M_CXGBE);
 985 #endif
 986 #ifdef DEV_NETMAP
 987         free(sc->sge.nm_rxq, M_CXGBE);
 988         free(sc->sge.nm_txq, M_CXGBE);
 989 #endif
 990         free(sc->irq, M_CXGBE);
 991         free(sc->sge.rxq, M_CXGBE);
 992         free(sc->sge.txq, M_CXGBE);
 993         free(sc->sge.ctrlq, M_CXGBE);
 994         free(sc->sge.iqmap, M_CXGBE);
 995         free(sc->sge.eqmap, M_CXGBE);
 996         free(sc->tids.ftid_tab, M_CXGBE);
 997         t4_destroy_dma_tag(sc);
 998         if (mtx_initialized(&sc->sc_lock)) {
 999                 sx_xlock(&t4_list_lock);
1000                 SLIST_REMOVE(&t4_list, sc, adapter, link);
1001                 sx_xunlock(&t4_list_lock);
1002                 mtx_destroy(&sc->sc_lock);
1003         }
1004
1005         if (mtx_initialized(&sc->tids.ftid_lock))
1006                 mtx_destroy(&sc->tids.ftid_lock);
1007         if (mtx_initialized(&sc->sfl_lock))
1008                 mtx_destroy(&sc->sfl_lock);
1009         if (mtx_initialized(&sc->ifp_lock))
1010                 mtx_destroy(&sc->ifp_lock);
1011
1012         bzero(sc, sizeof(*sc));
1013
1014         return (0);
1015 }
1016
1017 static int
1018 cxgbe_probe(device_t dev)
1019 {
1020         char buf[128];
1021         struct port_info *pi = device_get_softc(dev);
1022
1023         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
1024         device_set_desc_copy(dev, buf);
1025
1026         return (BUS_PROBE_DEFAULT);
1027 }
1028
1029 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
1030     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1031     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
1032 #define T4_CAP_ENABLE (T4_CAP)
1033
1034 static int
1035 cxgbe_attach(device_t dev)
1036 {
1037         struct port_info *pi = device_get_softc(dev);
1038         struct ifnet *ifp;
1039         char *s;
1040         int n, o;
1041
1042         /* Allocate an ifnet and set it up */
1043         ifp = if_alloc(IFT_ETHER);
1044         if (ifp == NULL) {
1045                 device_printf(dev, "Cannot allocate ifnet\n");
1046                 return (ENOMEM);
1047         }
1048         pi->ifp = ifp;
1049         ifp->if_softc = pi;
1050
1051         callout_init(&pi->tick, CALLOUT_MPSAFE);
1052
1053         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1054         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1055
1056         ifp->if_init = cxgbe_init;
1057         ifp->if_ioctl = cxgbe_ioctl;
1058         ifp->if_transmit = cxgbe_transmit;
1059         ifp->if_qflush = cxgbe_qflush;
1060
1061         ifp->if_capabilities = T4_CAP;
1062 #ifdef TCP_OFFLOAD
1063         if (is_offload(pi->adapter))
1064                 ifp->if_capabilities |= IFCAP_TOE;
1065 #endif
1066         ifp->if_capenable = T4_CAP_ENABLE;
1067         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1068             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1069
1070         ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1071         ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS;
1072         ifp->if_hw_tsomaxsegsize = 65536;
1073
1074         /* Initialize ifmedia for this port */
1075         ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1076             cxgbe_media_status);
1077         build_medialist(pi, &pi->media);
1078
1079         pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1080             EVENTHANDLER_PRI_ANY);
1081
1082         ether_ifattach(ifp, pi->hw_addr);
1083
1084         n = 128;
1085         s = malloc(n, M_CXGBE, M_WAITOK);
1086         o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq);
1087         MPASS(n > o);
1088 #ifdef TCP_OFFLOAD
1089         if (is_offload(pi->adapter)) {
1090                 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)",
1091                     pi->nofldtxq, pi->nofldrxq);
1092                 MPASS(n > o);
1093         }
1094 #endif
1095 #ifdef DEV_NETMAP
1096         o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq,
1097             pi->nnmrxq);
1098         MPASS(n > o);
1099 #endif
1100         device_printf(dev, "%s\n", s);
1101         free(s, M_CXGBE);
1102
1103 #ifdef DEV_NETMAP
1104         /* nm_media handled here to keep implementation private to this file */
1105         ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change,
1106             cxgbe_media_status);
1107         build_medialist(pi, &pi->nm_media);
1108         create_netmap_ifnet(pi);        /* logs errors it something fails */
1109 #endif
1110         cxgbe_sysctls(pi);
1111
1112         return (0);
1113 }
1114
1115 static int
1116 cxgbe_detach(device_t dev)
1117 {
1118         struct port_info *pi = device_get_softc(dev);
1119         struct adapter *sc = pi->adapter;
1120         struct ifnet *ifp = pi->ifp;
1121
1122         /* Tell if_ioctl and if_init that the port is going away */
1123         ADAPTER_LOCK(sc);
1124         SET_DOOMED(pi);
1125         wakeup(&sc->flags);
1126         while (IS_BUSY(sc))
1127                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
1128         SET_BUSY(sc);
1129 #ifdef INVARIANTS
1130         sc->last_op = "t4detach";
1131         sc->last_op_thr = curthread;
1132 #endif
1133         ADAPTER_UNLOCK(sc);
1134
1135         if (pi->flags & HAS_TRACEQ) {
1136                 sc->traceq = -1;        /* cloner should not create ifnet */
1137                 t4_tracer_port_detach(sc);
1138         }
1139
1140         if (pi->vlan_c)
1141                 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
1142
1143         PORT_LOCK(pi);
1144         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1145         callout_stop(&pi->tick);
1146         PORT_UNLOCK(pi);
1147         callout_drain(&pi->tick);
1148
1149         /* Let detach proceed even if these fail. */
1150         cxgbe_uninit_synchronized(pi);
1151         port_full_uninit(pi);
1152
1153         ifmedia_removeall(&pi->media);
1154         ether_ifdetach(pi->ifp);
1155         if_free(pi->ifp);
1156
1157 #ifdef DEV_NETMAP
1158         /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */
1159         destroy_netmap_ifnet(pi);
1160 #endif
1161
1162         ADAPTER_LOCK(sc);
1163         CLR_BUSY(sc);
1164         wakeup(&sc->flags);
1165         ADAPTER_UNLOCK(sc);
1166
1167         return (0);
1168 }
1169
1170 static void
1171 cxgbe_init(void *arg)
1172 {
1173         struct port_info *pi = arg;
1174         struct adapter *sc = pi->adapter;
1175
1176         if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
1177                 return;
1178         cxgbe_init_synchronized(pi);
1179         end_synchronized_op(sc, 0);
1180 }
1181
1182 static int
1183 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1184 {
1185         int rc = 0, mtu, flags, can_sleep;
1186         struct port_info *pi = ifp->if_softc;
1187         struct adapter *sc = pi->adapter;
1188         struct ifreq *ifr = (struct ifreq *)data;
1189         uint32_t mask;
1190
1191         switch (cmd) {
1192         case SIOCSIFMTU:
1193                 mtu = ifr->ifr_mtu;
1194                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1195                         return (EINVAL);
1196
1197                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
1198                 if (rc)
1199                         return (rc);
1200                 ifp->if_mtu = mtu;
1201                 if (pi->flags & PORT_INIT_DONE) {
1202                         t4_update_fl_bufsize(ifp);
1203                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1204                                 rc = update_mac_settings(ifp, XGMAC_MTU);
1205                 }
1206                 end_synchronized_op(sc, 0);
1207                 break;
1208
1209         case SIOCSIFFLAGS:
1210                 can_sleep = 0;
1211 redo_sifflags:
1212                 rc = begin_synchronized_op(sc, pi,
1213                     can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg");
1214                 if (rc)
1215                         return (rc);
1216
1217                 if (ifp->if_flags & IFF_UP) {
1218                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1219                                 flags = pi->if_flags;
1220                                 if ((ifp->if_flags ^ flags) &
1221                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1222                                         if (can_sleep == 1) {
1223                                                 end_synchronized_op(sc, 0);
1224                                                 can_sleep = 0;
1225                                                 goto redo_sifflags;
1226                                         }
1227                                         rc = update_mac_settings(ifp,
1228                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1229                                 }
1230                         } else {
1231                                 if (can_sleep == 0) {
1232                                         end_synchronized_op(sc, LOCK_HELD);
1233                                         can_sleep = 1;
1234                                         goto redo_sifflags;
1235                                 }
1236                                 rc = cxgbe_init_synchronized(pi);
1237                         }
1238                         pi->if_flags = ifp->if_flags;
1239                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1240                         if (can_sleep == 0) {
1241                                 end_synchronized_op(sc, LOCK_HELD);
1242                                 can_sleep = 1;
1243                                 goto redo_sifflags;
1244                         }
1245                         rc = cxgbe_uninit_synchronized(pi);
1246                 }
1247                 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD);
1248                 break;
1249
1250         case SIOCADDMULTI:
1251         case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1252                 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
1253                 if (rc)
1254                         return (rc);
1255                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1256                         rc = update_mac_settings(ifp, XGMAC_MCADDRS);
1257                 end_synchronized_op(sc, LOCK_HELD);
1258                 break;
1259
1260         case SIOCSIFCAP:
1261                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
1262                 if (rc)
1263                         return (rc);
1264
1265                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1266                 if (mask & IFCAP_TXCSUM) {
1267                         ifp->if_capenable ^= IFCAP_TXCSUM;
1268                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1269
1270                         if (IFCAP_TSO4 & ifp->if_capenable &&
1271                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1272                                 ifp->if_capenable &= ~IFCAP_TSO4;
1273                                 if_printf(ifp,
1274                                     "tso4 disabled due to -txcsum.\n");
1275                         }
1276                 }
1277                 if (mask & IFCAP_TXCSUM_IPV6) {
1278                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1279                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1280
1281                         if (IFCAP_TSO6 & ifp->if_capenable &&
1282                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1283                                 ifp->if_capenable &= ~IFCAP_TSO6;
1284                                 if_printf(ifp,
1285                                     "tso6 disabled due to -txcsum6.\n");
1286                         }
1287                 }
1288                 if (mask & IFCAP_RXCSUM)
1289                         ifp->if_capenable ^= IFCAP_RXCSUM;
1290                 if (mask & IFCAP_RXCSUM_IPV6)
1291                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1292
1293                 /*
1294                  * Note that we leave CSUM_TSO alone (it is always set).  The
1295                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1296                  * sending a TSO request our way, so it's sufficient to toggle
1297                  * IFCAP_TSOx only.
1298                  */
1299                 if (mask & IFCAP_TSO4) {
1300                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1301                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1302                                 if_printf(ifp, "enable txcsum first.\n");
1303                                 rc = EAGAIN;
1304                                 goto fail;
1305                         }
1306                         ifp->if_capenable ^= IFCAP_TSO4;
1307                 }
1308                 if (mask & IFCAP_TSO6) {
1309                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1310                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1311                                 if_printf(ifp, "enable txcsum6 first.\n");
1312                                 rc = EAGAIN;
1313                                 goto fail;
1314                         }
1315                         ifp->if_capenable ^= IFCAP_TSO6;
1316                 }
1317                 if (mask & IFCAP_LRO) {
1318 #if defined(INET) || defined(INET6)
1319                         int i;
1320                         struct sge_rxq *rxq;
1321
1322                         ifp->if_capenable ^= IFCAP_LRO;
1323                         for_each_rxq(pi, i, rxq) {
1324                                 if (ifp->if_capenable & IFCAP_LRO)
1325                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1326                                 else
1327                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1328                         }
1329 #endif
1330                 }
1331 #ifdef TCP_OFFLOAD
1332                 if (mask & IFCAP_TOE) {
1333                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1334
1335                         rc = toe_capability(pi, enable);
1336                         if (rc != 0)
1337                                 goto fail;
1338
1339                         ifp->if_capenable ^= mask;
1340                 }
1341 #endif
1342                 if (mask & IFCAP_VLAN_HWTAGGING) {
1343                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1344                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1345                                 rc = update_mac_settings(ifp, XGMAC_VLANEX);
1346                 }
1347                 if (mask & IFCAP_VLAN_MTU) {
1348                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1349
1350                         /* Need to find out how to disable auto-mtu-inflation */
1351                 }
1352                 if (mask & IFCAP_VLAN_HWTSO)
1353                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1354                 if (mask & IFCAP_VLAN_HWCSUM)
1355                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1356
1357 #ifdef VLAN_CAPABILITIES
1358                 VLAN_CAPABILITIES(ifp);
1359 #endif
1360 fail:
1361                 end_synchronized_op(sc, 0);
1362                 break;
1363
1364         case SIOCSIFMEDIA:
1365         case SIOCGIFMEDIA:
1366                 ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1367                 break;
1368
1369         default:
1370                 rc = ether_ioctl(ifp, cmd, data);
1371         }
1372
1373         return (rc);
1374 }
1375
1376 static int
1377 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1378 {
1379         struct port_info *pi = ifp->if_softc;
1380         struct adapter *sc = pi->adapter;
1381         struct sge_txq *txq = &sc->sge.txq[pi->first_txq];
1382         struct buf_ring *br;
1383         int rc;
1384
1385         M_ASSERTPKTHDR(m);
1386
1387         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1388                 m_freem(m);
1389                 return (ENETDOWN);
1390         }
1391
1392         if (m->m_flags & M_FLOWID)
1393                 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq))
1394                     + pi->rsrv_noflowq);
1395         br = txq->br;
1396
1397         if (TXQ_TRYLOCK(txq) == 0) {
1398                 struct sge_eq *eq = &txq->eq;
1399
1400                 /*
1401                  * It is possible that t4_eth_tx finishes up and releases the
1402                  * lock between the TRYLOCK above and the drbr_enqueue here.  We
1403                  * need to make sure that this mbuf doesn't just sit there in
1404                  * the drbr.
1405                  */
1406
1407                 rc = drbr_enqueue(ifp, br, m);
1408                 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 &&
1409                     !(eq->flags & EQ_DOOMED))
1410                         callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq);
1411                 return (rc);
1412         }
1413
1414         /*
1415          * txq->m is the mbuf that is held up due to a temporary shortage of
1416          * resources and it should be put on the wire first.  Then what's in
1417          * drbr and finally the mbuf that was just passed in to us.
1418          *
1419          * Return code should indicate the fate of the mbuf that was passed in
1420          * this time.
1421          */
1422
1423         TXQ_LOCK_ASSERT_OWNED(txq);
1424         if (drbr_needs_enqueue(ifp, br) || txq->m) {
1425
1426                 /* Queued for transmission. */
1427
1428                 rc = drbr_enqueue(ifp, br, m);
1429                 m = txq->m ? txq->m : drbr_dequeue(ifp, br);
1430                 (void) t4_eth_tx(ifp, txq, m);
1431                 TXQ_UNLOCK(txq);
1432                 return (rc);
1433         }
1434
1435         /* Direct transmission. */
1436         rc = t4_eth_tx(ifp, txq, m);
1437         if (rc != 0 && txq->m)
1438                 rc = 0; /* held, will be transmitted soon (hopefully) */
1439
1440         TXQ_UNLOCK(txq);
1441         return (rc);
1442 }
1443
1444 static void
1445 cxgbe_qflush(struct ifnet *ifp)
1446 {
1447         struct port_info *pi = ifp->if_softc;
1448         struct sge_txq *txq;
1449         int i;
1450         struct mbuf *m;
1451
1452         /* queues do not exist if !PORT_INIT_DONE. */
1453         if (pi->flags & PORT_INIT_DONE) {
1454                 for_each_txq(pi, i, txq) {
1455                         TXQ_LOCK(txq);
1456                         m_freem(txq->m);
1457                         txq->m = NULL;
1458                         while ((m = buf_ring_dequeue_sc(txq->br)) != NULL)
1459                                 m_freem(m);
1460                         TXQ_UNLOCK(txq);
1461                 }
1462         }
1463         if_qflush(ifp);
1464 }
1465
1466 static int
1467 cxgbe_media_change(struct ifnet *ifp)
1468 {
1469         struct port_info *pi = ifp->if_softc;
1470
1471         device_printf(pi->dev, "%s unimplemented.\n", __func__);
1472
1473         return (EOPNOTSUPP);
1474 }
1475
1476 static void
1477 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1478 {
1479         struct port_info *pi = ifp->if_softc;
1480         struct ifmedia *media = NULL;
1481         struct ifmedia_entry *cur;
1482         int speed = pi->link_cfg.speed;
1483         int data = (pi->port_type << 8) | pi->mod_type;
1484
1485         if (ifp == pi->ifp)
1486                 media = &pi->media;
1487 #ifdef DEV_NETMAP
1488         else if (ifp == pi->nm_ifp)
1489                 media = &pi->nm_media;
1490 #endif
1491         MPASS(media != NULL);
1492
1493         cur = media->ifm_cur;
1494         if (cur->ifm_data != data) {
1495                 build_medialist(pi, media);
1496                 cur = media->ifm_cur;
1497         }
1498
1499         ifmr->ifm_status = IFM_AVALID;
1500         if (!pi->link_cfg.link_ok)
1501                 return;
1502
1503         ifmr->ifm_status |= IFM_ACTIVE;
1504
1505         /* active and current will differ iff current media is autoselect. */
1506         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1507                 return;
1508
1509         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1510         if (speed == SPEED_10000)
1511                 ifmr->ifm_active |= IFM_10G_T;
1512         else if (speed == SPEED_1000)
1513                 ifmr->ifm_active |= IFM_1000_T;
1514         else if (speed == SPEED_100)
1515                 ifmr->ifm_active |= IFM_100_TX;
1516         else if (speed == SPEED_10)
1517                 ifmr->ifm_active |= IFM_10_T;
1518         else
1519                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1520                             speed));
1521 }
1522
1523 void
1524 t4_fatal_err(struct adapter *sc)
1525 {
1526         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1527         t4_intr_disable(sc);
1528         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1529             device_get_nameunit(sc->dev));
1530 }
1531
1532 static int
1533 map_bars_0_and_4(struct adapter *sc)
1534 {
1535         sc->regs_rid = PCIR_BAR(0);
1536         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1537             &sc->regs_rid, RF_ACTIVE);
1538         if (sc->regs_res == NULL) {
1539                 device_printf(sc->dev, "cannot map registers.\n");
1540                 return (ENXIO);
1541         }
1542         sc->bt = rman_get_bustag(sc->regs_res);
1543         sc->bh = rman_get_bushandle(sc->regs_res);
1544         sc->mmio_len = rman_get_size(sc->regs_res);
1545         setbit(&sc->doorbells, DOORBELL_KDB);
1546
1547         sc->msix_rid = PCIR_BAR(4);
1548         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1549             &sc->msix_rid, RF_ACTIVE);
1550         if (sc->msix_res == NULL) {
1551                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1552                 return (ENXIO);
1553         }
1554
1555         return (0);
1556 }
1557
1558 static int
1559 map_bar_2(struct adapter *sc)
1560 {
1561
1562         /*
1563          * T4: only iWARP driver uses the userspace doorbells.  There is no need
1564          * to map it if RDMA is disabled.
1565          */
1566         if (is_t4(sc) && sc->rdmacaps == 0)
1567                 return (0);
1568
1569         sc->udbs_rid = PCIR_BAR(2);
1570         sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1571             &sc->udbs_rid, RF_ACTIVE);
1572         if (sc->udbs_res == NULL) {
1573                 device_printf(sc->dev, "cannot map doorbell BAR.\n");
1574                 return (ENXIO);
1575         }
1576         sc->udbs_base = rman_get_virtual(sc->udbs_res);
1577
1578         if (is_t5(sc)) {
1579                 setbit(&sc->doorbells, DOORBELL_UDB);
1580 #if defined(__i386__) || defined(__amd64__)
1581                 if (t5_write_combine) {
1582                         int rc;
1583
1584                         /*
1585                          * Enable write combining on BAR2.  This is the
1586                          * userspace doorbell BAR and is split into 128B
1587                          * (UDBS_SEG_SIZE) doorbell regions, each associated
1588                          * with an egress queue.  The first 64B has the doorbell
1589                          * and the second 64B can be used to submit a tx work
1590                          * request with an implicit doorbell.
1591                          */
1592
1593                         rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
1594                             rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
1595                         if (rc == 0) {
1596                                 clrbit(&sc->doorbells, DOORBELL_UDB);
1597                                 setbit(&sc->doorbells, DOORBELL_WCWR);
1598                                 setbit(&sc->doorbells, DOORBELL_UDBWC);
1599                         } else {
1600                                 device_printf(sc->dev,
1601                                     "couldn't enable write combining: %d\n",
1602                                     rc);
1603                         }
1604
1605                         t4_write_reg(sc, A_SGE_STAT_CFG,
1606                             V_STATSOURCE_T5(7) | V_STATMODE(0));
1607                 }
1608 #endif
1609         }
1610
1611         return (0);
1612 }
1613
1614 static const struct memwin t4_memwin[] = {
1615         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1616         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1617         { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
1618 };
1619
1620 static const struct memwin t5_memwin[] = {
1621         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1622         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1623         { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1624 };
1625
1626 static void
1627 setup_memwin(struct adapter *sc)
1628 {
1629         const struct memwin *mw;
1630         int i, n;
1631         uint32_t bar0;
1632
1633         if (is_t4(sc)) {
1634                 /*
1635                  * Read low 32b of bar0 indirectly via the hardware backdoor
1636                  * mechanism.  Works from within PCI passthrough environments
1637                  * too, where rman_get_start() can return a different value.  We
1638                  * need to program the T4 memory window decoders with the actual
1639                  * addresses that will be coming across the PCIe link.
1640                  */
1641                 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1642                 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1643
1644                 mw = &t4_memwin[0];
1645                 n = nitems(t4_memwin);
1646         } else {
1647                 /* T5 uses the relative offset inside the PCIe BAR */
1648                 bar0 = 0;
1649
1650                 mw = &t5_memwin[0];
1651                 n = nitems(t5_memwin);
1652         }
1653
1654         for (i = 0; i < n; i++, mw++) {
1655                 t4_write_reg(sc,
1656                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
1657                     (mw->base + bar0) | V_BIR(0) |
1658                     V_WINDOW(ilog2(mw->aperture) - 10));
1659         }
1660
1661         /* flush */
1662         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1663 }
1664
1665 /*
1666  * Verify that the memory range specified by the addr/len pair is valid and lies
1667  * entirely within a single region (EDCx or MCx).
1668  */
1669 static int
1670 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
1671 {
1672         uint32_t em, addr_len, maddr, mlen;
1673
1674         /* Memory can only be accessed in naturally aligned 4 byte units */
1675         if (addr & 3 || len & 3 || len == 0)
1676                 return (EINVAL);
1677
1678         /* Enabled memories */
1679         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1680         if (em & F_EDRAM0_ENABLE) {
1681                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1682                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1683                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1684                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1685                     addr + len <= maddr + mlen)
1686                         return (0);
1687         }
1688         if (em & F_EDRAM1_ENABLE) {
1689                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1690                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1691                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1692                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1693                     addr + len <= maddr + mlen)
1694                         return (0);
1695         }
1696         if (em & F_EXT_MEM_ENABLE) {
1697                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1698                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1699                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1700                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1701                     addr + len <= maddr + mlen)
1702                         return (0);
1703         }
1704         if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
1705                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1706                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1707                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1708                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1709                     addr + len <= maddr + mlen)
1710                         return (0);
1711         }
1712
1713         return (EFAULT);
1714 }
1715
1716 static int
1717 fwmtype_to_hwmtype(int mtype)
1718 {
1719
1720         switch (mtype) {
1721         case FW_MEMTYPE_EDC0:
1722                 return (MEM_EDC0);
1723         case FW_MEMTYPE_EDC1:
1724                 return (MEM_EDC1);
1725         case FW_MEMTYPE_EXTMEM:
1726                 return (MEM_MC0);
1727         case FW_MEMTYPE_EXTMEM1:
1728                 return (MEM_MC1);
1729         default:
1730                 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
1731         }
1732 }
1733
1734 /*
1735  * Verify that the memory range specified by the memtype/offset/len pair is
1736  * valid and lies entirely within the memtype specified.  The global address of
1737  * the start of the range is returned in addr.
1738  */
1739 static int
1740 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1741     uint32_t *addr)
1742 {
1743         uint32_t em, addr_len, maddr, mlen;
1744
1745         /* Memory can only be accessed in naturally aligned 4 byte units */
1746         if (off & 3 || len & 3 || len == 0)
1747                 return (EINVAL);
1748
1749         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1750         switch (fwmtype_to_hwmtype(mtype)) {
1751         case MEM_EDC0:
1752                 if (!(em & F_EDRAM0_ENABLE))
1753                         return (EINVAL);
1754                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1755                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1756                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1757                 break;
1758         case MEM_EDC1:
1759                 if (!(em & F_EDRAM1_ENABLE))
1760                         return (EINVAL);
1761                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1762                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1763                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1764                 break;
1765         case MEM_MC:
1766                 if (!(em & F_EXT_MEM_ENABLE))
1767                         return (EINVAL);
1768                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1769                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1770                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1771                 break;
1772         case MEM_MC1:
1773                 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
1774                         return (EINVAL);
1775                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1776                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1777                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1778                 break;
1779         default:
1780                 return (EINVAL);
1781         }
1782
1783         if (mlen > 0 && off < mlen && off + len <= mlen) {
1784                 *addr = maddr + off;    /* global address */
1785                 return (0);
1786         }
1787
1788         return (EFAULT);
1789 }
1790
1791 static void
1792 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1793 {
1794         const struct memwin *mw;
1795
1796         if (is_t4(sc)) {
1797                 KASSERT(win >= 0 && win < nitems(t4_memwin),
1798                     ("%s: incorrect memwin# (%d)", __func__, win));
1799                 mw = &t4_memwin[win];
1800         } else {
1801                 KASSERT(win >= 0 && win < nitems(t5_memwin),
1802                     ("%s: incorrect memwin# (%d)", __func__, win));
1803                 mw = &t5_memwin[win];
1804         }
1805
1806         if (base != NULL)
1807                 *base = mw->base;
1808         if (aperture != NULL)
1809                 *aperture = mw->aperture;
1810 }
1811
1812 /*
1813  * Positions the memory window such that it can be used to access the specified
1814  * address in the chip's address space.  The return value is the offset of addr
1815  * from the start of the window.
1816  */
1817 static uint32_t
1818 position_memwin(struct adapter *sc, int n, uint32_t addr)
1819 {
1820         uint32_t start, pf;
1821         uint32_t reg;
1822
1823         KASSERT(n >= 0 && n <= 3,
1824             ("%s: invalid window %d.", __func__, n));
1825         KASSERT((addr & 3) == 0,
1826             ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
1827
1828         if (is_t4(sc)) {
1829                 pf = 0;
1830                 start = addr & ~0xf;    /* start must be 16B aligned */
1831         } else {
1832                 pf = V_PFNUM(sc->pf);
1833                 start = addr & ~0x7f;   /* start must be 128B aligned */
1834         }
1835         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1836
1837         t4_write_reg(sc, reg, start | pf);
1838         t4_read_reg(sc, reg);
1839
1840         return (addr - start);
1841 }
1842
1843 static int
1844 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1845     struct intrs_and_queues *iaq)
1846 {
1847         int rc, itype, navail, nrxq10g, nrxq1g, n;
1848         int nofldrxq10g = 0, nofldrxq1g = 0;
1849         int nnmrxq10g = 0, nnmrxq1g = 0;
1850
1851         bzero(iaq, sizeof(*iaq));
1852
1853         iaq->ntxq10g = t4_ntxq10g;
1854         iaq->ntxq1g = t4_ntxq1g;
1855         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1856         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1857         iaq->rsrv_noflowq = t4_rsrv_noflowq;
1858 #ifdef TCP_OFFLOAD
1859         if (is_offload(sc)) {
1860                 iaq->nofldtxq10g = t4_nofldtxq10g;
1861                 iaq->nofldtxq1g = t4_nofldtxq1g;
1862                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1863                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1864         }
1865 #endif
1866 #ifdef DEV_NETMAP
1867         iaq->nnmtxq10g = t4_nnmtxq10g;
1868         iaq->nnmtxq1g = t4_nnmtxq1g;
1869         iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g;
1870         iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g;
1871 #endif
1872
1873         for (itype = INTR_MSIX; itype; itype >>= 1) {
1874
1875                 if ((itype & t4_intr_types) == 0)
1876                         continue;       /* not allowed */
1877
1878                 if (itype == INTR_MSIX)
1879                         navail = pci_msix_count(sc->dev);
1880                 else if (itype == INTR_MSI)
1881                         navail = pci_msi_count(sc->dev);
1882                 else
1883                         navail = 1;
1884 restart:
1885                 if (navail == 0)
1886                         continue;
1887
1888                 iaq->intr_type = itype;
1889                 iaq->intr_flags_10g = 0;
1890                 iaq->intr_flags_1g = 0;
1891
1892                 /*
1893                  * Best option: an interrupt vector for errors, one for the
1894                  * firmware event queue, and one for every rxq (NIC, TOE, and
1895                  * netmap).
1896                  */
1897                 iaq->nirq = T4_EXTRA_INTR;
1898                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g);
1899                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g);
1900                 if (iaq->nirq <= navail &&
1901                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
1902                         iaq->intr_flags_10g = INTR_ALL;
1903                         iaq->intr_flags_1g = INTR_ALL;
1904                         goto allocate;
1905                 }
1906
1907                 /*
1908                  * Second best option: a vector for errors, one for the firmware
1909                  * event queue, and vectors for either all the NIC rx queues or
1910                  * all the TOE rx queues.  The queues that don't get vectors
1911                  * will forward their interrupts to those that do.
1912                  *
1913                  * Note: netmap rx queues cannot be created early and so they
1914                  * can't be setup to receive forwarded interrupts for others.
1915                  */
1916                 iaq->nirq = T4_EXTRA_INTR;
1917                 if (nrxq10g >= nofldrxq10g) {
1918                         iaq->intr_flags_10g = INTR_RXQ;
1919                         iaq->nirq += n10g * nrxq10g;
1920 #ifdef DEV_NETMAP
1921                         iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g);
1922 #endif
1923                 } else {
1924                         iaq->intr_flags_10g = INTR_OFLD_RXQ;
1925                         iaq->nirq += n10g * nofldrxq10g;
1926 #ifdef DEV_NETMAP
1927                         iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g);
1928 #endif
1929                 }
1930                 if (nrxq1g >= nofldrxq1g) {
1931                         iaq->intr_flags_1g = INTR_RXQ;
1932                         iaq->nirq += n1g * nrxq1g;
1933 #ifdef DEV_NETMAP
1934                         iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g);
1935 #endif
1936                 } else {
1937                         iaq->intr_flags_1g = INTR_OFLD_RXQ;
1938                         iaq->nirq += n1g * nofldrxq1g;
1939 #ifdef DEV_NETMAP
1940                         iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g);
1941 #endif
1942                 }
1943                 if (iaq->nirq <= navail &&
1944                     (itype != INTR_MSI || powerof2(iaq->nirq)))
1945                         goto allocate;
1946
1947                 /*
1948                  * Next best option: an interrupt vector for errors, one for the
1949                  * firmware event queue, and at least one per port.  At this
1950                  * point we know we'll have to downsize nrxq and/or nofldrxq
1951                  * and/or nnmrxq to fit what's available to us.
1952                  */
1953                 iaq->nirq = T4_EXTRA_INTR;
1954                 iaq->nirq += n10g + n1g;
1955                 if (iaq->nirq <= navail) {
1956                         int leftover = navail - iaq->nirq;
1957
1958                         if (n10g > 0) {
1959                                 int target = max(nrxq10g, nofldrxq10g);
1960
1961                                 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
1962                                     INTR_RXQ : INTR_OFLD_RXQ;
1963
1964                                 n = 1;
1965                                 while (n < target && leftover >= n10g) {
1966                                         leftover -= n10g;
1967                                         iaq->nirq += n10g;
1968                                         n++;
1969                                 }
1970                                 iaq->nrxq10g = min(n, nrxq10g);
1971 #ifdef TCP_OFFLOAD
1972                                 iaq->nofldrxq10g = min(n, nofldrxq10g);
1973 #endif
1974 #ifdef DEV_NETMAP
1975                                 iaq->nnmrxq10g = min(n, nnmrxq10g);
1976 #endif
1977                         }
1978
1979                         if (n1g > 0) {
1980                                 int target = max(nrxq1g, nofldrxq1g);
1981
1982                                 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
1983                                     INTR_RXQ : INTR_OFLD_RXQ;
1984
1985                                 n = 1;
1986                                 while (n < target && leftover >= n1g) {
1987                                         leftover -= n1g;
1988                                         iaq->nirq += n1g;
1989                                         n++;
1990                                 }
1991                                 iaq->nrxq1g = min(n, nrxq1g);
1992 #ifdef TCP_OFFLOAD
1993                                 iaq->nofldrxq1g = min(n, nofldrxq1g);
1994 #endif
1995 #ifdef DEV_NETMAP
1996                                 iaq->nnmrxq1g = min(n, nnmrxq1g);
1997 #endif
1998                         }
1999
2000                         if (itype != INTR_MSI || powerof2(iaq->nirq))
2001                                 goto allocate;
2002                 }
2003
2004                 /*
2005                  * Least desirable option: one interrupt vector for everything.
2006                  */
2007                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
2008                 iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
2009 #ifdef TCP_OFFLOAD
2010                 if (is_offload(sc))
2011                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2012 #endif
2013 #ifdef DEV_NETMAP
2014                 iaq->nnmrxq10g = iaq->nnmrxq1g = 1;
2015 #endif
2016
2017 allocate:
2018                 navail = iaq->nirq;
2019                 rc = 0;
2020                 if (itype == INTR_MSIX)
2021                         rc = pci_alloc_msix(sc->dev, &navail);
2022                 else if (itype == INTR_MSI)
2023                         rc = pci_alloc_msi(sc->dev, &navail);
2024
2025                 if (rc == 0) {
2026                         if (navail == iaq->nirq)
2027                                 return (0);
2028
2029                         /*
2030                          * Didn't get the number requested.  Use whatever number
2031                          * the kernel is willing to allocate (it's in navail).
2032                          */
2033                         device_printf(sc->dev, "fewer vectors than requested, "
2034                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
2035                             itype, iaq->nirq, navail);
2036                         pci_release_msi(sc->dev);
2037                         goto restart;
2038                 }
2039
2040                 device_printf(sc->dev,
2041                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
2042                     itype, rc, iaq->nirq, navail);
2043         }
2044
2045         device_printf(sc->dev,
2046             "failed to find a usable interrupt type.  "
2047             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
2048             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
2049
2050         return (ENXIO);
2051 }
2052
2053 #define FW_VERSION(chip) ( \
2054     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
2055     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
2056     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
2057     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
2058 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
2059
2060 struct fw_info {
2061         uint8_t chip;
2062         char *kld_name;
2063         char *fw_mod_name;
2064         struct fw_hdr fw_hdr;   /* XXX: waste of space, need a sparse struct */
2065 } fw_info[] = {
2066         {
2067                 .chip = CHELSIO_T4,
2068                 .kld_name = "t4fw_cfg",
2069                 .fw_mod_name = "t4fw",
2070                 .fw_hdr = {
2071                         .chip = FW_HDR_CHIP_T4,
2072                         .fw_ver = htobe32_const(FW_VERSION(T4)),
2073                         .intfver_nic = FW_INTFVER(T4, NIC),
2074                         .intfver_vnic = FW_INTFVER(T4, VNIC),
2075                         .intfver_ofld = FW_INTFVER(T4, OFLD),
2076                         .intfver_ri = FW_INTFVER(T4, RI),
2077                         .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
2078                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
2079                         .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
2080                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
2081                 },
2082         }, {
2083                 .chip = CHELSIO_T5,
2084                 .kld_name = "t5fw_cfg",
2085                 .fw_mod_name = "t5fw",
2086                 .fw_hdr = {
2087                         .chip = FW_HDR_CHIP_T5,
2088                         .fw_ver = htobe32_const(FW_VERSION(T5)),
2089                         .intfver_nic = FW_INTFVER(T5, NIC),
2090                         .intfver_vnic = FW_INTFVER(T5, VNIC),
2091                         .intfver_ofld = FW_INTFVER(T5, OFLD),
2092                         .intfver_ri = FW_INTFVER(T5, RI),
2093                         .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
2094                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
2095                         .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
2096                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
2097                 },
2098         }
2099 };
2100
2101 static struct fw_info *
2102 find_fw_info(int chip)
2103 {
2104         int i;
2105
2106         for (i = 0; i < nitems(fw_info); i++) {
2107                 if (fw_info[i].chip == chip)
2108                         return (&fw_info[i]);
2109         }
2110         return (NULL);
2111 }
2112
2113 /*
2114  * Is the given firmware API compatible with the one the driver was compiled
2115  * with?
2116  */
2117 static int
2118 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
2119 {
2120
2121         /* short circuit if it's the exact same firmware version */
2122         if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
2123                 return (1);
2124
2125         /*
2126          * XXX: Is this too conservative?  Perhaps I should limit this to the
2127          * features that are supported in the driver.
2128          */
2129 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
2130         if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
2131             SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
2132             SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
2133                 return (1);
2134 #undef SAME_INTF
2135
2136         return (0);
2137 }
2138
2139 /*
2140  * The firmware in the KLD is usable, but should it be installed?  This routine
2141  * explains itself in detail if it indicates the KLD firmware should be
2142  * installed.
2143  */
2144 static int
2145 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
2146 {
2147         const char *reason;
2148
2149         if (!card_fw_usable) {
2150                 reason = "incompatible or unusable";
2151                 goto install;
2152         }
2153
2154         if (k > c) {
2155                 reason = "older than the version bundled with this driver";
2156                 goto install;
2157         }
2158
2159         if (t4_fw_install == 2 && k != c) {
2160                 reason = "different than the version bundled with this driver";
2161                 goto install;
2162         }
2163
2164         return (0);
2165
2166 install:
2167         if (t4_fw_install == 0) {
2168                 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2169                     "but the driver is prohibited from installing a different "
2170                     "firmware on the card.\n",
2171                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2172                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2173
2174                 return (0);
2175         }
2176
2177         device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2178             "installing firmware %u.%u.%u.%u on card.\n",
2179             G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2180             G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2181             G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2182             G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2183
2184         return (1);
2185 }
2186 /*
2187  * Establish contact with the firmware and determine if we are the master driver
2188  * or not, and whether we are responsible for chip initialization.
2189  */
2190 static int
2191 prep_firmware(struct adapter *sc)
2192 {
2193         const struct firmware *fw = NULL, *default_cfg;
2194         int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2195         enum dev_state state;
2196         struct fw_info *fw_info;
2197         struct fw_hdr *card_fw;         /* fw on the card */
2198         const struct fw_hdr *kld_fw;    /* fw in the KLD */
2199         const struct fw_hdr *drv_fw;    /* fw header the driver was compiled
2200                                            against */
2201
2202         /* Contact firmware. */
2203         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2204         if (rc < 0 || state == DEV_STATE_ERR) {
2205                 rc = -rc;
2206                 device_printf(sc->dev,
2207                     "failed to connect to the firmware: %d, %d.\n", rc, state);
2208                 return (rc);
2209         }
2210         pf = rc;
2211         if (pf == sc->mbox)
2212                 sc->flags |= MASTER_PF;
2213         else if (state == DEV_STATE_UNINIT) {
2214                 /*
2215                  * We didn't get to be the master so we definitely won't be
2216                  * configuring the chip.  It's a bug if someone else hasn't
2217                  * configured it already.
2218                  */
2219                 device_printf(sc->dev, "couldn't be master(%d), "
2220                     "device not already initialized either(%d).\n", rc, state);
2221                 return (EDOOFUS);
2222         }
2223
2224         /* This is the firmware whose headers the driver was compiled against */
2225         fw_info = find_fw_info(chip_id(sc));
2226         if (fw_info == NULL) {
2227                 device_printf(sc->dev,
2228                     "unable to look up firmware information for chip %d.\n",
2229                     chip_id(sc));
2230                 return (EINVAL);
2231         }
2232         drv_fw = &fw_info->fw_hdr;
2233
2234         /*
2235          * The firmware KLD contains many modules.  The KLD name is also the
2236          * name of the module that contains the default config file.
2237          */
2238         default_cfg = firmware_get(fw_info->kld_name);
2239
2240         /* Read the header of the firmware on the card */
2241         card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2242         rc = -t4_read_flash(sc, FLASH_FW_START,
2243             sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2244         if (rc == 0)
2245                 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2246         else {
2247                 device_printf(sc->dev,
2248                     "Unable to read card's firmware header: %d\n", rc);
2249                 card_fw_usable = 0;
2250         }
2251
2252         /* This is the firmware in the KLD */
2253         fw = firmware_get(fw_info->fw_mod_name);
2254         if (fw != NULL) {
2255                 kld_fw = (const void *)fw->data;
2256                 kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2257         } else {
2258                 kld_fw = NULL;
2259                 kld_fw_usable = 0;
2260         }
2261
2262         if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2263             (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2264                 /*
2265                  * Common case: the firmware on the card is an exact match and
2266                  * the KLD is an exact match too, or the KLD is
2267                  * absent/incompatible.  Note that t4_fw_install = 2 is ignored
2268                  * here -- use cxgbetool loadfw if you want to reinstall the
2269                  * same firmware as the one on the card.
2270                  */
2271         } else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2272             should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2273             be32toh(card_fw->fw_ver))) {
2274
2275                 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
2276                 if (rc != 0) {
2277                         device_printf(sc->dev,
2278                             "failed to install firmware: %d\n", rc);
2279                         goto done;
2280                 }
2281
2282                 /* Installed successfully, update the cached header too. */
2283                 memcpy(card_fw, kld_fw, sizeof(*card_fw));
2284                 card_fw_usable = 1;
2285                 need_fw_reset = 0;      /* already reset as part of load_fw */
2286         }
2287
2288         if (!card_fw_usable) {
2289                 uint32_t d, c, k;
2290
2291                 d = ntohl(drv_fw->fw_ver);
2292                 c = ntohl(card_fw->fw_ver);
2293                 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
2294
2295                 device_printf(sc->dev, "Cannot find a usable firmware: "
2296                     "fw_install %d, chip state %d, "
2297                     "driver compiled with %d.%d.%d.%d, "
2298                     "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
2299                     t4_fw_install, state,
2300                     G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
2301                     G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
2302                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2303                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
2304                     G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2305                     G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2306                 rc = EINVAL;
2307                 goto done;
2308         }
2309
2310         /* We're using whatever's on the card and it's known to be good. */
2311         sc->params.fw_vers = ntohl(card_fw->fw_ver);
2312         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
2313             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2314             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2315             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2316             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2317         t4_get_tp_version(sc, &sc->params.tp_vers);
2318
2319         /* Reset device */
2320         if (need_fw_reset &&
2321             (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
2322                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
2323                 if (rc != ETIMEDOUT && rc != EIO)
2324                         t4_fw_bye(sc, sc->mbox);
2325                 goto done;
2326         }
2327         sc->flags |= FW_OK;
2328
2329         rc = get_params__pre_init(sc);
2330         if (rc != 0)
2331                 goto done; /* error message displayed already */
2332
2333         /* Partition adapter resources as specified in the config file. */
2334         if (state == DEV_STATE_UNINIT) {
2335
2336                 KASSERT(sc->flags & MASTER_PF,
2337                     ("%s: trying to change chip settings when not master.",
2338                     __func__));
2339
2340                 rc = partition_resources(sc, default_cfg, fw_info->kld_name);
2341                 if (rc != 0)
2342                         goto done;      /* error message displayed already */
2343
2344                 t4_tweak_chip_settings(sc);
2345
2346                 /* get basic stuff going */
2347                 rc = -t4_fw_initialize(sc, sc->mbox);
2348                 if (rc != 0) {
2349                         device_printf(sc->dev, "fw init failed: %d.\n", rc);
2350                         goto done;
2351                 }
2352         } else {
2353                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
2354                 sc->cfcsum = 0;
2355         }
2356
2357 done:
2358         free(card_fw, M_CXGBE);
2359         if (fw != NULL)
2360                 firmware_put(fw, FIRMWARE_UNLOAD);
2361         if (default_cfg != NULL)
2362                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
2363
2364         return (rc);
2365 }
2366
2367 #define FW_PARAM_DEV(param) \
2368         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2369          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2370 #define FW_PARAM_PFVF(param) \
2371         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2372          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2373
2374 /*
2375  * Partition chip resources for use between various PFs, VFs, etc.
2376  */
2377 static int
2378 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
2379     const char *name_prefix)
2380 {
2381         const struct firmware *cfg = NULL;
2382         int rc = 0;
2383         struct fw_caps_config_cmd caps;
2384         uint32_t mtype, moff, finicsum, cfcsum;
2385
2386         /*
2387          * Figure out what configuration file to use.  Pick the default config
2388          * file for the card if the user hasn't specified one explicitly.
2389          */
2390         snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
2391         if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
2392                 /* Card specific overrides go here. */
2393                 if (pci_get_device(sc->dev) == 0x440a)
2394                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
2395                 if (is_fpga(sc))
2396                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
2397         }
2398
2399         /*
2400          * We need to load another module if the profile is anything except
2401          * "default" or "flash".
2402          */
2403         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
2404             strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2405                 char s[32];
2406
2407                 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
2408                 cfg = firmware_get(s);
2409                 if (cfg == NULL) {
2410                         if (default_cfg != NULL) {
2411                                 device_printf(sc->dev,
2412                                     "unable to load module \"%s\" for "
2413                                     "configuration profile \"%s\", will use "
2414                                     "the default config file instead.\n",
2415                                     s, sc->cfg_file);
2416                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2417                                     "%s", DEFAULT_CF);
2418                         } else {
2419                                 device_printf(sc->dev,
2420                                     "unable to load module \"%s\" for "
2421                                     "configuration profile \"%s\", will use "
2422                                     "the config file on the card's flash "
2423                                     "instead.\n", s, sc->cfg_file);
2424                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2425                                     "%s", FLASH_CF);
2426                         }
2427                 }
2428         }
2429
2430         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
2431             default_cfg == NULL) {
2432                 device_printf(sc->dev,
2433                     "default config file not available, will use the config "
2434                     "file on the card's flash instead.\n");
2435                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
2436         }
2437
2438         if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2439                 u_int cflen, i, n;
2440                 const uint32_t *cfdata;
2441                 uint32_t param, val, addr, off, mw_base, mw_aperture;
2442
2443                 KASSERT(cfg != NULL || default_cfg != NULL,
2444                     ("%s: no config to upload", __func__));
2445
2446                 /*
2447                  * Ask the firmware where it wants us to upload the config file.
2448                  */
2449                 param = FW_PARAM_DEV(CF);
2450                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2451                 if (rc != 0) {
2452                         /* No support for config file?  Shouldn't happen. */
2453                         device_printf(sc->dev,
2454                             "failed to query config file location: %d.\n", rc);
2455                         goto done;
2456                 }
2457                 mtype = G_FW_PARAMS_PARAM_Y(val);
2458                 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
2459
2460                 /*
2461                  * XXX: sheer laziness.  We deliberately added 4 bytes of
2462                  * useless stuffing/comments at the end of the config file so
2463                  * it's ok to simply throw away the last remaining bytes when
2464                  * the config file is not an exact multiple of 4.  This also
2465                  * helps with the validate_mt_off_len check.
2466                  */
2467                 if (cfg != NULL) {
2468                         cflen = cfg->datasize & ~3;
2469                         cfdata = cfg->data;
2470                 } else {
2471                         cflen = default_cfg->datasize & ~3;
2472                         cfdata = default_cfg->data;
2473                 }
2474
2475                 if (cflen > FLASH_CFG_MAX_SIZE) {
2476                         device_printf(sc->dev,
2477                             "config file too long (%d, max allowed is %d).  "
2478                             "Will try to use the config on the card, if any.\n",
2479                             cflen, FLASH_CFG_MAX_SIZE);
2480                         goto use_config_on_flash;
2481                 }
2482
2483                 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
2484                 if (rc != 0) {
2485                         device_printf(sc->dev,
2486                             "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
2487                             "Will try to use the config on the card, if any.\n",
2488                             __func__, mtype, moff, cflen, rc);
2489                         goto use_config_on_flash;
2490                 }
2491
2492                 memwin_info(sc, 2, &mw_base, &mw_aperture);
2493                 while (cflen) {
2494                         off = position_memwin(sc, 2, addr);
2495                         n = min(cflen, mw_aperture - off);
2496                         for (i = 0; i < n; i += 4)
2497                                 t4_write_reg(sc, mw_base + off + i, *cfdata++);
2498                         cflen -= n;
2499                         addr += n;
2500                 }
2501         } else {
2502 use_config_on_flash:
2503                 mtype = FW_MEMTYPE_FLASH;
2504                 moff = t4_flash_cfg_addr(sc);
2505         }
2506
2507         bzero(&caps, sizeof(caps));
2508         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2509             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2510         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
2511             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
2512             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
2513         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2514         if (rc != 0) {
2515                 device_printf(sc->dev,
2516                     "failed to pre-process config file: %d "
2517                     "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
2518                 goto done;
2519         }
2520
2521         finicsum = be32toh(caps.finicsum);
2522         cfcsum = be32toh(caps.cfcsum);
2523         if (finicsum != cfcsum) {
2524                 device_printf(sc->dev,
2525                     "WARNING: config file checksum mismatch: %08x %08x\n",
2526                     finicsum, cfcsum);
2527         }
2528         sc->cfcsum = cfcsum;
2529
2530 #define LIMIT_CAPS(x) do { \
2531         caps.x &= htobe16(t4_##x##_allowed); \
2532 } while (0)
2533
2534         /*
2535          * Let the firmware know what features will (not) be used so it can tune
2536          * things accordingly.
2537          */
2538         LIMIT_CAPS(linkcaps);
2539         LIMIT_CAPS(niccaps);
2540         LIMIT_CAPS(toecaps);
2541         LIMIT_CAPS(rdmacaps);
2542         LIMIT_CAPS(iscsicaps);
2543         LIMIT_CAPS(fcoecaps);
2544 #undef LIMIT_CAPS
2545
2546         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2547             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
2548         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2549         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
2550         if (rc != 0) {
2551                 device_printf(sc->dev,
2552                     "failed to process config file: %d.\n", rc);
2553         }
2554 done:
2555         if (cfg != NULL)
2556                 firmware_put(cfg, FIRMWARE_UNLOAD);
2557         return (rc);
2558 }
2559
2560 /*
2561  * Retrieve parameters that are needed (or nice to have) very early.
2562  */
2563 static int
2564 get_params__pre_init(struct adapter *sc)
2565 {
2566         int rc;
2567         uint32_t param[2], val[2];
2568         struct fw_devlog_cmd cmd;
2569         struct devlog_params *dlog = &sc->params.devlog;
2570
2571         param[0] = FW_PARAM_DEV(PORTVEC);
2572         param[1] = FW_PARAM_DEV(CCLK);
2573         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2574         if (rc != 0) {
2575                 device_printf(sc->dev,
2576                     "failed to query parameters (pre_init): %d.\n", rc);
2577                 return (rc);
2578         }
2579
2580         sc->params.portvec = val[0];
2581         sc->params.nports = bitcount32(val[0]);
2582         sc->params.vpd.cclk = val[1];
2583
2584         /* Read device log parameters. */
2585         bzero(&cmd, sizeof(cmd));
2586         cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
2587             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2588         cmd.retval_len16 = htobe32(FW_LEN16(cmd));
2589         rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
2590         if (rc != 0) {
2591                 device_printf(sc->dev,
2592                     "failed to get devlog parameters: %d.\n", rc);
2593                 bzero(dlog, sizeof (*dlog));
2594                 rc = 0; /* devlog isn't critical for device operation */
2595         } else {
2596                 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
2597                 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
2598                 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
2599                 dlog->size = be32toh(cmd.memsize_devlog);
2600         }
2601
2602         return (rc);
2603 }
2604
2605 /*
2606  * Retrieve various parameters that are of interest to the driver.  The device
2607  * has been initialized by the firmware at this point.
2608  */
2609 static int
2610 get_params__post_init(struct adapter *sc)
2611 {
2612         int rc;
2613         uint32_t param[7], val[7];
2614         struct fw_caps_config_cmd caps;
2615
2616         param[0] = FW_PARAM_PFVF(IQFLINT_START);
2617         param[1] = FW_PARAM_PFVF(EQ_START);
2618         param[2] = FW_PARAM_PFVF(FILTER_START);
2619         param[3] = FW_PARAM_PFVF(FILTER_END);
2620         param[4] = FW_PARAM_PFVF(L2T_START);
2621         param[5] = FW_PARAM_PFVF(L2T_END);
2622         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2623         if (rc != 0) {
2624                 device_printf(sc->dev,
2625                     "failed to query parameters (post_init): %d.\n", rc);
2626                 return (rc);
2627         }
2628
2629         sc->sge.iq_start = val[0];
2630         sc->sge.eq_start = val[1];
2631         sc->tids.ftid_base = val[2];
2632         sc->tids.nftids = val[3] - val[2] + 1;
2633         sc->params.ftid_min = val[2];
2634         sc->params.ftid_max = val[3];
2635         sc->vres.l2t.start = val[4];
2636         sc->vres.l2t.size = val[5] - val[4] + 1;
2637         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
2638             ("%s: L2 table size (%u) larger than expected (%u)",
2639             __func__, sc->vres.l2t.size, L2T_SIZE));
2640
2641         /* get capabilites */
2642         bzero(&caps, sizeof(caps));
2643         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2644             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2645         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2646         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2647         if (rc != 0) {
2648                 device_printf(sc->dev,
2649                     "failed to get card capabilities: %d.\n", rc);
2650                 return (rc);
2651         }
2652
2653 #define READ_CAPS(x) do { \
2654         sc->x = htobe16(caps.x); \
2655 } while (0)
2656         READ_CAPS(linkcaps);
2657         READ_CAPS(niccaps);
2658         READ_CAPS(toecaps);
2659         READ_CAPS(rdmacaps);
2660         READ_CAPS(iscsicaps);
2661         READ_CAPS(fcoecaps);
2662
2663         if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
2664                 param[0] = FW_PARAM_PFVF(ETHOFLD_START);
2665                 param[1] = FW_PARAM_PFVF(ETHOFLD_END);
2666                 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2667                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
2668                 if (rc != 0) {
2669                         device_printf(sc->dev,
2670                             "failed to query NIC parameters: %d.\n", rc);
2671                         return (rc);
2672                 }
2673                 sc->tids.etid_base = val[0];
2674                 sc->params.etid_min = val[0];
2675                 sc->tids.netids = val[1] - val[0] + 1;
2676                 sc->params.netids = sc->tids.netids;
2677                 sc->params.eo_wr_cred = val[2];
2678                 sc->params.ethoffload = 1;
2679         }
2680
2681         if (sc->toecaps) {
2682                 /* query offload-related parameters */
2683                 param[0] = FW_PARAM_DEV(NTID);
2684                 param[1] = FW_PARAM_PFVF(SERVER_START);
2685                 param[2] = FW_PARAM_PFVF(SERVER_END);
2686                 param[3] = FW_PARAM_PFVF(TDDP_START);
2687                 param[4] = FW_PARAM_PFVF(TDDP_END);
2688                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2689                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2690                 if (rc != 0) {
2691                         device_printf(sc->dev,
2692                             "failed to query TOE parameters: %d.\n", rc);
2693                         return (rc);
2694                 }
2695                 sc->tids.ntids = val[0];
2696                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
2697                 sc->tids.stid_base = val[1];
2698                 sc->tids.nstids = val[2] - val[1] + 1;
2699                 sc->vres.ddp.start = val[3];
2700                 sc->vres.ddp.size = val[4] - val[3] + 1;
2701                 sc->params.ofldq_wr_cred = val[5];
2702                 sc->params.offload = 1;
2703         }
2704         if (sc->rdmacaps) {
2705                 param[0] = FW_PARAM_PFVF(STAG_START);
2706                 param[1] = FW_PARAM_PFVF(STAG_END);
2707                 param[2] = FW_PARAM_PFVF(RQ_START);
2708                 param[3] = FW_PARAM_PFVF(RQ_END);
2709                 param[4] = FW_PARAM_PFVF(PBL_START);
2710                 param[5] = FW_PARAM_PFVF(PBL_END);
2711                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2712                 if (rc != 0) {
2713                         device_printf(sc->dev,
2714                             "failed to query RDMA parameters(1): %d.\n", rc);
2715                         return (rc);
2716                 }
2717                 sc->vres.stag.start = val[0];
2718                 sc->vres.stag.size = val[1] - val[0] + 1;
2719                 sc->vres.rq.start = val[2];
2720                 sc->vres.rq.size = val[3] - val[2] + 1;
2721                 sc->vres.pbl.start = val[4];
2722                 sc->vres.pbl.size = val[5] - val[4] + 1;
2723
2724                 param[0] = FW_PARAM_PFVF(SQRQ_START);
2725                 param[1] = FW_PARAM_PFVF(SQRQ_END);
2726                 param[2] = FW_PARAM_PFVF(CQ_START);
2727                 param[3] = FW_PARAM_PFVF(CQ_END);
2728                 param[4] = FW_PARAM_PFVF(OCQ_START);
2729                 param[5] = FW_PARAM_PFVF(OCQ_END);
2730                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2731                 if (rc != 0) {
2732                         device_printf(sc->dev,
2733                             "failed to query RDMA parameters(2): %d.\n", rc);
2734                         return (rc);
2735                 }
2736                 sc->vres.qp.start = val[0];
2737                 sc->vres.qp.size = val[1] - val[0] + 1;
2738                 sc->vres.cq.start = val[2];
2739                 sc->vres.cq.size = val[3] - val[2] + 1;
2740                 sc->vres.ocq.start = val[4];
2741                 sc->vres.ocq.size = val[5] - val[4] + 1;
2742         }
2743         if (sc->iscsicaps) {
2744                 param[0] = FW_PARAM_PFVF(ISCSI_START);
2745                 param[1] = FW_PARAM_PFVF(ISCSI_END);
2746                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2747                 if (rc != 0) {
2748                         device_printf(sc->dev,
2749                             "failed to query iSCSI parameters: %d.\n", rc);
2750                         return (rc);
2751                 }
2752                 sc->vres.iscsi.start = val[0];
2753                 sc->vres.iscsi.size = val[1] - val[0] + 1;
2754         }
2755
2756         /*
2757          * We've got the params we wanted to query via the firmware.  Now grab
2758          * some others directly from the chip.
2759          */
2760         rc = t4_read_chip_settings(sc);
2761
2762         return (rc);
2763 }
2764
2765 static int
2766 set_params__post_init(struct adapter *sc)
2767 {
2768         uint32_t param, val;
2769
2770         /* ask for encapsulated CPLs */
2771         param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
2772         val = 1;
2773         (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2774
2775         return (0);
2776 }
2777
2778 #undef FW_PARAM_PFVF
2779 #undef FW_PARAM_DEV
2780
2781 static void
2782 t4_set_desc(struct adapter *sc)
2783 {
2784         char buf[128];
2785         struct adapter_params *p = &sc->params;
2786
2787         snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
2788             "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
2789             chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
2790
2791         device_set_desc_copy(sc->dev, buf);
2792 }
2793
2794 static void
2795 build_medialist(struct port_info *pi, struct ifmedia *media)
2796 {
2797         int data, m;
2798
2799         PORT_LOCK(pi);
2800
2801         ifmedia_removeall(media);
2802
2803         m = IFM_ETHER | IFM_FDX;
2804         data = (pi->port_type << 8) | pi->mod_type;
2805
2806         switch(pi->port_type) {
2807         case FW_PORT_TYPE_BT_XFI:
2808                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2809                 break;
2810
2811         case FW_PORT_TYPE_BT_XAUI:
2812                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2813                 /* fall through */
2814
2815         case FW_PORT_TYPE_BT_SGMII:
2816                 ifmedia_add(media, m | IFM_1000_T, data, NULL);
2817                 ifmedia_add(media, m | IFM_100_TX, data, NULL);
2818                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL);
2819                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2820                 break;
2821
2822         case FW_PORT_TYPE_CX4:
2823                 ifmedia_add(media, m | IFM_10G_CX4, data, NULL);
2824                 ifmedia_set(media, m | IFM_10G_CX4);
2825                 break;
2826
2827         case FW_PORT_TYPE_QSFP_10G:
2828         case FW_PORT_TYPE_SFP:
2829         case FW_PORT_TYPE_FIBER_XFI:
2830         case FW_PORT_TYPE_FIBER_XAUI:
2831                 switch (pi->mod_type) {
2832
2833                 case FW_PORT_MOD_TYPE_LR:
2834                         ifmedia_add(media, m | IFM_10G_LR, data, NULL);
2835                         ifmedia_set(media, m | IFM_10G_LR);
2836                         break;
2837
2838                 case FW_PORT_MOD_TYPE_SR:
2839                         ifmedia_add(media, m | IFM_10G_SR, data, NULL);
2840                         ifmedia_set(media, m | IFM_10G_SR);
2841                         break;
2842
2843                 case FW_PORT_MOD_TYPE_LRM:
2844                         ifmedia_add(media, m | IFM_10G_LRM, data, NULL);
2845                         ifmedia_set(media, m | IFM_10G_LRM);
2846                         break;
2847
2848                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2849                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2850                         ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL);
2851                         ifmedia_set(media, m | IFM_10G_TWINAX);
2852                         break;
2853
2854                 case FW_PORT_MOD_TYPE_NONE:
2855                         m &= ~IFM_FDX;
2856                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2857                         ifmedia_set(media, m | IFM_NONE);
2858                         break;
2859
2860                 case FW_PORT_MOD_TYPE_NA:
2861                 case FW_PORT_MOD_TYPE_ER:
2862                 default:
2863                         device_printf(pi->dev,
2864                             "unknown port_type (%d), mod_type (%d)\n",
2865                             pi->port_type, pi->mod_type);
2866                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2867                         ifmedia_set(media, m | IFM_UNKNOWN);
2868                         break;
2869                 }
2870                 break;
2871
2872         case FW_PORT_TYPE_QSFP:
2873                 switch (pi->mod_type) {
2874
2875                 case FW_PORT_MOD_TYPE_LR:
2876                         ifmedia_add(media, m | IFM_40G_LR4, data, NULL);
2877                         ifmedia_set(media, m | IFM_40G_LR4);
2878                         break;
2879
2880                 case FW_PORT_MOD_TYPE_SR:
2881                         ifmedia_add(media, m | IFM_40G_SR4, data, NULL);
2882                         ifmedia_set(media, m | IFM_40G_SR4);
2883                         break;
2884
2885                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2886                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2887                         ifmedia_add(media, m | IFM_40G_CR4, data, NULL);
2888                         ifmedia_set(media, m | IFM_40G_CR4);
2889                         break;
2890
2891                 case FW_PORT_MOD_TYPE_NONE:
2892                         m &= ~IFM_FDX;
2893                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2894                         ifmedia_set(media, m | IFM_NONE);
2895                         break;
2896
2897                 default:
2898                         device_printf(pi->dev,
2899                             "unknown port_type (%d), mod_type (%d)\n",
2900                             pi->port_type, pi->mod_type);
2901                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2902                         ifmedia_set(media, m | IFM_UNKNOWN);
2903                         break;
2904                 }
2905                 break;
2906
2907         default:
2908                 device_printf(pi->dev,
2909                     "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
2910                     pi->mod_type);
2911                 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2912                 ifmedia_set(media, m | IFM_UNKNOWN);
2913                 break;
2914         }
2915
2916         PORT_UNLOCK(pi);
2917 }
2918
2919 #define FW_MAC_EXACT_CHUNK      7
2920
2921 /*
2922  * Program the port's XGMAC based on parameters in ifnet.  The caller also
2923  * indicates which parameters should be programmed (the rest are left alone).
2924  */
2925 int
2926 update_mac_settings(struct ifnet *ifp, int flags)
2927 {
2928         int rc = 0;
2929         struct port_info *pi = ifp->if_softc;
2930         struct adapter *sc = pi->adapter;
2931         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
2932         uint16_t viid = 0xffff;
2933         int16_t *xact_addr_filt = NULL;
2934
2935         ASSERT_SYNCHRONIZED_OP(sc);
2936         KASSERT(flags, ("%s: not told what to update.", __func__));
2937
2938         if (ifp == pi->ifp) {
2939                 viid = pi->viid;
2940                 xact_addr_filt = &pi->xact_addr_filt;
2941         }
2942 #ifdef DEV_NETMAP
2943         else if (ifp == pi->nm_ifp) {
2944                 viid = pi->nm_viid;
2945                 xact_addr_filt = &pi->nm_xact_addr_filt;
2946         }
2947 #endif
2948         if (flags & XGMAC_MTU)
2949                 mtu = ifp->if_mtu;
2950
2951         if (flags & XGMAC_PROMISC)
2952                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
2953
2954         if (flags & XGMAC_ALLMULTI)
2955                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
2956
2957         if (flags & XGMAC_VLANEX)
2958                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
2959
2960         if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
2961                 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti,
2962                     1, vlanex, false);
2963                 if (rc) {
2964                         if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
2965                             rc);
2966                         return (rc);
2967                 }
2968         }
2969
2970         if (flags & XGMAC_UCADDR) {
2971                 uint8_t ucaddr[ETHER_ADDR_LEN];
2972
2973                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
2974                 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr,
2975                     true, true);
2976                 if (rc < 0) {
2977                         rc = -rc;
2978                         if_printf(ifp, "change_mac failed: %d\n", rc);
2979                         return (rc);
2980                 } else {
2981                         *xact_addr_filt = rc;
2982                         rc = 0;
2983                 }
2984         }
2985
2986         if (flags & XGMAC_MCADDRS) {
2987                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
2988                 int del = 1;
2989                 uint64_t hash = 0;
2990                 struct ifmultiaddr *ifma;
2991                 int i = 0, j;
2992
2993                 if_maddr_rlock(ifp);
2994                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2995                         if (ifma->ifma_addr->sa_family != AF_LINK)
2996                                 continue;
2997                         mcaddr[i++] =
2998                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
2999
3000                         if (i == FW_MAC_EXACT_CHUNK) {
3001                                 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del,
3002                                     i, mcaddr, NULL, &hash, 0);
3003                                 if (rc < 0) {
3004                                         rc = -rc;
3005                                         for (j = 0; j < i; j++) {
3006                                                 if_printf(ifp,
3007                                                     "failed to add mc address"
3008                                                     " %02x:%02x:%02x:"
3009                                                     "%02x:%02x:%02x rc=%d\n",
3010                                                     mcaddr[j][0], mcaddr[j][1],
3011                                                     mcaddr[j][2], mcaddr[j][3],
3012                                                     mcaddr[j][4], mcaddr[j][5],
3013                                                     rc);
3014                                         }
3015                                         goto mcfail;
3016                                 }
3017                                 del = 0;
3018                                 i = 0;
3019                         }
3020                 }
3021                 if (i > 0) {
3022                         rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i,
3023                             mcaddr, NULL, &hash, 0);
3024                         if (rc < 0) {
3025                                 rc = -rc;
3026                                 for (j = 0; j < i; j++) {
3027                                         if_printf(ifp,
3028                                             "failed to add mc address"
3029                                             " %02x:%02x:%02x:"
3030                                             "%02x:%02x:%02x rc=%d\n",
3031                                             mcaddr[j][0], mcaddr[j][1],
3032                                             mcaddr[j][2], mcaddr[j][3],
3033                                             mcaddr[j][4], mcaddr[j][5],
3034                                             rc);
3035                                 }
3036                                 goto mcfail;
3037                         }
3038                 }
3039
3040                 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0);
3041                 if (rc != 0)
3042                         if_printf(ifp, "failed to set mc address hash: %d", rc);
3043 mcfail:
3044                 if_maddr_runlock(ifp);
3045         }
3046
3047         return (rc);
3048 }
3049
3050 int
3051 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
3052     char *wmesg)
3053 {
3054         int rc, pri;
3055
3056 #ifdef WITNESS
3057         /* the caller thinks it's ok to sleep, but is it really? */
3058         if (flags & SLEEP_OK)
3059                 pause("t4slptst", 1);
3060 #endif
3061
3062         if (INTR_OK)
3063                 pri = PCATCH;
3064         else
3065                 pri = 0;
3066
3067         ADAPTER_LOCK(sc);
3068         for (;;) {
3069
3070                 if (pi && IS_DOOMED(pi)) {
3071                         rc = ENXIO;
3072                         goto done;
3073                 }
3074
3075                 if (!IS_BUSY(sc)) {
3076                         rc = 0;
3077                         break;
3078                 }
3079
3080                 if (!(flags & SLEEP_OK)) {
3081                         rc = EBUSY;
3082                         goto done;
3083                 }
3084
3085                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
3086                         rc = EINTR;
3087                         goto done;
3088                 }
3089         }
3090
3091         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
3092         SET_BUSY(sc);
3093 #ifdef INVARIANTS
3094         sc->last_op = wmesg;
3095         sc->last_op_thr = curthread;
3096 #endif
3097
3098 done:
3099         if (!(flags & HOLD_LOCK) || rc)
3100                 ADAPTER_UNLOCK(sc);
3101
3102         return (rc);
3103 }
3104
3105 void
3106 end_synchronized_op(struct adapter *sc, int flags)
3107 {
3108
3109         if (flags & LOCK_HELD)
3110                 ADAPTER_LOCK_ASSERT_OWNED(sc);
3111         else
3112                 ADAPTER_LOCK(sc);
3113
3114         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
3115         CLR_BUSY(sc);
3116         wakeup(&sc->flags);
3117         ADAPTER_UNLOCK(sc);
3118 }
3119
3120 static int
3121 cxgbe_init_synchronized(struct port_info *pi)
3122 {
3123         struct adapter *sc = pi->adapter;
3124         struct ifnet *ifp = pi->ifp;
3125         int rc = 0;
3126
3127         ASSERT_SYNCHRONIZED_OP(sc);
3128
3129         if (isset(&sc->open_device_map, pi->port_id)) {
3130                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
3131                     ("mismatch between open_device_map and if_drv_flags"));
3132                 return (0);     /* already running */
3133         }
3134
3135         if (!(sc->flags & FULL_INIT_DONE) &&
3136             ((rc = adapter_full_init(sc)) != 0))
3137                 return (rc);    /* error message displayed already */
3138
3139         if (!(pi->flags & PORT_INIT_DONE) &&
3140             ((rc = port_full_init(pi)) != 0))
3141                 return (rc); /* error message displayed already */
3142
3143         rc = update_mac_settings(ifp, XGMAC_ALL);
3144         if (rc)
3145                 goto done;      /* error message displayed already */
3146
3147         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
3148         if (rc != 0) {
3149                 if_printf(ifp, "enable_vi failed: %d\n", rc);
3150                 goto done;
3151         }
3152
3153         /*
3154          * The first iq of the first port to come up is used for tracing.
3155          */
3156         if (sc->traceq < 0) {
3157                 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
3158                 t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
3159                     A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
3160                     V_QUEUENUMBER(sc->traceq));
3161                 pi->flags |= HAS_TRACEQ;
3162         }
3163
3164         /* all ok */
3165         setbit(&sc->open_device_map, pi->port_id);
3166         PORT_LOCK(pi);
3167         ifp->if_drv_flags |= IFF_DRV_RUNNING;
3168         PORT_UNLOCK(pi);
3169
3170         callout_reset(&pi->tick, hz, cxgbe_tick, pi);
3171 done:
3172         if (rc != 0)
3173                 cxgbe_uninit_synchronized(pi);
3174
3175         return (rc);
3176 }
3177
3178 /*
3179  * Idempotent.
3180  */
3181 static int
3182 cxgbe_uninit_synchronized(struct port_info *pi)
3183 {
3184         struct adapter *sc = pi->adapter;
3185         struct ifnet *ifp = pi->ifp;
3186         int rc;
3187
3188         ASSERT_SYNCHRONIZED_OP(sc);
3189
3190         /*
3191          * Disable the VI so that all its data in either direction is discarded
3192          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
3193          * tick) intact as the TP can deliver negative advice or data that it's
3194          * holding in its RAM (for an offloaded connection) even after the VI is
3195          * disabled.
3196          */
3197         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
3198         if (rc) {
3199                 if_printf(ifp, "disable_vi failed: %d\n", rc);
3200                 return (rc);
3201         }
3202
3203         clrbit(&sc->open_device_map, pi->port_id);
3204         PORT_LOCK(pi);
3205         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3206         PORT_UNLOCK(pi);
3207
3208         pi->link_cfg.link_ok = 0;
3209         pi->link_cfg.speed = 0;
3210         pi->linkdnrc = -1;
3211         t4_os_link_changed(sc, pi->port_id, 0, -1);
3212
3213         return (0);
3214 }
3215
3216 /*
3217  * It is ok for this function to fail midway and return right away.  t4_detach
3218  * will walk the entire sc->irq list and clean up whatever is valid.
3219  */
3220 static int
3221 setup_intr_handlers(struct adapter *sc)
3222 {
3223         int rc, rid, p, q;
3224         char s[8];
3225         struct irq *irq;
3226         struct port_info *pi;
3227         struct sge_rxq *rxq;
3228 #ifdef TCP_OFFLOAD
3229         struct sge_ofld_rxq *ofld_rxq;
3230 #endif
3231 #ifdef DEV_NETMAP
3232         struct sge_nm_rxq *nm_rxq;
3233 #endif
3234
3235         /*
3236          * Setup interrupts.
3237          */
3238         irq = &sc->irq[0];
3239         rid = sc->intr_type == INTR_INTX ? 0 : 1;
3240         if (sc->intr_count == 1)
3241                 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
3242
3243         /* Multiple interrupts. */
3244         KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
3245             ("%s: too few intr.", __func__));
3246
3247         /* The first one is always error intr */
3248         rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
3249         if (rc != 0)
3250                 return (rc);
3251         irq++;
3252         rid++;
3253
3254         /* The second one is always the firmware event queue */
3255         rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
3256         if (rc != 0)
3257                 return (rc);
3258         irq++;
3259         rid++;
3260
3261         for_each_port(sc, p) {
3262                 pi = sc->port[p];
3263
3264                 if (pi->flags & INTR_RXQ) {
3265                         for_each_rxq(pi, q, rxq) {
3266                                 snprintf(s, sizeof(s), "%d.%d", p, q);
3267                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
3268                                     s);
3269                                 if (rc != 0)
3270                                         return (rc);
3271                                 irq++;
3272                                 rid++;
3273                         }
3274                 }
3275 #ifdef TCP_OFFLOAD
3276                 if (pi->flags & INTR_OFLD_RXQ) {
3277                         for_each_ofld_rxq(pi, q, ofld_rxq) {
3278                                 snprintf(s, sizeof(s), "%d,%d", p, q);
3279                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr,
3280                                     ofld_rxq, s);
3281                                 if (rc != 0)
3282                                         return (rc);
3283                                 irq++;
3284                                 rid++;
3285                         }
3286                 }
3287 #endif
3288 #ifdef DEV_NETMAP
3289                 if (pi->flags & INTR_NM_RXQ) {
3290                         for_each_nm_rxq(pi, q, nm_rxq) {
3291                                 snprintf(s, sizeof(s), "%d-%d", p, q);
3292                                 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr,
3293                                     nm_rxq, s);
3294                                 if (rc != 0)
3295                                         return (rc);
3296                                 irq++;
3297                                 rid++;
3298                         }
3299                 }
3300 #endif
3301         }
3302         MPASS(irq == &sc->irq[sc->intr_count]);
3303
3304         return (0);
3305 }
3306
3307 int
3308 adapter_full_init(struct adapter *sc)
3309 {
3310         int rc, i;
3311
3312         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3313         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
3314             ("%s: FULL_INIT_DONE already", __func__));
3315
3316         /*
3317          * queues that belong to the adapter (not any particular port).
3318          */
3319         rc = t4_setup_adapter_queues(sc);
3320         if (rc != 0)
3321                 goto done;
3322
3323         for (i = 0; i < nitems(sc->tq); i++) {
3324                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
3325                     taskqueue_thread_enqueue, &sc->tq[i]);
3326                 if (sc->tq[i] == NULL) {
3327                         device_printf(sc->dev,
3328                             "failed to allocate task queue %d\n", i);
3329                         rc = ENOMEM;
3330                         goto done;
3331                 }
3332                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
3333                     device_get_nameunit(sc->dev), i);
3334         }
3335
3336         t4_intr_enable(sc);
3337         sc->flags |= FULL_INIT_DONE;
3338 done:
3339         if (rc != 0)
3340                 adapter_full_uninit(sc);
3341
3342         return (rc);
3343 }
3344
3345 int
3346 adapter_full_uninit(struct adapter *sc)
3347 {
3348         int i;
3349
3350         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3351
3352         t4_teardown_adapter_queues(sc);
3353
3354         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
3355                 taskqueue_free(sc->tq[i]);
3356                 sc->tq[i] = NULL;
3357         }
3358
3359         sc->flags &= ~FULL_INIT_DONE;
3360
3361         return (0);
3362 }
3363
3364 int
3365 port_full_init(struct port_info *pi)
3366 {
3367         struct adapter *sc = pi->adapter;
3368         struct ifnet *ifp = pi->ifp;
3369         uint16_t *rss;
3370         struct sge_rxq *rxq;
3371         int rc, i, j;
3372
3373         ASSERT_SYNCHRONIZED_OP(sc);
3374         KASSERT((pi->flags & PORT_INIT_DONE) == 0,
3375             ("%s: PORT_INIT_DONE already", __func__));
3376
3377         sysctl_ctx_init(&pi->ctx);
3378         pi->flags |= PORT_SYSCTL_CTX;
3379
3380         /*
3381          * Allocate tx/rx/fl queues for this port.
3382          */
3383         rc = t4_setup_port_queues(pi);
3384         if (rc != 0)
3385                 goto done;      /* error message displayed already */
3386
3387         /*
3388          * Setup RSS for this port.  Save a copy of the RSS table for later use.
3389          */
3390         rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
3391         for (i = 0; i < pi->rss_size;) {
3392                 for_each_rxq(pi, j, rxq) {
3393                         rss[i++] = rxq->iq.abs_id;
3394                         if (i == pi->rss_size)
3395                                 break;
3396                 }
3397         }
3398
3399         rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
3400             pi->rss_size);
3401         if (rc != 0) {
3402                 if_printf(ifp, "rss_config failed: %d\n", rc);
3403                 goto done;
3404         }
3405
3406         pi->rss = rss;
3407         pi->flags |= PORT_INIT_DONE;
3408 done:
3409         if (rc != 0)
3410                 port_full_uninit(pi);
3411
3412         return (rc);
3413 }
3414
3415 /*
3416  * Idempotent.
3417  */
3418 int
3419 port_full_uninit(struct port_info *pi)
3420 {
3421         struct adapter *sc = pi->adapter;
3422         int i;
3423         struct sge_rxq *rxq;
3424         struct sge_txq *txq;
3425 #ifdef TCP_OFFLOAD
3426         struct sge_ofld_rxq *ofld_rxq;
3427         struct sge_wrq *ofld_txq;
3428 #endif
3429
3430         if (pi->flags & PORT_INIT_DONE) {
3431
3432                 /* Need to quiesce queues.  XXX: ctrl queues? */
3433
3434                 for_each_txq(pi, i, txq) {
3435                         quiesce_eq(sc, &txq->eq);
3436                 }
3437
3438 #ifdef TCP_OFFLOAD
3439                 for_each_ofld_txq(pi, i, ofld_txq) {
3440                         quiesce_eq(sc, &ofld_txq->eq);
3441                 }
3442 #endif
3443
3444                 for_each_rxq(pi, i, rxq) {
3445                         quiesce_iq(sc, &rxq->iq);
3446                         quiesce_fl(sc, &rxq->fl);
3447                 }
3448
3449 #ifdef TCP_OFFLOAD
3450                 for_each_ofld_rxq(pi, i, ofld_rxq) {
3451                         quiesce_iq(sc, &ofld_rxq->iq);
3452                         quiesce_fl(sc, &ofld_rxq->fl);
3453                 }
3454 #endif
3455                 free(pi->rss, M_CXGBE);
3456         }
3457
3458         t4_teardown_port_queues(pi);
3459         pi->flags &= ~PORT_INIT_DONE;
3460
3461         return (0);
3462 }
3463
3464 static void
3465 quiesce_eq(struct adapter *sc, struct sge_eq *eq)
3466 {
3467         EQ_LOCK(eq);
3468         eq->flags |= EQ_DOOMED;
3469
3470         /*
3471          * Wait for the response to a credit flush if one's
3472          * pending.
3473          */
3474         while (eq->flags & EQ_CRFLUSHED)
3475                 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0);
3476         EQ_UNLOCK(eq);
3477
3478         callout_drain(&eq->tx_callout); /* XXX: iffy */
3479         pause("callout", 10);           /* Still iffy */
3480
3481         taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task);
3482 }
3483
3484 static void
3485 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
3486 {
3487         (void) sc;      /* unused */
3488
3489         /* Synchronize with the interrupt handler */
3490         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
3491                 pause("iqfree", 1);
3492 }
3493
3494 static void
3495 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
3496 {
3497         mtx_lock(&sc->sfl_lock);
3498         FL_LOCK(fl);
3499         fl->flags |= FL_DOOMED;
3500         FL_UNLOCK(fl);
3501         mtx_unlock(&sc->sfl_lock);
3502
3503         callout_drain(&sc->sfl_callout);
3504         KASSERT((fl->flags & FL_STARVING) == 0,
3505             ("%s: still starving", __func__));
3506 }
3507
3508 static int
3509 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
3510     driver_intr_t *handler, void *arg, char *name)
3511 {
3512         int rc;
3513
3514         irq->rid = rid;
3515         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
3516             RF_SHAREABLE | RF_ACTIVE);
3517         if (irq->res == NULL) {
3518                 device_printf(sc->dev,
3519                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
3520                 return (ENOMEM);
3521         }
3522
3523         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
3524             NULL, handler, arg, &irq->tag);
3525         if (rc != 0) {
3526                 device_printf(sc->dev,
3527                     "failed to setup interrupt for rid %d, name %s: %d\n",
3528                     rid, name, rc);
3529         } else if (name)
3530                 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
3531
3532         return (rc);
3533 }
3534
3535 static int
3536 t4_free_irq(struct adapter *sc, struct irq *irq)
3537 {
3538         if (irq->tag)
3539                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
3540         if (irq->res)
3541                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
3542
3543         bzero(irq, sizeof(*irq));
3544
3545         return (0);
3546 }
3547
3548 static void
3549 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
3550     unsigned int end)
3551 {
3552         uint32_t *p = (uint32_t *)(buf + start);
3553
3554         for ( ; start <= end; start += sizeof(uint32_t))
3555                 *p++ = t4_read_reg(sc, start);
3556 }
3557
3558 static void
3559 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
3560 {
3561         int i, n;
3562         const unsigned int *reg_ranges;
3563         static const unsigned int t4_reg_ranges[] = {
3564                 0x1008, 0x1108,
3565                 0x1180, 0x11b4,
3566                 0x11fc, 0x123c,
3567                 0x1300, 0x173c,
3568                 0x1800, 0x18fc,
3569                 0x3000, 0x30d8,
3570                 0x30e0, 0x5924,
3571                 0x5960, 0x59d4,
3572                 0x5a00, 0x5af8,
3573                 0x6000, 0x6098,
3574                 0x6100, 0x6150,
3575                 0x6200, 0x6208,
3576                 0x6240, 0x6248,
3577                 0x6280, 0x6338,
3578                 0x6370, 0x638c,
3579                 0x6400, 0x643c,
3580                 0x6500, 0x6524,
3581                 0x6a00, 0x6a38,
3582                 0x6a60, 0x6a78,
3583                 0x6b00, 0x6b84,
3584                 0x6bf0, 0x6c84,
3585                 0x6cf0, 0x6d84,
3586                 0x6df0, 0x6e84,
3587                 0x6ef0, 0x6f84,
3588                 0x6ff0, 0x7084,
3589                 0x70f0, 0x7184,
3590                 0x71f0, 0x7284,
3591                 0x72f0, 0x7384,
3592                 0x73f0, 0x7450,
3593                 0x7500, 0x7530,
3594                 0x7600, 0x761c,
3595                 0x7680, 0x76cc,
3596                 0x7700, 0x7798,
3597                 0x77c0, 0x77fc,
3598                 0x7900, 0x79fc,
3599                 0x7b00, 0x7c38,
3600                 0x7d00, 0x7efc,
3601                 0x8dc0, 0x8e1c,
3602                 0x8e30, 0x8e78,
3603                 0x8ea0, 0x8f6c,
3604                 0x8fc0, 0x9074,
3605                 0x90fc, 0x90fc,
3606                 0x9400, 0x9458,
3607                 0x9600, 0x96bc,
3608                 0x9800, 0x9808,
3609                 0x9820, 0x983c,
3610                 0x9850, 0x9864,
3611                 0x9c00, 0x9c6c,
3612                 0x9c80, 0x9cec,
3613                 0x9d00, 0x9d6c,
3614                 0x9d80, 0x9dec,
3615                 0x9e00, 0x9e6c,
3616                 0x9e80, 0x9eec,
3617                 0x9f00, 0x9f6c,
3618                 0x9f80, 0x9fec,
3619                 0xd004, 0xd03c,
3620                 0xdfc0, 0xdfe0,
3621                 0xe000, 0xea7c,
3622                 0xf000, 0x11110,
3623                 0x11118, 0x11190,
3624                 0x19040, 0x1906c,
3625                 0x19078, 0x19080,
3626                 0x1908c, 0x19124,
3627                 0x19150, 0x191b0,
3628                 0x191d0, 0x191e8,
3629                 0x19238, 0x1924c,
3630                 0x193f8, 0x19474,
3631                 0x19490, 0x194f8,
3632                 0x19800, 0x19f30,
3633                 0x1a000, 0x1a06c,
3634                 0x1a0b0, 0x1a120,
3635                 0x1a128, 0x1a138,
3636                 0x1a190, 0x1a1c4,
3637                 0x1a1fc, 0x1a1fc,
3638                 0x1e040, 0x1e04c,
3639                 0x1e284, 0x1e28c,
3640                 0x1e2c0, 0x1e2c0,
3641                 0x1e2e0, 0x1e2e0,
3642                 0x1e300, 0x1e384,
3643                 0x1e3c0, 0x1e3c8,
3644                 0x1e440, 0x1e44c,
3645                 0x1e684, 0x1e68c,
3646                 0x1e6c0, 0x1e6c0,
3647                 0x1e6e0, 0x1e6e0,
3648                 0x1e700, 0x1e784,
3649                 0x1e7c0, 0x1e7c8,
3650                 0x1e840, 0x1e84c,
3651                 0x1ea84, 0x1ea8c,
3652                 0x1eac0, 0x1eac0,
3653                 0x1eae0, 0x1eae0,
3654                 0x1eb00, 0x1eb84,
3655                 0x1ebc0, 0x1ebc8,
3656                 0x1ec40, 0x1ec4c,
3657                 0x1ee84, 0x1ee8c,
3658                 0x1eec0, 0x1eec0,
3659                 0x1eee0, 0x1eee0,
3660                 0x1ef00, 0x1ef84,
3661                 0x1efc0, 0x1efc8,
3662                 0x1f040, 0x1f04c,
3663                 0x1f284, 0x1f28c,
3664                 0x1f2c0, 0x1f2c0,
3665                 0x1f2e0, 0x1f2e0,
3666                 0x1f300, 0x1f384,
3667                 0x1f3c0, 0x1f3c8,
3668                 0x1f440, 0x1f44c,
3669                 0x1f684, 0x1f68c,
3670                 0x1f6c0, 0x1f6c0,
3671                 0x1f6e0, 0x1f6e0,
3672                 0x1f700, 0x1f784,
3673                 0x1f7c0, 0x1f7c8,
3674                 0x1f840, 0x1f84c,
3675                 0x1fa84, 0x1fa8c,
3676                 0x1fac0, 0x1fac0,
3677                 0x1fae0, 0x1fae0,
3678                 0x1fb00, 0x1fb84,
3679                 0x1fbc0, 0x1fbc8,
3680                 0x1fc40, 0x1fc4c,
3681                 0x1fe84, 0x1fe8c,
3682                 0x1fec0, 0x1fec0,
3683                 0x1fee0, 0x1fee0,
3684                 0x1ff00, 0x1ff84,
3685                 0x1ffc0, 0x1ffc8,
3686                 0x20000, 0x2002c,
3687                 0x20100, 0x2013c,
3688                 0x20190, 0x201c8,
3689                 0x20200, 0x20318,
3690                 0x20400, 0x20528,
3691                 0x20540, 0x20614,
3692                 0x21000, 0x21040,
3693                 0x2104c, 0x21060,
3694                 0x210c0, 0x210ec,
3695                 0x21200, 0x21268,
3696                 0x21270, 0x21284,
3697                 0x212fc, 0x21388,
3698                 0x21400, 0x21404,
3699                 0x21500, 0x21518,
3700                 0x2152c, 0x2153c,
3701                 0x21550, 0x21554,
3702                 0x21600, 0x21600,
3703                 0x21608, 0x21628,
3704                 0x21630, 0x2163c,
3705                 0x21700, 0x2171c,
3706                 0x21780, 0x2178c,
3707                 0x21800, 0x21c38,
3708                 0x21c80, 0x21d7c,
3709                 0x21e00, 0x21e04,
3710                 0x22000, 0x2202c,
3711                 0x22100, 0x2213c,
3712                 0x22190, 0x221c8,
3713                 0x22200, 0x22318,
3714                 0x22400, 0x22528,
3715                 0x22540, 0x22614,
3716                 0x23000, 0x23040,
3717                 0x2304c, 0x23060,
3718                 0x230c0, 0x230ec,
3719                 0x23200, 0x23268,
3720                 0x23270, 0x23284,
3721                 0x232fc, 0x23388,
3722                 0x23400, 0x23404,
3723                 0x23500, 0x23518,
3724                 0x2352c, 0x2353c,
3725                 0x23550, 0x23554,
3726                 0x23600, 0x23600,
3727                 0x23608, 0x23628,
3728                 0x23630, 0x2363c,
3729                 0x23700, 0x2371c,
3730                 0x23780, 0x2378c,
3731                 0x23800, 0x23c38,
3732                 0x23c80, 0x23d7c,
3733                 0x23e00, 0x23e04,
3734                 0x24000, 0x2402c,
3735                 0x24100, 0x2413c,
3736                 0x24190, 0x241c8,
3737                 0x24200, 0x24318,
3738                 0x24400, 0x24528,
3739                 0x24540, 0x24614,
3740                 0x25000, 0x25040,
3741                 0x2504c, 0x25060,
3742                 0x250c0, 0x250ec,
3743                 0x25200, 0x25268,
3744                 0x25270, 0x25284,
3745                 0x252fc, 0x25388,
3746                 0x25400, 0x25404,
3747                 0x25500, 0x25518,
3748                 0x2552c, 0x2553c,
3749                 0x25550, 0x25554,
3750                 0x25600, 0x25600,
3751                 0x25608, 0x25628,
3752                 0x25630, 0x2563c,
3753                 0x25700, 0x2571c,
3754                 0x25780, 0x2578c,
3755                 0x25800, 0x25c38,
3756                 0x25c80, 0x25d7c,
3757                 0x25e00, 0x25e04,
3758                 0x26000, 0x2602c,
3759                 0x26100, 0x2613c,
3760                 0x26190, 0x261c8,
3761                 0x26200, 0x26318,
3762                 0x26400, 0x26528,
3763                 0x26540, 0x26614,
3764                 0x27000, 0x27040,
3765                 0x2704c, 0x27060,
3766                 0x270c0, 0x270ec,
3767                 0x27200, 0x27268,
3768                 0x27270, 0x27284,
3769                 0x272fc, 0x27388,
3770                 0x27400, 0x27404,
3771                 0x27500, 0x27518,
3772                 0x2752c, 0x2753c,
3773                 0x27550, 0x27554,
3774                 0x27600, 0x27600,
3775                 0x27608, 0x27628,
3776                 0x27630, 0x2763c,
3777                 0x27700, 0x2771c,
3778                 0x27780, 0x2778c,
3779                 0x27800, 0x27c38,
3780                 0x27c80, 0x27d7c,
3781                 0x27e00, 0x27e04
3782         };
3783         static const unsigned int t5_reg_ranges[] = {
3784                 0x1008, 0x1148,
3785                 0x1180, 0x11b4,
3786                 0x11fc, 0x123c,
3787                 0x1280, 0x173c,
3788                 0x1800, 0x18fc,
3789                 0x3000, 0x3028,
3790                 0x3060, 0x30d8,
3791                 0x30e0, 0x30fc,
3792                 0x3140, 0x357c,
3793                 0x35a8, 0x35cc,
3794                 0x35ec, 0x35ec,
3795                 0x3600, 0x5624,
3796                 0x56cc, 0x575c,
3797                 0x580c, 0x5814,
3798                 0x5890, 0x58bc,
3799                 0x5940, 0x59dc,
3800                 0x59fc, 0x5a18,
3801                 0x5a60, 0x5a9c,
3802                 0x5b94, 0x5bfc,
3803                 0x6000, 0x6040,
3804                 0x6058, 0x614c,
3805                 0x7700, 0x7798,
3806                 0x77c0, 0x78fc,
3807                 0x7b00, 0x7c54,
3808                 0x7d00, 0x7efc,
3809                 0x8dc0, 0x8de0,
3810                 0x8df8, 0x8e84,
3811                 0x8ea0, 0x8f84,
3812                 0x8fc0, 0x90f8,
3813                 0x9400, 0x9470,
3814                 0x9600, 0x96f4,
3815                 0x9800, 0x9808,
3816                 0x9820, 0x983c,
3817                 0x9850, 0x9864,
3818                 0x9c00, 0x9c6c,
3819                 0x9c80, 0x9cec,
3820                 0x9d00, 0x9d6c,
3821                 0x9d80, 0x9dec,
3822                 0x9e00, 0x9e6c,
3823                 0x9e80, 0x9eec,
3824                 0x9f00, 0x9f6c,
3825                 0x9f80, 0xa020,
3826                 0xd004, 0xd03c,
3827                 0xdfc0, 0xdfe0,
3828                 0xe000, 0x11088,
3829                 0x1109c, 0x11110,
3830                 0x11118, 0x1117c,
3831                 0x11190, 0x11204,
3832                 0x19040, 0x1906c,
3833                 0x19078, 0x19080,
3834                 0x1908c, 0x19124,
3835                 0x19150, 0x191b0,
3836                 0x191d0, 0x191e8,
3837                 0x19238, 0x19290,
3838                 0x193f8, 0x19474,
3839                 0x19490, 0x194cc,
3840                 0x194f0, 0x194f8,
3841                 0x19c00, 0x19c60,
3842                 0x19c94, 0x19e10,
3843                 0x19e50, 0x19f34,
3844                 0x19f40, 0x19f50,
3845                 0x19f90, 0x19fe4,
3846                 0x1a000, 0x1a06c,
3847                 0x1a0b0, 0x1a120,
3848                 0x1a128, 0x1a138,
3849                 0x1a190, 0x1a1c4,
3850                 0x1a1fc, 0x1a1fc,
3851                 0x1e008, 0x1e00c,
3852                 0x1e040, 0x1e04c,
3853                 0x1e284, 0x1e290,
3854                 0x1e2c0, 0x1e2c0,
3855                 0x1e2e0, 0x1e2e0,
3856                 0x1e300, 0x1e384,
3857                 0x1e3c0, 0x1e3c8,
3858                 0x1e408, 0x1e40c,
3859                 0x1e440, 0x1e44c,
3860                 0x1e684, 0x1e690,
3861                 0x1e6c0, 0x1e6c0,
3862                 0x1e6e0, 0x1e6e0,
3863                 0x1e700, 0x1e784,
3864                 0x1e7c0, 0x1e7c8,
3865                 0x1e808, 0x1e80c,
3866                 0x1e840, 0x1e84c,
3867                 0x1ea84, 0x1ea90,
3868                 0x1eac0, 0x1eac0,
3869                 0x1eae0, 0x1eae0,
3870                 0x1eb00, 0x1eb84,
3871                 0x1ebc0, 0x1ebc8,
3872                 0x1ec08, 0x1ec0c,
3873                 0x1ec40, 0x1ec4c,
3874                 0x1ee84, 0x1ee90,
3875                 0x1eec0, 0x1eec0,
3876                 0x1eee0, 0x1eee0,
3877                 0x1ef00, 0x1ef84,
3878                 0x1efc0, 0x1efc8,
3879                 0x1f008, 0x1f00c,
3880                 0x1f040, 0x1f04c,
3881                 0x1f284, 0x1f290,
3882                 0x1f2c0, 0x1f2c0,
3883                 0x1f2e0, 0x1f2e0,
3884                 0x1f300, 0x1f384,
3885                 0x1f3c0, 0x1f3c8,
3886                 0x1f408, 0x1f40c,
3887                 0x1f440, 0x1f44c,
3888                 0x1f684, 0x1f690,
3889                 0x1f6c0, 0x1f6c0,
3890                 0x1f6e0, 0x1f6e0,
3891                 0x1f700, 0x1f784,
3892                 0x1f7c0, 0x1f7c8,
3893                 0x1f808, 0x1f80c,
3894                 0x1f840, 0x1f84c,
3895                 0x1fa84, 0x1fa90,
3896                 0x1fac0, 0x1fac0,
3897                 0x1fae0, 0x1fae0,
3898                 0x1fb00, 0x1fb84,
3899                 0x1fbc0, 0x1fbc8,
3900                 0x1fc08, 0x1fc0c,
3901                 0x1fc40, 0x1fc4c,
3902                 0x1fe84, 0x1fe90,
3903                 0x1fec0, 0x1fec0,
3904                 0x1fee0, 0x1fee0,
3905                 0x1ff00, 0x1ff84,
3906                 0x1ffc0, 0x1ffc8,
3907                 0x30000, 0x30030,
3908                 0x30100, 0x30144,
3909                 0x30190, 0x301d0,
3910                 0x30200, 0x30318,
3911                 0x30400, 0x3052c,
3912                 0x30540, 0x3061c,
3913                 0x30800, 0x30834,
3914                 0x308c0, 0x30908,
3915                 0x30910, 0x309ac,
3916                 0x30a00, 0x30a2c,
3917                 0x30a44, 0x30a50,
3918                 0x30a74, 0x30c24,
3919                 0x30d00, 0x30d00,
3920                 0x30d08, 0x30d14,
3921                 0x30d1c, 0x30d20,
3922                 0x30d3c, 0x30d50,
3923                 0x31200, 0x3120c,
3924                 0x31220, 0x31220,
3925                 0x31240, 0x31240,
3926                 0x31600, 0x3160c,
3927                 0x31a00, 0x31a1c,
3928                 0x31e00, 0x31e20,
3929                 0x31e38, 0x31e3c,
3930                 0x31e80, 0x31e80,
3931                 0x31e88, 0x31ea8,
3932                 0x31eb0, 0x31eb4,
3933                 0x31ec8, 0x31ed4,
3934                 0x31fb8, 0x32004,
3935                 0x32200, 0x32200,
3936                 0x32208, 0x32240,
3937                 0x32248, 0x32280,
3938                 0x32288, 0x322c0,
3939                 0x322c8, 0x322fc,
3940                 0x32600, 0x32630,
3941                 0x32a00, 0x32abc,
3942                 0x32b00, 0x32b70,
3943                 0x33000, 0x33048,
3944                 0x33060, 0x3309c,
3945                 0x330f0, 0x33148,
3946                 0x33160, 0x3319c,
3947                 0x331f0, 0x332e4,
3948                 0x332f8, 0x333e4,
3949                 0x333f8, 0x33448,
3950                 0x33460, 0x3349c,
3951                 0x334f0, 0x33548,
3952                 0x33560, 0x3359c,
3953                 0x335f0, 0x336e4,
3954                 0x336f8, 0x337e4,
3955                 0x337f8, 0x337fc,
3956                 0x33814, 0x33814,
3957                 0x3382c, 0x3382c,
3958                 0x33880, 0x3388c,
3959                 0x338e8, 0x338ec,
3960                 0x33900, 0x33948,
3961                 0x33960, 0x3399c,
3962                 0x339f0, 0x33ae4,
3963                 0x33af8, 0x33b10,
3964                 0x33b28, 0x33b28,
3965                 0x33b3c, 0x33b50,
3966                 0x33bf0, 0x33c10,
3967                 0x33c28, 0x33c28,
3968                 0x33c3c, 0x33c50,
3969                 0x33cf0, 0x33cfc,
3970                 0x34000, 0x34030,
3971                 0x34100, 0x34144,
3972                 0x34190, 0x341d0,
3973                 0x34200, 0x34318,
3974                 0x34400, 0x3452c,
3975                 0x34540, 0x3461c,
3976                 0x34800, 0x34834,
3977                 0x348c0, 0x34908,
3978                 0x34910, 0x349ac,
3979                 0x34a00, 0x34a2c,
3980                 0x34a44, 0x34a50,
3981                 0x34a74, 0x34c24,
3982                 0x34d00, 0x34d00,
3983                 0x34d08, 0x34d14,
3984                 0x34d1c, 0x34d20,
3985                 0x34d3c, 0x34d50,
3986                 0x35200, 0x3520c,
3987                 0x35220, 0x35220,
3988                 0x35240, 0x35240,
3989                 0x35600, 0x3560c,
3990                 0x35a00, 0x35a1c,
3991                 0x35e00, 0x35e20,
3992                 0x35e38, 0x35e3c,
3993                 0x35e80, 0x35e80,
3994                 0x35e88, 0x35ea8,
3995                 0x35eb0, 0x35eb4,
3996                 0x35ec8, 0x35ed4,
3997                 0x35fb8, 0x36004,
3998                 0x36200, 0x36200,
3999                 0x36208, 0x36240,
4000                 0x36248, 0x36280,
4001                 0x36288, 0x362c0,
4002                 0x362c8, 0x362fc,
4003                 0x36600, 0x36630,
4004                 0x36a00, 0x36abc,
4005                 0x36b00, 0x36b70,
4006                 0x37000, 0x37048,
4007                 0x37060, 0x3709c,
4008                 0x370f0, 0x37148,
4009                 0x37160, 0x3719c,
4010                 0x371f0, 0x372e4,
4011                 0x372f8, 0x373e4,
4012                 0x373f8, 0x37448,
4013                 0x37460, 0x3749c,
4014                 0x374f0, 0x37548,
4015                 0x37560, 0x3759c,
4016                 0x375f0, 0x376e4,
4017                 0x376f8, 0x377e4,
4018                 0x377f8, 0x377fc,
4019                 0x37814, 0x37814,
4020                 0x3782c, 0x3782c,
4021                 0x37880, 0x3788c,
4022                 0x378e8, 0x378ec,
4023                 0x37900, 0x37948,
4024                 0x37960, 0x3799c,
4025                 0x379f0, 0x37ae4,
4026                 0x37af8, 0x37b10,
4027                 0x37b28, 0x37b28,
4028                 0x37b3c, 0x37b50,
4029                 0x37bf0, 0x37c10,
4030                 0x37c28, 0x37c28,
4031                 0x37c3c, 0x37c50,
4032                 0x37cf0, 0x37cfc,
4033                 0x38000, 0x38030,
4034                 0x38100, 0x38144,
4035                 0x38190, 0x381d0,
4036                 0x38200, 0x38318,
4037                 0x38400, 0x3852c,
4038                 0x38540, 0x3861c,
4039                 0x38800, 0x38834,
4040                 0x388c0, 0x38908,
4041                 0x38910, 0x389ac,
4042                 0x38a00, 0x38a2c,
4043                 0x38a44, 0x38a50,
4044                 0x38a74, 0x38c24,
4045                 0x38d00, 0x38d00,
4046                 0x38d08, 0x38d14,
4047                 0x38d1c, 0x38d20,
4048                 0x38d3c, 0x38d50,
4049                 0x39200, 0x3920c,
4050                 0x39220, 0x39220,
4051                 0x39240, 0x39240,
4052                 0x39600, 0x3960c,
4053                 0x39a00, 0x39a1c,
4054                 0x39e00, 0x39e20,
4055                 0x39e38, 0x39e3c,
4056                 0x39e80, 0x39e80,
4057                 0x39e88, 0x39ea8,
4058                 0x39eb0, 0x39eb4,
4059                 0x39ec8, 0x39ed4,
4060                 0x39fb8, 0x3a004,
4061                 0x3a200, 0x3a200,
4062                 0x3a208, 0x3a240,
4063                 0x3a248, 0x3a280,
4064                 0x3a288, 0x3a2c0,
4065                 0x3a2c8, 0x3a2fc,
4066                 0x3a600, 0x3a630,
4067                 0x3aa00, 0x3aabc,
4068                 0x3ab00, 0x3ab70,
4069                 0x3b000, 0x3b048,
4070                 0x3b060, 0x3b09c,
4071                 0x3b0f0, 0x3b148,
4072                 0x3b160, 0x3b19c,
4073                 0x3b1f0, 0x3b2e4,
4074                 0x3b2f8, 0x3b3e4,
4075                 0x3b3f8, 0x3b448,
4076                 0x3b460, 0x3b49c,
4077                 0x3b4f0, 0x3b548,
4078                 0x3b560, 0x3b59c,
4079                 0x3b5f0, 0x3b6e4,
4080                 0x3b6f8, 0x3b7e4,
4081                 0x3b7f8, 0x3b7fc,
4082                 0x3b814, 0x3b814,
4083                 0x3b82c, 0x3b82c,
4084                 0x3b880, 0x3b88c,
4085                 0x3b8e8, 0x3b8ec,
4086                 0x3b900, 0x3b948,
4087                 0x3b960, 0x3b99c,
4088                 0x3b9f0, 0x3bae4,
4089                 0x3baf8, 0x3bb10,
4090                 0x3bb28, 0x3bb28,
4091                 0x3bb3c, 0x3bb50,
4092                 0x3bbf0, 0x3bc10,
4093                 0x3bc28, 0x3bc28,
4094                 0x3bc3c, 0x3bc50,
4095                 0x3bcf0, 0x3bcfc,
4096                 0x3c000, 0x3c030,
4097                 0x3c100, 0x3c144,
4098                 0x3c190, 0x3c1d0,
4099                 0x3c200, 0x3c318,
4100                 0x3c400, 0x3c52c,
4101                 0x3c540, 0x3c61c,
4102                 0x3c800, 0x3c834,
4103                 0x3c8c0, 0x3c908,
4104                 0x3c910, 0x3c9ac,
4105                 0x3ca00, 0x3ca2c,
4106                 0x3ca44, 0x3ca50,
4107                 0x3ca74, 0x3cc24,
4108                 0x3cd00, 0x3cd00,
4109                 0x3cd08, 0x3cd14,
4110                 0x3cd1c, 0x3cd20,
4111                 0x3cd3c, 0x3cd50,
4112                 0x3d200, 0x3d20c,
4113                 0x3d220, 0x3d220,
4114                 0x3d240, 0x3d240,
4115                 0x3d600, 0x3d60c,
4116                 0x3da00, 0x3da1c,
4117                 0x3de00, 0x3de20,
4118                 0x3de38, 0x3de3c,
4119                 0x3de80, 0x3de80,
4120                 0x3de88, 0x3dea8,
4121                 0x3deb0, 0x3deb4,
4122                 0x3dec8, 0x3ded4,
4123                 0x3dfb8, 0x3e004,
4124                 0x3e200, 0x3e200,
4125                 0x3e208, 0x3e240,
4126                 0x3e248, 0x3e280,
4127                 0x3e288, 0x3e2c0,
4128                 0x3e2c8, 0x3e2fc,
4129                 0x3e600, 0x3e630,
4130                 0x3ea00, 0x3eabc,
4131                 0x3eb00, 0x3eb70,
4132                 0x3f000, 0x3f048,
4133                 0x3f060, 0x3f09c,
4134                 0x3f0f0, 0x3f148,
4135                 0x3f160, 0x3f19c,
4136                 0x3f1f0, 0x3f2e4,
4137                 0x3f2f8, 0x3f3e4,
4138                 0x3f3f8, 0x3f448,
4139                 0x3f460, 0x3f49c,
4140                 0x3f4f0, 0x3f548,
4141                 0x3f560, 0x3f59c,
4142                 0x3f5f0, 0x3f6e4,
4143                 0x3f6f8, 0x3f7e4,
4144                 0x3f7f8, 0x3f7fc,
4145                 0x3f814, 0x3f814,
4146                 0x3f82c, 0x3f82c,
4147                 0x3f880, 0x3f88c,
4148                 0x3f8e8, 0x3f8ec,
4149                 0x3f900, 0x3f948,
4150                 0x3f960, 0x3f99c,
4151                 0x3f9f0, 0x3fae4,
4152                 0x3faf8, 0x3fb10,
4153                 0x3fb28, 0x3fb28,
4154                 0x3fb3c, 0x3fb50,
4155                 0x3fbf0, 0x3fc10,
4156                 0x3fc28, 0x3fc28,
4157                 0x3fc3c, 0x3fc50,
4158                 0x3fcf0, 0x3fcfc,
4159                 0x40000, 0x4000c,
4160                 0x40040, 0x40068,
4161                 0x4007c, 0x40144,
4162                 0x40180, 0x4018c,
4163                 0x40200, 0x40298,
4164                 0x402ac, 0x4033c,
4165                 0x403f8, 0x403fc,
4166                 0x41304, 0x413c4,
4167                 0x41400, 0x4141c,
4168                 0x41480, 0x414d0,
4169                 0x44000, 0x44078,
4170                 0x440c0, 0x44278,
4171                 0x442c0, 0x44478,
4172                 0x444c0, 0x44678,
4173                 0x446c0, 0x44878,
4174                 0x448c0, 0x449fc,
4175                 0x45000, 0x45068,
4176                 0x45080, 0x45084,
4177                 0x450a0, 0x450b0,
4178                 0x45200, 0x45268,
4179                 0x45280, 0x45284,
4180                 0x452a0, 0x452b0,
4181                 0x460c0, 0x460e4,
4182                 0x47000, 0x4708c,
4183                 0x47200, 0x47250,
4184                 0x47400, 0x47420,
4185                 0x47600, 0x47618,
4186                 0x47800, 0x47814,
4187                 0x48000, 0x4800c,
4188                 0x48040, 0x48068,
4189                 0x4807c, 0x48144,
4190                 0x48180, 0x4818c,
4191                 0x48200, 0x48298,
4192                 0x482ac, 0x4833c,
4193                 0x483f8, 0x483fc,
4194                 0x49304, 0x493c4,
4195                 0x49400, 0x4941c,
4196                 0x49480, 0x494d0,
4197                 0x4c000, 0x4c078,
4198                 0x4c0c0, 0x4c278,
4199                 0x4c2c0, 0x4c478,
4200                 0x4c4c0, 0x4c678,
4201                 0x4c6c0, 0x4c878,
4202                 0x4c8c0, 0x4c9fc,
4203                 0x4d000, 0x4d068,
4204                 0x4d080, 0x4d084,
4205                 0x4d0a0, 0x4d0b0,
4206                 0x4d200, 0x4d268,
4207                 0x4d280, 0x4d284,
4208                 0x4d2a0, 0x4d2b0,
4209                 0x4e0c0, 0x4e0e4,
4210                 0x4f000, 0x4f08c,
4211                 0x4f200, 0x4f250,
4212                 0x4f400, 0x4f420,
4213                 0x4f600, 0x4f618,
4214                 0x4f800, 0x4f814,
4215                 0x50000, 0x500cc,
4216                 0x50400, 0x50400,
4217                 0x50800, 0x508cc,
4218                 0x50c00, 0x50c00,
4219                 0x51000, 0x5101c,
4220                 0x51300, 0x51308,
4221         };
4222
4223         if (is_t4(sc)) {
4224                 reg_ranges = &t4_reg_ranges[0];
4225                 n = nitems(t4_reg_ranges);
4226         } else {
4227                 reg_ranges = &t5_reg_ranges[0];
4228                 n = nitems(t5_reg_ranges);
4229         }
4230
4231         regs->version = chip_id(sc) | chip_rev(sc) << 10;
4232         for (i = 0; i < n; i += 2)
4233                 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
4234 }
4235
4236 static void
4237 cxgbe_tick(void *arg)
4238 {
4239         struct port_info *pi = arg;
4240         struct adapter *sc = pi->adapter;
4241         struct ifnet *ifp = pi->ifp;
4242         struct sge_txq *txq;
4243         int i, drops;
4244         struct port_stats *s = &pi->stats;
4245
4246         PORT_LOCK(pi);
4247         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4248                 PORT_UNLOCK(pi);
4249                 return; /* without scheduling another callout */
4250         }
4251
4252         t4_get_port_stats(sc, pi->tx_chan, s);
4253
4254         ifp->if_opackets = s->tx_frames - s->tx_pause;
4255         ifp->if_ipackets = s->rx_frames - s->rx_pause;
4256         ifp->if_obytes = s->tx_octets - s->tx_pause * 64;
4257         ifp->if_ibytes = s->rx_octets - s->rx_pause * 64;
4258         ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause;
4259         ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause;
4260         ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
4261             s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
4262             s->rx_trunc3;
4263         for (i = 0; i < 4; i++) {
4264                 if (pi->rx_chan_map & (1 << i)) {
4265                         uint32_t v;
4266
4267                         /*
4268                          * XXX: indirect reads from the same ADDR/DATA pair can
4269                          * race with each other.
4270                          */
4271                         t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
4272                             1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4273                         ifp->if_iqdrops += v;
4274                 }
4275         }
4276
4277         drops = s->tx_drop;
4278         for_each_txq(pi, i, txq)
4279                 drops += txq->br->br_drops;
4280         ifp->if_snd.ifq_drops = drops;
4281
4282         ifp->if_oerrors = s->tx_error_frames;
4283         ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long +
4284             s->rx_fcs_err + s->rx_len_err;
4285
4286         callout_schedule(&pi->tick, hz);
4287         PORT_UNLOCK(pi);
4288 }
4289
4290 static void
4291 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4292 {
4293         struct ifnet *vlan;
4294
4295         if (arg != ifp || ifp->if_type != IFT_ETHER)
4296                 return;
4297
4298         vlan = VLAN_DEVAT(ifp, vid);
4299         VLAN_SETCOOKIE(vlan, ifp);
4300 }
4301
4302 static int
4303 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4304 {
4305
4306 #ifdef INVARIANTS
4307         panic("%s: opcode 0x%02x on iq %p with payload %p",
4308             __func__, rss->opcode, iq, m);
4309 #else
4310         log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
4311             __func__, rss->opcode, iq, m);
4312         m_freem(m);
4313 #endif
4314         return (EDOOFUS);
4315 }
4316
4317 int
4318 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
4319 {
4320         uintptr_t *loc, new;
4321
4322         if (opcode >= nitems(sc->cpl_handler))
4323                 return (EINVAL);
4324
4325         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
4326         loc = (uintptr_t *) &sc->cpl_handler[opcode];
4327         atomic_store_rel_ptr(loc, new);
4328
4329         return (0);
4330 }
4331
4332 static int
4333 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
4334 {
4335
4336 #ifdef INVARIANTS
4337         panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
4338 #else
4339         log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
4340             __func__, iq, ctrl);
4341 #endif
4342         return (EDOOFUS);
4343 }
4344
4345 int
4346 t4_register_an_handler(struct adapter *sc, an_handler_t h)
4347 {
4348         uintptr_t *loc, new;
4349
4350         new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
4351         loc = (uintptr_t *) &sc->an_handler;
4352         atomic_store_rel_ptr(loc, new);
4353
4354         return (0);
4355 }
4356
4357 static int
4358 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
4359 {
4360         const struct cpl_fw6_msg *cpl =
4361             __containerof(rpl, struct cpl_fw6_msg, data[0]);
4362
4363 #ifdef INVARIANTS
4364         panic("%s: fw_msg type %d", __func__, cpl->type);
4365 #else
4366         log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
4367 #endif
4368         return (EDOOFUS);
4369 }
4370
4371 int
4372 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
4373 {
4374         uintptr_t *loc, new;
4375
4376         if (type >= nitems(sc->fw_msg_handler))
4377                 return (EINVAL);
4378
4379         /*
4380          * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
4381          * handler dispatch table.  Reject any attempt to install a handler for
4382          * this subtype.
4383          */
4384         if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
4385                 return (EINVAL);
4386
4387         new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
4388         loc = (uintptr_t *) &sc->fw_msg_handler[type];
4389         atomic_store_rel_ptr(loc, new);
4390
4391         return (0);
4392 }
4393
4394 static int
4395 t4_sysctls(struct adapter *sc)
4396 {
4397         struct sysctl_ctx_list *ctx;
4398         struct sysctl_oid *oid;
4399         struct sysctl_oid_list *children, *c0;
4400         static char *caps[] = {
4401                 "\20\1PPP\2QFC\3DCBX",                  /* caps[0] linkcaps */
4402                 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL"        /* caps[1] niccaps */
4403                     "\6HASHFILTER\7ETHOFLD",
4404                 "\20\1TOE",                             /* caps[2] toecaps */
4405                 "\20\1RDDP\2RDMAC",                     /* caps[3] rdmacaps */
4406                 "\20\1INITIATOR_PDU\2TARGET_PDU"        /* caps[4] iscsicaps */
4407                     "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
4408                     "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
4409                 "\20\1INITIATOR\2TARGET\3CTRL_OFLD"     /* caps[5] fcoecaps */
4410                     "\4PO_INITIAOR\5PO_TARGET"
4411         };
4412         static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
4413
4414         ctx = device_get_sysctl_ctx(sc->dev);
4415
4416         /*
4417          * dev.t4nex.X.
4418          */
4419         oid = device_get_sysctl_tree(sc->dev);
4420         c0 = children = SYSCTL_CHILDREN(oid);
4421
4422         sc->sc_do_rxcopy = 1;
4423         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
4424             &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
4425
4426         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
4427             sc->params.nports, "# of ports");
4428
4429         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
4430             NULL, chip_rev(sc), "chip hardware revision");
4431
4432         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
4433             CTLFLAG_RD, sc->fw_version, 0, "firmware version");
4434
4435         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
4436             CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
4437
4438         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
4439             sc->cfcsum, "config file checksum");
4440
4441         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
4442             CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
4443             sysctl_bitfield, "A", "available doorbells");
4444
4445         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
4446             CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
4447             sysctl_bitfield, "A", "available link capabilities");
4448
4449         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
4450             CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
4451             sysctl_bitfield, "A", "available NIC capabilities");
4452
4453         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
4454             CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
4455             sysctl_bitfield, "A", "available TCP offload capabilities");
4456
4457         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
4458             CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
4459             sysctl_bitfield, "A", "available RDMA capabilities");
4460
4461         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
4462             CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
4463             sysctl_bitfield, "A", "available iSCSI capabilities");
4464
4465         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
4466             CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
4467             sysctl_bitfield, "A", "available FCoE capabilities");
4468
4469         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
4470             sc->params.vpd.cclk, "core clock frequency (in KHz)");
4471
4472         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
4473             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
4474             sizeof(sc->sge.timer_val), sysctl_int_array, "A",
4475             "interrupt holdoff timer values (us)");
4476
4477         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
4478             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
4479             sizeof(sc->sge.counter_val), sysctl_int_array, "A",
4480             "interrupt holdoff packet counter values");
4481
4482         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
4483             NULL, sc->tids.nftids, "number of filters");
4484
4485         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
4486             CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
4487             "chip temperature (in Celsius)");
4488
4489         t4_sge_sysctls(sc, ctx, children);
4490
4491         sc->lro_timeout = 100;
4492         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
4493             &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
4494
4495 #ifdef SBUF_DRAIN
4496         /*
4497          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
4498          */
4499         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
4500             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
4501             "logs and miscellaneous information");
4502         children = SYSCTL_CHILDREN(oid);
4503
4504         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
4505             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4506             sysctl_cctrl, "A", "congestion control");
4507
4508         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
4509             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4510             sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
4511
4512         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
4513             CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
4514             sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
4515
4516         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
4517             CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
4518             sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
4519
4520         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
4521             CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
4522             sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
4523
4524         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
4525             CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
4526             sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
4527
4528         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
4529             CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
4530             sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
4531
4532         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
4533             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4534             sysctl_cim_la, "A", "CIM logic analyzer");
4535
4536         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
4537             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4538             sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
4539
4540         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
4541             CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
4542             sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
4543
4544         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
4545             CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
4546             sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
4547
4548         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
4549             CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
4550             sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
4551
4552         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
4553             CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
4554             sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
4555
4556         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
4557             CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
4558             sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
4559
4560         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
4561             CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
4562             sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
4563
4564         if (is_t5(sc)) {
4565                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
4566                     CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
4567                     sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
4568
4569                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
4570                     CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
4571                     sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
4572         }
4573
4574         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
4575             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4576             sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
4577
4578         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
4579             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4580             sysctl_cim_qcfg, "A", "CIM queue configuration");
4581
4582         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
4583             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4584             sysctl_cpl_stats, "A", "CPL statistics");
4585
4586         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
4587             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4588             sysctl_ddp_stats, "A", "DDP statistics");
4589
4590         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
4591             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4592             sysctl_devlog, "A", "firmware's device log");
4593
4594         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
4595             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4596             sysctl_fcoe_stats, "A", "FCoE statistics");
4597
4598         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
4599             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4600             sysctl_hw_sched, "A", "hardware scheduler ");
4601
4602         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
4603             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4604             sysctl_l2t, "A", "hardware L2 table");
4605
4606         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
4607             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4608             sysctl_lb_stats, "A", "loopback statistics");
4609
4610         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
4611             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4612             sysctl_meminfo, "A", "memory regions");
4613
4614         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
4615             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4616             sysctl_mps_tcam, "A", "MPS TCAM entries");
4617
4618         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
4619             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4620             sysctl_path_mtus, "A", "path MTUs");
4621
4622         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
4623             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4624             sysctl_pm_stats, "A", "PM statistics");
4625
4626         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
4627             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4628             sysctl_rdma_stats, "A", "RDMA statistics");
4629
4630         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
4631             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4632             sysctl_tcp_stats, "A", "TCP statistics");
4633
4634         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
4635             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4636             sysctl_tids, "A", "TID information");
4637
4638         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
4639             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4640             sysctl_tp_err_stats, "A", "TP error statistics");
4641
4642         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
4643             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4644             sysctl_tp_la, "A", "TP logic analyzer");
4645
4646         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
4647             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4648             sysctl_tx_rate, "A", "Tx rate");
4649
4650         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
4651             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4652             sysctl_ulprx_la, "A", "ULPRX logic analyzer");
4653
4654         if (is_t5(sc)) {
4655                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
4656                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4657                     sysctl_wcwr_stats, "A", "write combined work requests");
4658         }
4659 #endif
4660
4661 #ifdef TCP_OFFLOAD
4662         if (is_offload(sc)) {
4663                 /*
4664                  * dev.t4nex.X.toe.
4665                  */
4666                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
4667                     NULL, "TOE parameters");
4668                 children = SYSCTL_CHILDREN(oid);
4669
4670                 sc->tt.sndbuf = 256 * 1024;
4671                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
4672                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
4673
4674                 sc->tt.ddp = 0;
4675                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
4676                     &sc->tt.ddp, 0, "DDP allowed");
4677
4678                 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
4679                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
4680                     &sc->tt.indsz, 0, "DDP max indicate size allowed");
4681
4682                 sc->tt.ddp_thres =
4683                     G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
4684                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
4685                     &sc->tt.ddp_thres, 0, "DDP threshold");
4686
4687                 sc->tt.rx_coalesce = 1;
4688                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
4689                     CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
4690         }
4691 #endif
4692
4693
4694         return (0);
4695 }
4696
4697 static int
4698 cxgbe_sysctls(struct port_info *pi)
4699 {
4700         struct sysctl_ctx_list *ctx;
4701         struct sysctl_oid *oid;
4702         struct sysctl_oid_list *children;
4703         struct adapter *sc = pi->adapter;
4704
4705         ctx = device_get_sysctl_ctx(pi->dev);
4706
4707         /*
4708          * dev.cxgbe.X.
4709          */
4710         oid = device_get_sysctl_tree(pi->dev);
4711         children = SYSCTL_CHILDREN(oid);
4712
4713         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
4714            CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
4715         if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
4716                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
4717                     CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
4718                     "PHY temperature (in Celsius)");
4719                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
4720                     CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
4721                     "PHY firmware version");
4722         }
4723         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
4724             &pi->nrxq, 0, "# of rx queues");
4725         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
4726             &pi->ntxq, 0, "# of tx queues");
4727         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
4728             &pi->first_rxq, 0, "index of first rx queue");
4729         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
4730             &pi->first_txq, 0, "index of first tx queue");
4731         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
4732             CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU",
4733             "Reserve queue 0 for non-flowid packets");
4734
4735 #ifdef TCP_OFFLOAD
4736         if (is_offload(sc)) {
4737                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
4738                     &pi->nofldrxq, 0,
4739                     "# of rx queues for offloaded TCP connections");
4740                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
4741                     &pi->nofldtxq, 0,
4742                     "# of tx queues for offloaded TCP connections");
4743                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
4744                     CTLFLAG_RD, &pi->first_ofld_rxq, 0,
4745                     "index of first TOE rx queue");
4746                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
4747                     CTLFLAG_RD, &pi->first_ofld_txq, 0,
4748                     "index of first TOE tx queue");
4749         }
4750 #endif
4751 #ifdef DEV_NETMAP
4752         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
4753             &pi->nnmrxq, 0, "# of rx queues for netmap");
4754         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
4755             &pi->nnmtxq, 0, "# of tx queues for netmap");
4756         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
4757             CTLFLAG_RD, &pi->first_nm_rxq, 0,
4758             "index of first netmap rx queue");
4759         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
4760             CTLFLAG_RD, &pi->first_nm_txq, 0,
4761             "index of first netmap tx queue");
4762 #endif
4763
4764         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
4765             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
4766             "holdoff timer index");
4767         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
4768             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
4769             "holdoff packet counter index");
4770
4771         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
4772             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
4773             "rx queue size");
4774         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
4775             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
4776             "tx queue size");
4777
4778         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
4779             CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings,
4780             "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
4781
4782         /*
4783          * dev.cxgbe.X.stats.
4784          */
4785         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
4786             NULL, "port statistics");
4787         children = SYSCTL_CHILDREN(oid);
4788
4789 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
4790         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
4791             CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
4792             sysctl_handle_t4_reg64, "QU", desc)
4793
4794         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
4795             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
4796         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
4797             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
4798         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
4799             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
4800         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
4801             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
4802         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
4803             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
4804         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
4805             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
4806         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
4807             "# of tx frames in this range",
4808             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
4809         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
4810             "# of tx frames in this range",
4811             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
4812         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
4813             "# of tx frames in this range",
4814             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
4815         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
4816             "# of tx frames in this range",
4817             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
4818         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
4819             "# of tx frames in this range",
4820             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
4821         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
4822             "# of tx frames in this range",
4823             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
4824         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
4825             "# of tx frames in this range",
4826             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
4827         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
4828             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
4829         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
4830             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
4831         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
4832             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
4833         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
4834             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
4835         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
4836             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
4837         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
4838             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
4839         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
4840             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
4841         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
4842             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
4843         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
4844             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
4845         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
4846             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
4847
4848         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
4849             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
4850         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
4851             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
4852         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
4853             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
4854         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
4855             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
4856         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
4857             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
4858         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
4859             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
4860         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
4861             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
4862         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
4863             "# of frames received with bad FCS",
4864             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
4865         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
4866             "# of frames received with length error",
4867             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
4868         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
4869             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
4870         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
4871             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
4872         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
4873             "# of rx frames in this range",
4874             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
4875         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
4876             "# of rx frames in this range",
4877             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
4878         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
4879             "# of rx frames in this range",
4880             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
4881         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
4882             "# of rx frames in this range",
4883             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
4884         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
4885             "# of rx frames in this range",
4886             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
4887         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
4888             "# of rx frames in this range",
4889             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
4890         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
4891             "# of rx frames in this range",
4892             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
4893         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
4894             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
4895         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
4896             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
4897         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
4898             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
4899         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
4900             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
4901         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
4902             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
4903         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
4904             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
4905         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
4906             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
4907         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
4908             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
4909         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
4910             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
4911
4912 #undef SYSCTL_ADD_T4_REG64
4913
4914 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
4915         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
4916             &pi->stats.name, desc)
4917
4918         /* We get these from port_stats and they may be stale by upto 1s */
4919         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
4920             "# drops due to buffer-group 0 overflows");
4921         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
4922             "# drops due to buffer-group 1 overflows");
4923         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
4924             "# drops due to buffer-group 2 overflows");
4925         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
4926             "# drops due to buffer-group 3 overflows");
4927         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
4928             "# of buffer-group 0 truncated packets");
4929         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
4930             "# of buffer-group 1 truncated packets");
4931         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
4932             "# of buffer-group 2 truncated packets");
4933         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
4934             "# of buffer-group 3 truncated packets");
4935
4936 #undef SYSCTL_ADD_T4_PORTSTAT
4937
4938         return (0);
4939 }
4940
4941 static int
4942 sysctl_int_array(SYSCTL_HANDLER_ARGS)
4943 {
4944         int rc, *i;
4945         struct sbuf sb;
4946
4947         sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
4948         for (i = arg1; arg2; arg2 -= sizeof(int), i++)
4949                 sbuf_printf(&sb, "%d ", *i);
4950         sbuf_trim(&sb);
4951         sbuf_finish(&sb);
4952         rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
4953         sbuf_delete(&sb);
4954         return (rc);
4955 }
4956
4957 static int
4958 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
4959 {
4960         int rc;
4961         struct sbuf *sb;
4962
4963         rc = sysctl_wire_old_buffer(req, 0);
4964         if (rc != 0)
4965                 return(rc);
4966
4967         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
4968         if (sb == NULL)
4969                 return (ENOMEM);
4970
4971         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
4972         rc = sbuf_finish(sb);
4973         sbuf_delete(sb);
4974
4975         return (rc);
4976 }
4977
4978 static int
4979 sysctl_btphy(SYSCTL_HANDLER_ARGS)
4980 {
4981         struct port_info *pi = arg1;
4982         int op = arg2;
4983         struct adapter *sc = pi->adapter;
4984         u_int v;
4985         int rc;
4986
4987         rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt");
4988         if (rc)
4989                 return (rc);
4990         /* XXX: magic numbers */
4991         rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
4992             &v);
4993         end_synchronized_op(sc, 0);
4994         if (rc)
4995                 return (rc);
4996         if (op == 0)
4997                 v /= 256;
4998
4999         rc = sysctl_handle_int(oidp, &v, 0, req);
5000         return (rc);
5001 }
5002
5003 static int
5004 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
5005 {
5006         struct port_info *pi = arg1;
5007         int rc, val;
5008
5009         val = pi->rsrv_noflowq;
5010         rc = sysctl_handle_int(oidp, &val, 0, req);
5011         if (rc != 0 || req->newptr == NULL)
5012                 return (rc);
5013
5014         if ((val >= 1) && (pi->ntxq > 1))
5015                 pi->rsrv_noflowq = 1;
5016         else
5017                 pi->rsrv_noflowq = 0;
5018
5019         return (rc);
5020 }
5021
5022 static int
5023 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
5024 {
5025         struct port_info *pi = arg1;
5026         struct adapter *sc = pi->adapter;
5027         int idx, rc, i;
5028         struct sge_rxq *rxq;
5029 #ifdef TCP_OFFLOAD
5030         struct sge_ofld_rxq *ofld_rxq;
5031 #endif
5032         uint8_t v;
5033
5034         idx = pi->tmr_idx;
5035
5036         rc = sysctl_handle_int(oidp, &idx, 0, req);
5037         if (rc != 0 || req->newptr == NULL)
5038                 return (rc);
5039
5040         if (idx < 0 || idx >= SGE_NTIMERS)
5041                 return (EINVAL);
5042
5043         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5044             "t4tmr");
5045         if (rc)
5046                 return (rc);
5047
5048         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
5049         for_each_rxq(pi, i, rxq) {
5050 #ifdef atomic_store_rel_8
5051                 atomic_store_rel_8(&rxq->iq.intr_params, v);
5052 #else
5053                 rxq->iq.intr_params = v;
5054 #endif
5055         }
5056 #ifdef TCP_OFFLOAD
5057         for_each_ofld_rxq(pi, i, ofld_rxq) {
5058 #ifdef atomic_store_rel_8
5059                 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
5060 #else
5061                 ofld_rxq->iq.intr_params = v;
5062 #endif
5063         }
5064 #endif
5065         pi->tmr_idx = idx;
5066
5067         end_synchronized_op(sc, LOCK_HELD);
5068         return (0);
5069 }
5070
5071 static int
5072 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
5073 {
5074         struct port_info *pi = arg1;
5075         struct adapter *sc = pi->adapter;
5076         int idx, rc;
5077
5078         idx = pi->pktc_idx;
5079
5080         rc = sysctl_handle_int(oidp, &idx, 0, req);
5081         if (rc != 0 || req->newptr == NULL)
5082                 return (rc);
5083
5084         if (idx < -1 || idx >= SGE_NCOUNTERS)
5085                 return (EINVAL);
5086
5087         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5088             "t4pktc");
5089         if (rc)
5090                 return (rc);
5091
5092         if (pi->flags & PORT_INIT_DONE)
5093                 rc = EBUSY; /* cannot be changed once the queues are created */
5094         else
5095                 pi->pktc_idx = idx;
5096
5097         end_synchronized_op(sc, LOCK_HELD);
5098         return (rc);
5099 }
5100
5101 static int
5102 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
5103 {
5104         struct port_info *pi = arg1;
5105         struct adapter *sc = pi->adapter;
5106         int qsize, rc;
5107
5108         qsize = pi->qsize_rxq;
5109
5110         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5111         if (rc != 0 || req->newptr == NULL)
5112                 return (rc);
5113
5114         if (qsize < 128 || (qsize & 7))
5115                 return (EINVAL);
5116
5117         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5118             "t4rxqs");
5119         if (rc)
5120                 return (rc);
5121
5122         if (pi->flags & PORT_INIT_DONE)
5123                 rc = EBUSY; /* cannot be changed once the queues are created */
5124         else
5125                 pi->qsize_rxq = qsize;
5126
5127         end_synchronized_op(sc, LOCK_HELD);
5128         return (rc);
5129 }
5130
5131 static int
5132 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
5133 {
5134         struct port_info *pi = arg1;
5135         struct adapter *sc = pi->adapter;
5136         int qsize, rc;
5137
5138         qsize = pi->qsize_txq;
5139
5140         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5141         if (rc != 0 || req->newptr == NULL)
5142                 return (rc);
5143
5144         /* bufring size must be powerof2 */
5145         if (qsize < 128 || !powerof2(qsize))
5146                 return (EINVAL);
5147
5148         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5149             "t4txqs");
5150         if (rc)
5151                 return (rc);
5152
5153         if (pi->flags & PORT_INIT_DONE)
5154                 rc = EBUSY; /* cannot be changed once the queues are created */
5155         else
5156                 pi->qsize_txq = qsize;
5157
5158         end_synchronized_op(sc, LOCK_HELD);
5159         return (rc);
5160 }
5161
5162 static int
5163 sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
5164 {
5165         struct port_info *pi = arg1;
5166         struct adapter *sc = pi->adapter;
5167         struct link_config *lc = &pi->link_cfg;
5168         int rc;
5169
5170         if (req->newptr == NULL) {
5171                 struct sbuf *sb;
5172                 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX";
5173
5174                 rc = sysctl_wire_old_buffer(req, 0);
5175                 if (rc != 0)
5176                         return(rc);
5177
5178                 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5179                 if (sb == NULL)
5180                         return (ENOMEM);
5181
5182                 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits);
5183                 rc = sbuf_finish(sb);
5184                 sbuf_delete(sb);
5185         } else {
5186                 char s[2];
5187                 int n;
5188
5189                 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX));
5190                 s[1] = 0;
5191
5192                 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
5193                 if (rc != 0)
5194                         return(rc);
5195
5196                 if (s[1] != 0)
5197                         return (EINVAL);
5198                 if (s[0] < '0' || s[0] > '9')
5199                         return (EINVAL);        /* not a number */
5200                 n = s[0] - '0';
5201                 if (n & ~(PAUSE_TX | PAUSE_RX))
5202                         return (EINVAL);        /* some other bit is set too */
5203
5204                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE");
5205                 if (rc)
5206                         return (rc);
5207                 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) {
5208                         int link_ok = lc->link_ok;
5209
5210                         lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX);
5211                         lc->requested_fc |= n;
5212                         rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc);
5213                         lc->link_ok = link_ok;  /* restore */
5214                 }
5215                 end_synchronized_op(sc, 0);
5216         }
5217
5218         return (rc);
5219 }
5220
5221 static int
5222 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
5223 {
5224         struct adapter *sc = arg1;
5225         int reg = arg2;
5226         uint64_t val;
5227
5228         val = t4_read_reg64(sc, reg);
5229
5230         return (sysctl_handle_64(oidp, &val, 0, req));
5231 }
5232
5233 static int
5234 sysctl_temperature(SYSCTL_HANDLER_ARGS)
5235 {
5236         struct adapter *sc = arg1;
5237         int rc, t;
5238         uint32_t param, val;
5239
5240         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
5241         if (rc)
5242                 return (rc);
5243         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5244             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5245             V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
5246         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
5247         end_synchronized_op(sc, 0);
5248         if (rc)
5249                 return (rc);
5250
5251         /* unknown is returned as 0 but we display -1 in that case */
5252         t = val == 0 ? -1 : val;
5253
5254         rc = sysctl_handle_int(oidp, &t, 0, req);
5255         return (rc);
5256 }
5257
5258 #ifdef SBUF_DRAIN
5259 static int
5260 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
5261 {
5262         struct adapter *sc = arg1;
5263         struct sbuf *sb;
5264         int rc, i;
5265         uint16_t incr[NMTUS][NCCTRL_WIN];
5266         static const char *dec_fac[] = {
5267                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
5268                 "0.9375"
5269         };
5270
5271         rc = sysctl_wire_old_buffer(req, 0);
5272         if (rc != 0)
5273                 return (rc);
5274
5275         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5276         if (sb == NULL)
5277                 return (ENOMEM);
5278
5279         t4_read_cong_tbl(sc, incr);
5280
5281         for (i = 0; i < NCCTRL_WIN; ++i) {
5282                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
5283                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
5284                     incr[5][i], incr[6][i], incr[7][i]);
5285                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
5286                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
5287                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
5288                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
5289         }
5290
5291         rc = sbuf_finish(sb);
5292         sbuf_delete(sb);
5293
5294         return (rc);
5295 }
5296
5297 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
5298         "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",   /* ibq's */
5299         "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
5300         "SGE0-RX", "SGE1-RX"    /* additional obq's (T5 onwards) */
5301 };
5302
5303 static int
5304 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
5305 {
5306         struct adapter *sc = arg1;
5307         struct sbuf *sb;
5308         int rc, i, n, qid = arg2;
5309         uint32_t *buf, *p;
5310         char *qtype;
5311         u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
5312
5313         KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
5314             ("%s: bad qid %d\n", __func__, qid));
5315
5316         if (qid < CIM_NUM_IBQ) {
5317                 /* inbound queue */
5318                 qtype = "IBQ";
5319                 n = 4 * CIM_IBQ_SIZE;
5320                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5321                 rc = t4_read_cim_ibq(sc, qid, buf, n);
5322         } else {
5323                 /* outbound queue */
5324                 qtype = "OBQ";
5325                 qid -= CIM_NUM_IBQ;
5326                 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
5327                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5328                 rc = t4_read_cim_obq(sc, qid, buf, n);
5329         }
5330
5331         if (rc < 0) {
5332                 rc = -rc;
5333                 goto done;
5334         }
5335         n = rc * sizeof(uint32_t);      /* rc has # of words actually read */
5336
5337         rc = sysctl_wire_old_buffer(req, 0);
5338         if (rc != 0)
5339                 goto done;
5340
5341         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5342         if (sb == NULL) {
5343                 rc = ENOMEM;
5344                 goto done;
5345         }
5346
5347         sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
5348         for (i = 0, p = buf; i < n; i += 16, p += 4)
5349                 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
5350                     p[2], p[3]);
5351
5352         rc = sbuf_finish(sb);
5353         sbuf_delete(sb);
5354 done:
5355         free(buf, M_CXGBE);
5356         return (rc);
5357 }
5358
5359 static int
5360 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
5361 {
5362         struct adapter *sc = arg1;
5363         u_int cfg;
5364         struct sbuf *sb;
5365         uint32_t *buf, *p;
5366         int rc;
5367
5368         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
5369         if (rc != 0)
5370                 return (rc);
5371
5372         rc = sysctl_wire_old_buffer(req, 0);
5373         if (rc != 0)
5374                 return (rc);
5375
5376         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5377         if (sb == NULL)
5378                 return (ENOMEM);
5379
5380         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
5381             M_ZERO | M_WAITOK);
5382
5383         rc = -t4_cim_read_la(sc, buf, NULL);
5384         if (rc != 0)
5385                 goto done;
5386
5387         sbuf_printf(sb, "Status   Data      PC%s",
5388             cfg & F_UPDBGLACAPTPCONLY ? "" :
5389             "     LS0Stat  LS0Addr             LS0Data");
5390
5391         KASSERT((sc->params.cim_la_size & 7) == 0,
5392             ("%s: p will walk off the end of buf", __func__));
5393
5394         for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
5395                 if (cfg & F_UPDBGLACAPTPCONLY) {
5396                         sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
5397                             p[6], p[7]);
5398                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
5399                             (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
5400                             p[4] & 0xff, p[5] >> 8);
5401                         sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
5402                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5403                             p[1] & 0xf, p[2] >> 4);
5404                 } else {
5405                         sbuf_printf(sb,
5406                             "\n  %02x   %x%07x %x%07x %08x %08x "
5407                             "%08x%08x%08x%08x",
5408                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5409                             p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
5410                             p[6], p[7]);
5411                 }
5412         }
5413
5414         rc = sbuf_finish(sb);
5415         sbuf_delete(sb);
5416 done:
5417         free(buf, M_CXGBE);
5418         return (rc);
5419 }
5420
5421 static int
5422 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
5423 {
5424         struct adapter *sc = arg1;
5425         u_int i;
5426         struct sbuf *sb;
5427         uint32_t *buf, *p;
5428         int rc;
5429
5430         rc = sysctl_wire_old_buffer(req, 0);
5431         if (rc != 0)
5432                 return (rc);
5433
5434         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5435         if (sb == NULL)
5436                 return (ENOMEM);
5437
5438         buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
5439             M_ZERO | M_WAITOK);
5440
5441         t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
5442         p = buf;
5443
5444         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5445                 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
5446                     p[1], p[0]);
5447         }
5448
5449         sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
5450         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5451                 sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
5452                     (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
5453                     (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
5454                     (p[1] >> 2) | ((p[2] & 3) << 30),
5455                     (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
5456                     p[0] & 1);
5457         }
5458
5459         rc = sbuf_finish(sb);
5460         sbuf_delete(sb);
5461         free(buf, M_CXGBE);
5462         return (rc);
5463 }
5464
5465 static int
5466 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
5467 {
5468         struct adapter *sc = arg1;
5469         u_int i;
5470         struct sbuf *sb;
5471         uint32_t *buf, *p;
5472         int rc;
5473
5474         rc = sysctl_wire_old_buffer(req, 0);
5475         if (rc != 0)
5476                 return (rc);
5477
5478         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5479         if (sb == NULL)
5480                 return (ENOMEM);
5481
5482         buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
5483             M_ZERO | M_WAITOK);
5484
5485         t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
5486         p = buf;
5487
5488         sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
5489         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5490                 sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
5491                     (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
5492                     p[4], p[3], p[2], p[1], p[0]);
5493         }
5494
5495         sbuf_printf(sb, "\n\nCntl ID               Data");
5496         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5497                 sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
5498                     (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
5499         }
5500
5501         rc = sbuf_finish(sb);
5502         sbuf_delete(sb);
5503         free(buf, M_CXGBE);
5504         return (rc);
5505 }
5506
5507 static int
5508 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
5509 {
5510         struct adapter *sc = arg1;
5511         struct sbuf *sb;
5512         int rc, i;
5513         uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5514         uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5515         uint16_t thres[CIM_NUM_IBQ];
5516         uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
5517         uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
5518         u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
5519
5520         if (is_t4(sc)) {
5521                 cim_num_obq = CIM_NUM_OBQ;
5522                 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
5523                 obq_rdaddr = A_UP_OBQ_0_REALADDR;
5524         } else {
5525                 cim_num_obq = CIM_NUM_OBQ_T5;
5526                 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
5527                 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
5528         }
5529         nq = CIM_NUM_IBQ + cim_num_obq;
5530
5531         rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
5532         if (rc == 0)
5533                 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
5534         if (rc != 0)
5535                 return (rc);
5536
5537         t4_read_cimq_cfg(sc, base, size, thres);
5538
5539         rc = sysctl_wire_old_buffer(req, 0);
5540         if (rc != 0)
5541                 return (rc);
5542
5543         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5544         if (sb == NULL)
5545                 return (ENOMEM);
5546
5547         sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
5548
5549         for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
5550                 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
5551                     qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
5552                     G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5553                     G_QUEREMFLITS(p[2]) * 16);
5554         for ( ; i < nq; i++, p += 4, wr += 2)
5555                 sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
5556                     base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
5557                     wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5558                     G_QUEREMFLITS(p[2]) * 16);
5559
5560         rc = sbuf_finish(sb);
5561         sbuf_delete(sb);
5562
5563         return (rc);
5564 }
5565
5566 static int
5567 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
5568 {
5569         struct adapter *sc = arg1;
5570         struct sbuf *sb;
5571         int rc;
5572         struct tp_cpl_stats stats;
5573
5574         rc = sysctl_wire_old_buffer(req, 0);
5575         if (rc != 0)
5576                 return (rc);
5577
5578         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5579         if (sb == NULL)
5580                 return (ENOMEM);
5581
5582         t4_tp_get_cpl_stats(sc, &stats);
5583
5584         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
5585             "channel 3\n");
5586         sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
5587                    stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
5588         sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
5589                    stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
5590
5591         rc = sbuf_finish(sb);
5592         sbuf_delete(sb);
5593
5594         return (rc);
5595 }
5596
5597 static int
5598 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
5599 {
5600         struct adapter *sc = arg1;
5601         struct sbuf *sb;
5602         int rc;
5603         struct tp_usm_stats stats;
5604
5605         rc = sysctl_wire_old_buffer(req, 0);
5606         if (rc != 0)
5607                 return(rc);
5608
5609         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5610         if (sb == NULL)
5611                 return (ENOMEM);
5612
5613         t4_get_usm_stats(sc, &stats);
5614
5615         sbuf_printf(sb, "Frames: %u\n", stats.frames);
5616         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
5617         sbuf_printf(sb, "Drops:  %u", stats.drops);
5618
5619         rc = sbuf_finish(sb);
5620         sbuf_delete(sb);
5621
5622         return (rc);
5623 }
5624
5625 const char *devlog_level_strings[] = {
5626         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
5627         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
5628         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
5629         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
5630         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
5631         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
5632 };
5633
5634 const char *devlog_facility_strings[] = {
5635         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
5636         [FW_DEVLOG_FACILITY_CF]         = "CF",
5637         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
5638         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
5639         [FW_DEVLOG_FACILITY_RES]        = "RES",
5640         [FW_DEVLOG_FACILITY_HW]         = "HW",
5641         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
5642         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
5643         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
5644         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
5645         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
5646         [FW_DEVLOG_FACILITY_VI]         = "VI",
5647         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
5648         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
5649         [FW_DEVLOG_FACILITY_TM]         = "TM",
5650         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
5651         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
5652         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
5653         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
5654         [FW_DEVLOG_FACILITY_RI]         = "RI",
5655         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
5656         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
5657         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
5658         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE"
5659 };
5660
5661 static int
5662 sysctl_devlog(SYSCTL_HANDLER_ARGS)
5663 {
5664         struct adapter *sc = arg1;
5665         struct devlog_params *dparams = &sc->params.devlog;
5666         struct fw_devlog_e *buf, *e;
5667         int i, j, rc, nentries, first = 0, m;
5668         struct sbuf *sb;
5669         uint64_t ftstamp = UINT64_MAX;
5670
5671         if (dparams->start == 0) {
5672                 dparams->memtype = FW_MEMTYPE_EDC0;
5673                 dparams->start = 0x84000;
5674                 dparams->size = 32768;
5675         }
5676
5677         nentries = dparams->size / sizeof(struct fw_devlog_e);
5678
5679         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
5680         if (buf == NULL)
5681                 return (ENOMEM);
5682
5683         m = fwmtype_to_hwmtype(dparams->memtype);
5684         rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
5685         if (rc != 0)
5686                 goto done;
5687
5688         for (i = 0; i < nentries; i++) {
5689                 e = &buf[i];
5690
5691                 if (e->timestamp == 0)
5692                         break;  /* end */
5693
5694                 e->timestamp = be64toh(e->timestamp);
5695                 e->seqno = be32toh(e->seqno);
5696                 for (j = 0; j < 8; j++)
5697                         e->params[j] = be32toh(e->params[j]);
5698
5699                 if (e->timestamp < ftstamp) {
5700                         ftstamp = e->timestamp;
5701                         first = i;
5702                 }
5703         }
5704
5705         if (buf[first].timestamp == 0)
5706                 goto done;      /* nothing in the log */
5707
5708         rc = sysctl_wire_old_buffer(req, 0);
5709         if (rc != 0)
5710                 goto done;
5711
5712         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5713         if (sb == NULL) {
5714                 rc = ENOMEM;
5715                 goto done;
5716         }
5717         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
5718             "Seq#", "Tstamp", "Level", "Facility", "Message");
5719
5720         i = first;
5721         do {
5722                 e = &buf[i];
5723                 if (e->timestamp == 0)
5724                         break;  /* end */
5725
5726                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
5727                     e->seqno, e->timestamp,
5728                     (e->level < nitems(devlog_level_strings) ?
5729                         devlog_level_strings[e->level] : "UNKNOWN"),
5730                     (e->facility < nitems(devlog_facility_strings) ?
5731                         devlog_facility_strings[e->facility] : "UNKNOWN"));
5732                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
5733                     e->params[2], e->params[3], e->params[4],
5734                     e->params[5], e->params[6], e->params[7]);
5735
5736                 if (++i == nentries)
5737                         i = 0;
5738         } while (i != first);
5739
5740         rc = sbuf_finish(sb);
5741         sbuf_delete(sb);
5742 done:
5743         free(buf, M_CXGBE);
5744         return (rc);
5745 }
5746
5747 static int
5748 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
5749 {
5750         struct adapter *sc = arg1;
5751         struct sbuf *sb;
5752         int rc;
5753         struct tp_fcoe_stats stats[4];
5754
5755         rc = sysctl_wire_old_buffer(req, 0);
5756         if (rc != 0)
5757                 return (rc);
5758
5759         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5760         if (sb == NULL)
5761                 return (ENOMEM);
5762
5763         t4_get_fcoe_stats(sc, 0, &stats[0]);
5764         t4_get_fcoe_stats(sc, 1, &stats[1]);
5765         t4_get_fcoe_stats(sc, 2, &stats[2]);
5766         t4_get_fcoe_stats(sc, 3, &stats[3]);
5767
5768         sbuf_printf(sb, "                   channel 0        channel 1        "
5769             "channel 2        channel 3\n");
5770         sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
5771             stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
5772             stats[3].octetsDDP);
5773         sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
5774             stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
5775         sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
5776             stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
5777             stats[3].framesDrop);
5778
5779         rc = sbuf_finish(sb);
5780         sbuf_delete(sb);
5781
5782         return (rc);
5783 }
5784
5785 static int
5786 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
5787 {
5788         struct adapter *sc = arg1;
5789         struct sbuf *sb;
5790         int rc, i;
5791         unsigned int map, kbps, ipg, mode;
5792         unsigned int pace_tab[NTX_SCHED];
5793
5794         rc = sysctl_wire_old_buffer(req, 0);
5795         if (rc != 0)
5796                 return (rc);
5797
5798         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5799         if (sb == NULL)
5800                 return (ENOMEM);
5801
5802         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
5803         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
5804         t4_read_pace_tbl(sc, pace_tab);
5805
5806         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
5807             "Class IPG (0.1 ns)   Flow IPG (us)");
5808
5809         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
5810                 t4_get_tx_sched(sc, i, &kbps, &ipg);
5811                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
5812                     (mode & (1 << i)) ? "flow" : "class", map & 3);
5813                 if (kbps)
5814                         sbuf_printf(sb, "%9u     ", kbps);
5815                 else
5816                         sbuf_printf(sb, " disabled     ");
5817
5818                 if (ipg)
5819                         sbuf_printf(sb, "%13u        ", ipg);
5820                 else
5821                         sbuf_printf(sb, "     disabled        ");
5822
5823                 if (pace_tab[i])
5824                         sbuf_printf(sb, "%10u", pace_tab[i]);
5825                 else
5826                         sbuf_printf(sb, "  disabled");
5827         }
5828
5829         rc = sbuf_finish(sb);
5830         sbuf_delete(sb);
5831
5832         return (rc);
5833 }
5834
5835 static int
5836 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
5837 {
5838         struct adapter *sc = arg1;
5839         struct sbuf *sb;
5840         int rc, i, j;
5841         uint64_t *p0, *p1;
5842         struct lb_port_stats s[2];
5843         static const char *stat_name[] = {
5844                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
5845                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
5846                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
5847                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
5848                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
5849                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
5850                 "BG2FramesTrunc:", "BG3FramesTrunc:"
5851         };
5852
5853         rc = sysctl_wire_old_buffer(req, 0);
5854         if (rc != 0)
5855                 return (rc);
5856
5857         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5858         if (sb == NULL)
5859                 return (ENOMEM);
5860
5861         memset(s, 0, sizeof(s));
5862
5863         for (i = 0; i < 4; i += 2) {
5864                 t4_get_lb_stats(sc, i, &s[0]);
5865                 t4_get_lb_stats(sc, i + 1, &s[1]);
5866
5867                 p0 = &s[0].octets;
5868                 p1 = &s[1].octets;
5869                 sbuf_printf(sb, "%s                       Loopback %u"
5870                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
5871
5872                 for (j = 0; j < nitems(stat_name); j++)
5873                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
5874                                    *p0++, *p1++);
5875         }
5876
5877         rc = sbuf_finish(sb);
5878         sbuf_delete(sb);
5879
5880         return (rc);
5881 }
5882
5883 static int
5884 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
5885 {
5886         int rc = 0;
5887         struct port_info *pi = arg1;
5888         struct sbuf *sb;
5889         static const char *linkdnreasons[] = {
5890                 "non-specific", "remote fault", "autoneg failed", "reserved3",
5891                 "PHY overheated", "unknown", "rx los", "reserved7"
5892         };
5893
5894         rc = sysctl_wire_old_buffer(req, 0);
5895         if (rc != 0)
5896                 return(rc);
5897         sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
5898         if (sb == NULL)
5899                 return (ENOMEM);
5900
5901         if (pi->linkdnrc < 0)
5902                 sbuf_printf(sb, "n/a");
5903         else if (pi->linkdnrc < nitems(linkdnreasons))
5904                 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
5905         else
5906                 sbuf_printf(sb, "%d", pi->linkdnrc);
5907
5908         rc = sbuf_finish(sb);
5909         sbuf_delete(sb);
5910
5911         return (rc);
5912 }
5913
5914 struct mem_desc {
5915         unsigned int base;
5916         unsigned int limit;
5917         unsigned int idx;
5918 };
5919
5920 static int
5921 mem_desc_cmp(const void *a, const void *b)
5922 {
5923         return ((const struct mem_desc *)a)->base -
5924                ((const struct mem_desc *)b)->base;
5925 }
5926
5927 static void
5928 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
5929     unsigned int to)
5930 {
5931         unsigned int size;
5932
5933         size = to - from + 1;
5934         if (size == 0)
5935                 return;
5936
5937         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
5938         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
5939 }
5940
5941 static int
5942 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
5943 {
5944         struct adapter *sc = arg1;
5945         struct sbuf *sb;
5946         int rc, i, n;
5947         uint32_t lo, hi, used, alloc;
5948         static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
5949         static const char *region[] = {
5950                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
5951                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
5952                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
5953                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
5954                 "RQUDP region:", "PBL region:", "TXPBL region:",
5955                 "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
5956                 "On-chip queues:"
5957         };
5958         struct mem_desc avail[4];
5959         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
5960         struct mem_desc *md = mem;
5961
5962         rc = sysctl_wire_old_buffer(req, 0);
5963         if (rc != 0)
5964                 return (rc);
5965
5966         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5967         if (sb == NULL)
5968                 return (ENOMEM);
5969
5970         for (i = 0; i < nitems(mem); i++) {
5971                 mem[i].limit = 0;
5972                 mem[i].idx = i;
5973         }
5974
5975         /* Find and sort the populated memory ranges */
5976         i = 0;
5977         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
5978         if (lo & F_EDRAM0_ENABLE) {
5979                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
5980                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
5981                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
5982                 avail[i].idx = 0;
5983                 i++;
5984         }
5985         if (lo & F_EDRAM1_ENABLE) {
5986                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
5987                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
5988                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
5989                 avail[i].idx = 1;
5990                 i++;
5991         }
5992         if (lo & F_EXT_MEM_ENABLE) {
5993                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
5994                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
5995                 avail[i].limit = avail[i].base +
5996                     (G_EXT_MEM_SIZE(hi) << 20);
5997                 avail[i].idx = is_t4(sc) ? 2 : 3;       /* Call it MC for T4 */
5998                 i++;
5999         }
6000         if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
6001                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
6002                 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
6003                 avail[i].limit = avail[i].base +
6004                     (G_EXT_MEM1_SIZE(hi) << 20);
6005                 avail[i].idx = 4;
6006                 i++;
6007         }
6008         if (!i)                                    /* no memory available */
6009                 return 0;
6010         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
6011
6012         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
6013         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
6014         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
6015         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
6016         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
6017         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
6018         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
6019         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
6020         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
6021
6022         /* the next few have explicit upper bounds */
6023         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
6024         md->limit = md->base - 1 +
6025                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
6026                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
6027         md++;
6028
6029         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
6030         md->limit = md->base - 1 +
6031                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
6032                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
6033         md++;
6034
6035         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6036                 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
6037                 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
6038                 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
6039         } else {
6040                 md->base = 0;
6041                 md->idx = nitems(region);  /* hide it */
6042         }
6043         md++;
6044
6045 #define ulp_region(reg) \
6046         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
6047         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
6048
6049         ulp_region(RX_ISCSI);
6050         ulp_region(RX_TDDP);
6051         ulp_region(TX_TPT);
6052         ulp_region(RX_STAG);
6053         ulp_region(RX_RQ);
6054         ulp_region(RX_RQUDP);
6055         ulp_region(RX_PBL);
6056         ulp_region(TX_PBL);
6057 #undef ulp_region
6058
6059         md->base = 0;
6060         md->idx = nitems(region);
6061         if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
6062                 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
6063                 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
6064                     A_SGE_DBVFIFO_SIZE))) << 2) - 1;
6065         }
6066         md++;
6067
6068         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
6069         md->limit = md->base + sc->tids.ntids - 1;
6070         md++;
6071         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
6072         md->limit = md->base + sc->tids.ntids - 1;
6073         md++;
6074
6075         md->base = sc->vres.ocq.start;
6076         if (sc->vres.ocq.size)
6077                 md->limit = md->base + sc->vres.ocq.size - 1;
6078         else
6079                 md->idx = nitems(region);  /* hide it */
6080         md++;
6081
6082         /* add any address-space holes, there can be up to 3 */
6083         for (n = 0; n < i - 1; n++)
6084                 if (avail[n].limit < avail[n + 1].base)
6085                         (md++)->base = avail[n].limit;
6086         if (avail[n].limit)
6087                 (md++)->base = avail[n].limit;
6088
6089         n = md - mem;
6090         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
6091
6092         for (lo = 0; lo < i; lo++)
6093                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
6094                                 avail[lo].limit - 1);
6095
6096         sbuf_printf(sb, "\n");
6097         for (i = 0; i < n; i++) {
6098                 if (mem[i].idx >= nitems(region))
6099                         continue;                        /* skip holes */
6100                 if (!mem[i].limit)
6101                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
6102                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
6103                                 mem[i].limit);
6104         }
6105
6106         sbuf_printf(sb, "\n");
6107         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
6108         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
6109         mem_region_show(sb, "uP RAM:", lo, hi);
6110
6111         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
6112         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
6113         mem_region_show(sb, "uP Extmem2:", lo, hi);
6114
6115         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
6116         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
6117                    G_PMRXMAXPAGE(lo),
6118                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
6119                    (lo & F_PMRXNUMCHN) ? 2 : 1);
6120
6121         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
6122         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
6123         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
6124                    G_PMTXMAXPAGE(lo),
6125                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
6126                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
6127         sbuf_printf(sb, "%u p-structs\n",
6128                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
6129
6130         for (i = 0; i < 4; i++) {
6131                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
6132                 if (is_t4(sc)) {
6133                         used = G_USED(lo);
6134                         alloc = G_ALLOC(lo);
6135                 } else {
6136                         used = G_T5_USED(lo);
6137                         alloc = G_T5_ALLOC(lo);
6138                 }
6139                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
6140                            i, used, alloc);
6141         }
6142         for (i = 0; i < 4; i++) {
6143                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
6144                 if (is_t4(sc)) {
6145                         used = G_USED(lo);
6146                         alloc = G_ALLOC(lo);
6147                 } else {
6148                         used = G_T5_USED(lo);
6149                         alloc = G_T5_ALLOC(lo);
6150                 }
6151                 sbuf_printf(sb,
6152                            "\nLoopback %d using %u pages out of %u allocated",
6153                            i, used, alloc);
6154         }
6155
6156         rc = sbuf_finish(sb);
6157         sbuf_delete(sb);
6158
6159         return (rc);
6160 }
6161
6162 static inline void
6163 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
6164 {
6165         *mask = x | y;
6166         y = htobe64(y);
6167         memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
6168 }
6169
6170 static int
6171 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
6172 {
6173         struct adapter *sc = arg1;
6174         struct sbuf *sb;
6175         int rc, i, n;
6176
6177         rc = sysctl_wire_old_buffer(req, 0);
6178         if (rc != 0)
6179                 return (rc);
6180
6181         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6182         if (sb == NULL)
6183                 return (ENOMEM);
6184
6185         sbuf_printf(sb,
6186             "Idx  Ethernet address     Mask     Vld Ports PF"
6187             "  VF              Replication             P0 P1 P2 P3  ML");
6188         n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
6189             NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
6190         for (i = 0; i < n; i++) {
6191                 uint64_t tcamx, tcamy, mask;
6192                 uint32_t cls_lo, cls_hi;
6193                 uint8_t addr[ETHER_ADDR_LEN];
6194
6195                 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
6196                 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
6197                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
6198                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
6199
6200                 if (tcamx & tcamy)
6201                         continue;
6202
6203                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
6204                 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
6205                            "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
6206                            addr[3], addr[4], addr[5], (uintmax_t)mask,
6207                            (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
6208                            G_PORTMAP(cls_hi), G_PF(cls_lo),
6209                            (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
6210
6211                 if (cls_lo & F_REPLICATE) {
6212                         struct fw_ldst_cmd ldst_cmd;
6213
6214                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
6215                         ldst_cmd.op_to_addrspace =
6216                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
6217                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
6218                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
6219                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
6220                         ldst_cmd.u.mps.fid_ctl =
6221                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
6222                                 V_FW_LDST_CMD_CTL(i));
6223
6224                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
6225                             "t4mps");
6226                         if (rc)
6227                                 break;
6228                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
6229                             sizeof(ldst_cmd), &ldst_cmd);
6230                         end_synchronized_op(sc, 0);
6231
6232                         if (rc != 0) {
6233                                 sbuf_printf(sb,
6234                                     " ------------ error %3u ------------", rc);
6235                                 rc = 0;
6236                         } else {
6237                                 sbuf_printf(sb, " %08x %08x %08x %08x",
6238                                     be32toh(ldst_cmd.u.mps.rplc127_96),
6239                                     be32toh(ldst_cmd.u.mps.rplc95_64),
6240                                     be32toh(ldst_cmd.u.mps.rplc63_32),
6241                                     be32toh(ldst_cmd.u.mps.rplc31_0));
6242                         }
6243                 } else
6244                         sbuf_printf(sb, "%36s", "");
6245
6246                 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
6247                     G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
6248                     G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
6249         }
6250
6251         if (rc)
6252                 (void) sbuf_finish(sb);
6253         else
6254                 rc = sbuf_finish(sb);
6255         sbuf_delete(sb);
6256
6257         return (rc);
6258 }
6259
6260 static int
6261 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
6262 {
6263         struct adapter *sc = arg1;
6264         struct sbuf *sb;
6265         int rc;
6266         uint16_t mtus[NMTUS];
6267
6268         rc = sysctl_wire_old_buffer(req, 0);
6269         if (rc != 0)
6270                 return (rc);
6271
6272         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6273         if (sb == NULL)
6274                 return (ENOMEM);
6275
6276         t4_read_mtu_tbl(sc, mtus, NULL);
6277
6278         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
6279             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
6280             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
6281             mtus[14], mtus[15]);
6282
6283         rc = sbuf_finish(sb);
6284         sbuf_delete(sb);
6285
6286         return (rc);
6287 }
6288
6289 static int
6290 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
6291 {
6292         struct adapter *sc = arg1;
6293         struct sbuf *sb;
6294         int rc, i;
6295         uint32_t cnt[PM_NSTATS];
6296         uint64_t cyc[PM_NSTATS];
6297         static const char *rx_stats[] = {
6298                 "Read:", "Write bypass:", "Write mem:", "Flush:"
6299         };
6300         static const char *tx_stats[] = {
6301                 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
6302         };
6303
6304         rc = sysctl_wire_old_buffer(req, 0);
6305         if (rc != 0)
6306                 return (rc);
6307
6308         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6309         if (sb == NULL)
6310                 return (ENOMEM);
6311
6312         t4_pmtx_get_stats(sc, cnt, cyc);
6313         sbuf_printf(sb, "                Tx pcmds             Tx bytes");
6314         for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
6315                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
6316                     cyc[i]);
6317
6318         t4_pmrx_get_stats(sc, cnt, cyc);
6319         sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
6320         for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
6321                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
6322                     cyc[i]);
6323
6324         rc = sbuf_finish(sb);
6325         sbuf_delete(sb);
6326
6327         return (rc);
6328 }
6329
6330 static int
6331 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
6332 {
6333         struct adapter *sc = arg1;
6334         struct sbuf *sb;
6335         int rc;
6336         struct tp_rdma_stats stats;
6337
6338         rc = sysctl_wire_old_buffer(req, 0);
6339         if (rc != 0)
6340                 return (rc);
6341
6342         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6343         if (sb == NULL)
6344                 return (ENOMEM);
6345
6346         t4_tp_get_rdma_stats(sc, &stats);
6347         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
6348         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
6349
6350         rc = sbuf_finish(sb);
6351         sbuf_delete(sb);
6352
6353         return (rc);
6354 }
6355
6356 static int
6357 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
6358 {
6359         struct adapter *sc = arg1;
6360         struct sbuf *sb;
6361         int rc;
6362         struct tp_tcp_stats v4, v6;
6363
6364         rc = sysctl_wire_old_buffer(req, 0);
6365         if (rc != 0)
6366                 return (rc);
6367
6368         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6369         if (sb == NULL)
6370                 return (ENOMEM);
6371
6372         t4_tp_get_tcp_stats(sc, &v4, &v6);
6373         sbuf_printf(sb,
6374             "                                IP                 IPv6\n");
6375         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
6376             v4.tcpOutRsts, v6.tcpOutRsts);
6377         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
6378             v4.tcpInSegs, v6.tcpInSegs);
6379         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
6380             v4.tcpOutSegs, v6.tcpOutSegs);
6381         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
6382             v4.tcpRetransSegs, v6.tcpRetransSegs);
6383
6384         rc = sbuf_finish(sb);
6385         sbuf_delete(sb);
6386
6387         return (rc);
6388 }
6389
6390 static int
6391 sysctl_tids(SYSCTL_HANDLER_ARGS)
6392 {
6393         struct adapter *sc = arg1;
6394         struct sbuf *sb;
6395         int rc;
6396         struct tid_info *t = &sc->tids;
6397
6398         rc = sysctl_wire_old_buffer(req, 0);
6399         if (rc != 0)
6400                 return (rc);
6401
6402         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6403         if (sb == NULL)
6404                 return (ENOMEM);
6405
6406         if (t->natids) {
6407                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
6408                     t->atids_in_use);
6409         }
6410
6411         if (t->ntids) {
6412                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6413                         uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
6414
6415                         if (b) {
6416                                 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
6417                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6418                                     t->ntids - 1);
6419                         } else {
6420                                 sbuf_printf(sb, "TID range: %u-%u",
6421                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6422                                     t->ntids - 1);
6423                         }
6424                 } else
6425                         sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
6426                 sbuf_printf(sb, ", in use: %u\n",
6427                     atomic_load_acq_int(&t->tids_in_use));
6428         }
6429
6430         if (t->nstids) {
6431                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
6432                     t->stid_base + t->nstids - 1, t->stids_in_use);
6433         }
6434
6435         if (t->nftids) {
6436                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
6437                     t->ftid_base + t->nftids - 1);
6438         }
6439
6440         if (t->netids) {
6441                 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
6442                     t->etid_base + t->netids - 1);
6443         }
6444
6445         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
6446             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
6447             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
6448
6449         rc = sbuf_finish(sb);
6450         sbuf_delete(sb);
6451
6452         return (rc);
6453 }
6454
6455 static int
6456 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
6457 {
6458         struct adapter *sc = arg1;
6459         struct sbuf *sb;
6460         int rc;
6461         struct tp_err_stats stats;
6462
6463         rc = sysctl_wire_old_buffer(req, 0);
6464         if (rc != 0)
6465                 return (rc);
6466
6467         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6468         if (sb == NULL)
6469                 return (ENOMEM);
6470
6471         t4_tp_get_err_stats(sc, &stats);
6472
6473         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
6474                       "channel 3\n");
6475         sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
6476             stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
6477             stats.macInErrs[3]);
6478         sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
6479             stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
6480             stats.hdrInErrs[3]);
6481         sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
6482             stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
6483             stats.tcpInErrs[3]);
6484         sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
6485             stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
6486             stats.tcp6InErrs[3]);
6487         sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
6488             stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
6489             stats.tnlCongDrops[3]);
6490         sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
6491             stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
6492             stats.tnlTxDrops[3]);
6493         sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
6494             stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
6495             stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
6496         sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
6497             stats.ofldChanDrops[0], stats.ofldChanDrops[1],
6498             stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
6499         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
6500             stats.ofldNoNeigh, stats.ofldCongDefer);
6501
6502         rc = sbuf_finish(sb);
6503         sbuf_delete(sb);
6504
6505         return (rc);
6506 }
6507
6508 struct field_desc {
6509         const char *name;
6510         u_int start;
6511         u_int width;
6512 };
6513
6514 static void
6515 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
6516 {
6517         char buf[32];
6518         int line_size = 0;
6519
6520         while (f->name) {
6521                 uint64_t mask = (1ULL << f->width) - 1;
6522                 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
6523                     ((uintmax_t)v >> f->start) & mask);
6524
6525                 if (line_size + len >= 79) {
6526                         line_size = 8;
6527                         sbuf_printf(sb, "\n        ");
6528                 }
6529                 sbuf_printf(sb, "%s ", buf);
6530                 line_size += len + 1;
6531                 f++;
6532         }
6533         sbuf_printf(sb, "\n");
6534 }
6535
6536 static struct field_desc tp_la0[] = {
6537         { "RcfOpCodeOut", 60, 4 },
6538         { "State", 56, 4 },
6539         { "WcfState", 52, 4 },
6540         { "RcfOpcSrcOut", 50, 2 },
6541         { "CRxError", 49, 1 },
6542         { "ERxError", 48, 1 },
6543         { "SanityFailed", 47, 1 },
6544         { "SpuriousMsg", 46, 1 },
6545         { "FlushInputMsg", 45, 1 },
6546         { "FlushInputCpl", 44, 1 },
6547         { "RssUpBit", 43, 1 },
6548         { "RssFilterHit", 42, 1 },
6549         { "Tid", 32, 10 },
6550         { "InitTcb", 31, 1 },
6551         { "LineNumber", 24, 7 },
6552         { "Emsg", 23, 1 },
6553         { "EdataOut", 22, 1 },
6554         { "Cmsg", 21, 1 },
6555         { "CdataOut", 20, 1 },
6556         { "EreadPdu", 19, 1 },
6557         { "CreadPdu", 18, 1 },
6558         { "TunnelPkt", 17, 1 },
6559         { "RcfPeerFin", 16, 1 },
6560         { "RcfReasonOut", 12, 4 },
6561         { "TxCchannel", 10, 2 },
6562         { "RcfTxChannel", 8, 2 },
6563         { "RxEchannel", 6, 2 },
6564         { "RcfRxChannel", 5, 1 },
6565         { "RcfDataOutSrdy", 4, 1 },
6566         { "RxDvld", 3, 1 },
6567         { "RxOoDvld", 2, 1 },
6568         { "RxCongestion", 1, 1 },
6569         { "TxCongestion", 0, 1 },
6570         { NULL }
6571 };
6572
6573 static struct field_desc tp_la1[] = {
6574         { "CplCmdIn", 56, 8 },
6575         { "CplCmdOut", 48, 8 },
6576         { "ESynOut", 47, 1 },
6577         { "EAckOut", 46, 1 },
6578         { "EFinOut", 45, 1 },
6579         { "ERstOut", 44, 1 },
6580         { "SynIn", 43, 1 },
6581         { "AckIn", 42, 1 },
6582         { "FinIn", 41, 1 },
6583         { "RstIn", 40, 1 },
6584         { "DataIn", 39, 1 },
6585         { "DataInVld", 38, 1 },
6586         { "PadIn", 37, 1 },
6587         { "RxBufEmpty", 36, 1 },
6588         { "RxDdp", 35, 1 },
6589         { "RxFbCongestion", 34, 1 },
6590         { "TxFbCongestion", 33, 1 },
6591         { "TxPktSumSrdy", 32, 1 },
6592         { "RcfUlpType", 28, 4 },
6593         { "Eread", 27, 1 },
6594         { "Ebypass", 26, 1 },
6595         { "Esave", 25, 1 },
6596         { "Static0", 24, 1 },
6597         { "Cread", 23, 1 },
6598         { "Cbypass", 22, 1 },
6599         { "Csave", 21, 1 },
6600         { "CPktOut", 20, 1 },
6601         { "RxPagePoolFull", 18, 2 },
6602         { "RxLpbkPkt", 17, 1 },
6603         { "TxLpbkPkt", 16, 1 },
6604         { "RxVfValid", 15, 1 },
6605         { "SynLearned", 14, 1 },
6606         { "SetDelEntry", 13, 1 },
6607         { "SetInvEntry", 12, 1 },
6608         { "CpcmdDvld", 11, 1 },
6609         { "CpcmdSave", 10, 1 },
6610         { "RxPstructsFull", 8, 2 },
6611         { "EpcmdDvld", 7, 1 },
6612         { "EpcmdFlush", 6, 1 },
6613         { "EpcmdTrimPrefix", 5, 1 },
6614         { "EpcmdTrimPostfix", 4, 1 },
6615         { "ERssIp4Pkt", 3, 1 },
6616         { "ERssIp6Pkt", 2, 1 },
6617         { "ERssTcpUdpPkt", 1, 1 },
6618         { "ERssFceFipPkt", 0, 1 },
6619         { NULL }
6620 };
6621
6622 static struct field_desc tp_la2[] = {
6623         { "CplCmdIn", 56, 8 },
6624         { "MpsVfVld", 55, 1 },
6625         { "MpsPf", 52, 3 },
6626         { "MpsVf", 44, 8 },
6627         { "SynIn", 43, 1 },
6628         { "AckIn", 42, 1 },
6629         { "FinIn", 41, 1 },
6630         { "RstIn", 40, 1 },
6631         { "DataIn", 39, 1 },
6632         { "DataInVld", 38, 1 },
6633         { "PadIn", 37, 1 },
6634         { "RxBufEmpty", 36, 1 },
6635         { "RxDdp", 35, 1 },
6636         { "RxFbCongestion", 34, 1 },
6637         { "TxFbCongestion", 33, 1 },
6638         { "TxPktSumSrdy", 32, 1 },
6639         { "RcfUlpType", 28, 4 },
6640         { "Eread", 27, 1 },
6641         { "Ebypass", 26, 1 },
6642         { "Esave", 25, 1 },
6643         { "Static0", 24, 1 },
6644         { "Cread", 23, 1 },
6645         { "Cbypass", 22, 1 },
6646         { "Csave", 21, 1 },
6647         { "CPktOut", 20, 1 },
6648         { "RxPagePoolFull", 18, 2 },
6649         { "RxLpbkPkt", 17, 1 },
6650         { "TxLpbkPkt", 16, 1 },
6651         { "RxVfValid", 15, 1 },
6652         { "SynLearned", 14, 1 },
6653         { "SetDelEntry", 13, 1 },
6654         { "SetInvEntry", 12, 1 },
6655         { "CpcmdDvld", 11, 1 },
6656         { "CpcmdSave", 10, 1 },
6657         { "RxPstructsFull", 8, 2 },
6658         { "EpcmdDvld", 7, 1 },
6659         { "EpcmdFlush", 6, 1 },
6660         { "EpcmdTrimPrefix", 5, 1 },
6661         { "EpcmdTrimPostfix", 4, 1 },
6662         { "ERssIp4Pkt", 3, 1 },
6663         { "ERssIp6Pkt", 2, 1 },
6664         { "ERssTcpUdpPkt", 1, 1 },
6665         { "ERssFceFipPkt", 0, 1 },
6666         { NULL }
6667 };
6668
6669 static void
6670 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
6671 {
6672
6673         field_desc_show(sb, *p, tp_la0);
6674 }
6675
6676 static void
6677 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
6678 {
6679
6680         if (idx)
6681                 sbuf_printf(sb, "\n");
6682         field_desc_show(sb, p[0], tp_la0);
6683         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6684                 field_desc_show(sb, p[1], tp_la0);
6685 }
6686
6687 static void
6688 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
6689 {
6690
6691         if (idx)
6692                 sbuf_printf(sb, "\n");
6693         field_desc_show(sb, p[0], tp_la0);
6694         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6695                 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
6696 }
6697
6698 static int
6699 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
6700 {
6701         struct adapter *sc = arg1;
6702         struct sbuf *sb;
6703         uint64_t *buf, *p;
6704         int rc;
6705         u_int i, inc;
6706         void (*show_func)(struct sbuf *, uint64_t *, int);
6707
6708         rc = sysctl_wire_old_buffer(req, 0);
6709         if (rc != 0)
6710                 return (rc);
6711
6712         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6713         if (sb == NULL)
6714                 return (ENOMEM);
6715
6716         buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
6717
6718         t4_tp_read_la(sc, buf, NULL);
6719         p = buf;
6720
6721         switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
6722         case 2:
6723                 inc = 2;
6724                 show_func = tp_la_show2;
6725                 break;
6726         case 3:
6727                 inc = 2;
6728                 show_func = tp_la_show3;
6729                 break;
6730         default:
6731                 inc = 1;
6732                 show_func = tp_la_show;
6733         }
6734
6735         for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
6736                 (*show_func)(sb, p, i);
6737
6738         rc = sbuf_finish(sb);
6739         sbuf_delete(sb);
6740         free(buf, M_CXGBE);
6741         return (rc);
6742 }
6743
6744 static int
6745 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
6746 {
6747         struct adapter *sc = arg1;
6748         struct sbuf *sb;
6749         int rc;
6750         u64 nrate[NCHAN], orate[NCHAN];
6751
6752         rc = sysctl_wire_old_buffer(req, 0);
6753         if (rc != 0)
6754                 return (rc);
6755
6756         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6757         if (sb == NULL)
6758                 return (ENOMEM);
6759
6760         t4_get_chan_txrate(sc, nrate, orate);
6761         sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
6762                  "channel 3\n");
6763         sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
6764             nrate[0], nrate[1], nrate[2], nrate[3]);
6765         sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
6766             orate[0], orate[1], orate[2], orate[3]);
6767
6768         rc = sbuf_finish(sb);
6769         sbuf_delete(sb);
6770
6771         return (rc);
6772 }
6773
6774 static int
6775 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
6776 {
6777         struct adapter *sc = arg1;
6778         struct sbuf *sb;
6779         uint32_t *buf, *p;
6780         int rc, i;
6781
6782         rc = sysctl_wire_old_buffer(req, 0);
6783         if (rc != 0)
6784                 return (rc);
6785
6786         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6787         if (sb == NULL)
6788                 return (ENOMEM);
6789
6790         buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
6791             M_ZERO | M_WAITOK);
6792
6793         t4_ulprx_read_la(sc, buf);
6794         p = buf;
6795
6796         sbuf_printf(sb, "      Pcmd        Type   Message"
6797             "                Data");
6798         for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
6799                 sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
6800                     p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
6801         }
6802
6803         rc = sbuf_finish(sb);
6804         sbuf_delete(sb);
6805         free(buf, M_CXGBE);
6806         return (rc);
6807 }
6808
6809 static int
6810 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
6811 {
6812         struct adapter *sc = arg1;
6813         struct sbuf *sb;
6814         int rc, v;
6815
6816         rc = sysctl_wire_old_buffer(req, 0);
6817         if (rc != 0)
6818                 return (rc);
6819
6820         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6821         if (sb == NULL)
6822                 return (ENOMEM);
6823
6824         v = t4_read_reg(sc, A_SGE_STAT_CFG);
6825         if (G_STATSOURCE_T5(v) == 7) {
6826                 if (G_STATMODE(v) == 0) {
6827                         sbuf_printf(sb, "total %d, incomplete %d",
6828                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6829                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6830                 } else if (G_STATMODE(v) == 1) {
6831                         sbuf_printf(sb, "total %d, data overflow %d",
6832                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6833                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6834                 }
6835         }
6836         rc = sbuf_finish(sb);
6837         sbuf_delete(sb);
6838
6839         return (rc);
6840 }
6841 #endif
6842
6843 static inline void
6844 txq_start(struct ifnet *ifp, struct sge_txq *txq)
6845 {
6846         struct buf_ring *br;
6847         struct mbuf *m;
6848
6849         TXQ_LOCK_ASSERT_OWNED(txq);
6850
6851         br = txq->br;
6852         m = txq->m ? txq->m : drbr_dequeue(ifp, br);
6853         if (m)
6854                 t4_eth_tx(ifp, txq, m);
6855 }
6856
6857 void
6858 t4_tx_callout(void *arg)
6859 {
6860         struct sge_eq *eq = arg;
6861         struct adapter *sc;
6862
6863         if (EQ_TRYLOCK(eq) == 0)
6864                 goto reschedule;
6865
6866         if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) {
6867                 EQ_UNLOCK(eq);
6868 reschedule:
6869                 if (__predict_true(!(eq->flags && EQ_DOOMED)))
6870                         callout_schedule(&eq->tx_callout, 1);
6871                 return;
6872         }
6873
6874         EQ_LOCK_ASSERT_OWNED(eq);
6875
6876         if (__predict_true((eq->flags & EQ_DOOMED) == 0)) {
6877
6878                 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
6879                         struct sge_txq *txq = arg;
6880                         struct port_info *pi = txq->ifp->if_softc;
6881
6882                         sc = pi->adapter;
6883                 } else {
6884                         struct sge_wrq *wrq = arg;
6885
6886                         sc = wrq->adapter;
6887                 }
6888
6889                 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task);
6890         }
6891
6892         EQ_UNLOCK(eq);
6893 }
6894
6895 void
6896 t4_tx_task(void *arg, int count)
6897 {
6898         struct sge_eq *eq = arg;
6899
6900         EQ_LOCK(eq);
6901         if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
6902                 struct sge_txq *txq = arg;
6903                 txq_start(txq->ifp, txq);
6904         } else {
6905                 struct sge_wrq *wrq = arg;
6906                 t4_wrq_tx_locked(wrq->adapter, wrq, NULL);
6907         }
6908         EQ_UNLOCK(eq);
6909 }
6910
6911 static uint32_t
6912 fconf_to_mode(uint32_t fconf)
6913 {
6914         uint32_t mode;
6915
6916         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
6917             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
6918
6919         if (fconf & F_FRAGMENTATION)
6920                 mode |= T4_FILTER_IP_FRAGMENT;
6921
6922         if (fconf & F_MPSHITTYPE)
6923                 mode |= T4_FILTER_MPS_HIT_TYPE;
6924
6925         if (fconf & F_MACMATCH)
6926                 mode |= T4_FILTER_MAC_IDX;
6927
6928         if (fconf & F_ETHERTYPE)
6929                 mode |= T4_FILTER_ETH_TYPE;
6930
6931         if (fconf & F_PROTOCOL)
6932                 mode |= T4_FILTER_IP_PROTO;
6933
6934         if (fconf & F_TOS)
6935                 mode |= T4_FILTER_IP_TOS;
6936
6937         if (fconf & F_VLAN)
6938                 mode |= T4_FILTER_VLAN;
6939
6940         if (fconf & F_VNIC_ID)
6941                 mode |= T4_FILTER_VNIC;
6942
6943         if (fconf & F_PORT)
6944                 mode |= T4_FILTER_PORT;
6945
6946         if (fconf & F_FCOE)
6947                 mode |= T4_FILTER_FCoE;
6948
6949         return (mode);
6950 }
6951
6952 static uint32_t
6953 mode_to_fconf(uint32_t mode)
6954 {
6955         uint32_t fconf = 0;
6956
6957         if (mode & T4_FILTER_IP_FRAGMENT)
6958                 fconf |= F_FRAGMENTATION;
6959
6960         if (mode & T4_FILTER_MPS_HIT_TYPE)
6961                 fconf |= F_MPSHITTYPE;
6962
6963         if (mode & T4_FILTER_MAC_IDX)
6964                 fconf |= F_MACMATCH;
6965
6966         if (mode & T4_FILTER_ETH_TYPE)
6967                 fconf |= F_ETHERTYPE;
6968
6969         if (mode & T4_FILTER_IP_PROTO)
6970                 fconf |= F_PROTOCOL;
6971
6972         if (mode & T4_FILTER_IP_TOS)
6973                 fconf |= F_TOS;
6974
6975         if (mode & T4_FILTER_VLAN)
6976                 fconf |= F_VLAN;
6977
6978         if (mode & T4_FILTER_VNIC)
6979                 fconf |= F_VNIC_ID;
6980
6981         if (mode & T4_FILTER_PORT)
6982                 fconf |= F_PORT;
6983
6984         if (mode & T4_FILTER_FCoE)
6985                 fconf |= F_FCOE;
6986
6987         return (fconf);
6988 }
6989
6990 static uint32_t
6991 fspec_to_fconf(struct t4_filter_specification *fs)
6992 {
6993         uint32_t fconf = 0;
6994
6995         if (fs->val.frag || fs->mask.frag)
6996                 fconf |= F_FRAGMENTATION;
6997
6998         if (fs->val.matchtype || fs->mask.matchtype)
6999                 fconf |= F_MPSHITTYPE;
7000
7001         if (fs->val.macidx || fs->mask.macidx)
7002                 fconf |= F_MACMATCH;
7003
7004         if (fs->val.ethtype || fs->mask.ethtype)
7005                 fconf |= F_ETHERTYPE;
7006
7007         if (fs->val.proto || fs->mask.proto)
7008                 fconf |= F_PROTOCOL;
7009
7010         if (fs->val.tos || fs->mask.tos)
7011                 fconf |= F_TOS;
7012
7013         if (fs->val.vlan_vld || fs->mask.vlan_vld)
7014                 fconf |= F_VLAN;
7015
7016         if (fs->val.vnic_vld || fs->mask.vnic_vld)
7017                 fconf |= F_VNIC_ID;
7018
7019         if (fs->val.iport || fs->mask.iport)
7020                 fconf |= F_PORT;
7021
7022         if (fs->val.fcoe || fs->mask.fcoe)
7023                 fconf |= F_FCOE;
7024
7025         return (fconf);
7026 }
7027
7028 static int
7029 get_filter_mode(struct adapter *sc, uint32_t *mode)
7030 {
7031         int rc;
7032         uint32_t fconf;
7033
7034         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7035             "t4getfm");
7036         if (rc)
7037                 return (rc);
7038
7039         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
7040             A_TP_VLAN_PRI_MAP);
7041
7042         if (sc->params.tp.vlan_pri_map != fconf) {
7043                 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
7044                     device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
7045                     fconf);
7046                 sc->params.tp.vlan_pri_map = fconf;
7047         }
7048
7049         *mode = fconf_to_mode(sc->params.tp.vlan_pri_map);
7050
7051         end_synchronized_op(sc, LOCK_HELD);
7052         return (0);
7053 }
7054
7055 static int
7056 set_filter_mode(struct adapter *sc, uint32_t mode)
7057 {
7058         uint32_t fconf;
7059         int rc;
7060
7061         fconf = mode_to_fconf(mode);
7062
7063         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7064             "t4setfm");
7065         if (rc)
7066                 return (rc);
7067
7068         if (sc->tids.ftids_in_use > 0) {
7069                 rc = EBUSY;
7070                 goto done;
7071         }
7072
7073 #ifdef TCP_OFFLOAD
7074         if (sc->offload_map) {
7075                 rc = EBUSY;
7076                 goto done;
7077         }
7078 #endif
7079
7080 #ifdef notyet
7081         rc = -t4_set_filter_mode(sc, fconf);
7082         if (rc == 0)
7083                 sc->filter_mode = fconf;
7084 #else
7085         rc = ENOTSUP;
7086 #endif
7087
7088 done:
7089         end_synchronized_op(sc, LOCK_HELD);
7090         return (rc);
7091 }
7092
7093 static inline uint64_t
7094 get_filter_hits(struct adapter *sc, uint32_t fid)
7095 {
7096         uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
7097         uint64_t hits;
7098
7099         memwin_info(sc, 0, &mw_base, NULL);
7100         off = position_memwin(sc, 0,
7101             tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
7102         if (is_t4(sc)) {
7103                 hits = t4_read_reg64(sc, mw_base + off + 16);
7104                 hits = be64toh(hits);
7105         } else {
7106                 hits = t4_read_reg(sc, mw_base + off + 24);
7107                 hits = be32toh(hits);
7108         }
7109
7110         return (hits);
7111 }
7112
7113 static int
7114 get_filter(struct adapter *sc, struct t4_filter *t)
7115 {
7116         int i, rc, nfilters = sc->tids.nftids;
7117         struct filter_entry *f;
7118
7119         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7120             "t4getf");
7121         if (rc)
7122                 return (rc);
7123
7124         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
7125             t->idx >= nfilters) {
7126                 t->idx = 0xffffffff;
7127                 goto done;
7128         }
7129
7130         f = &sc->tids.ftid_tab[t->idx];
7131         for (i = t->idx; i < nfilters; i++, f++) {
7132                 if (f->valid) {
7133                         t->idx = i;
7134                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
7135                         t->smtidx = f->smtidx;
7136                         if (f->fs.hitcnts)
7137                                 t->hits = get_filter_hits(sc, t->idx);
7138                         else
7139                                 t->hits = UINT64_MAX;
7140                         t->fs = f->fs;
7141
7142                         goto done;
7143                 }
7144         }
7145
7146         t->idx = 0xffffffff;
7147 done:
7148         end_synchronized_op(sc, LOCK_HELD);
7149         return (0);
7150 }
7151
7152 static int
7153 set_filter(struct adapter *sc, struct t4_filter *t)
7154 {
7155         unsigned int nfilters, nports;
7156         struct filter_entry *f;
7157         int i, rc;
7158
7159         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
7160         if (rc)
7161                 return (rc);
7162
7163         nfilters = sc->tids.nftids;
7164         nports = sc->params.nports;
7165
7166         if (nfilters == 0) {
7167                 rc = ENOTSUP;
7168                 goto done;
7169         }
7170
7171         if (!(sc->flags & FULL_INIT_DONE)) {
7172                 rc = EAGAIN;
7173                 goto done;
7174         }
7175
7176         if (t->idx >= nfilters) {
7177                 rc = EINVAL;
7178                 goto done;
7179         }
7180
7181         /* Validate against the global filter mode */
7182         if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
7183             sc->params.tp.vlan_pri_map) {
7184                 rc = E2BIG;
7185                 goto done;
7186         }
7187
7188         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
7189                 rc = EINVAL;
7190                 goto done;
7191         }
7192
7193         if (t->fs.val.iport >= nports) {
7194                 rc = EINVAL;
7195                 goto done;
7196         }
7197
7198         /* Can't specify an iq if not steering to it */
7199         if (!t->fs.dirsteer && t->fs.iq) {
7200                 rc = EINVAL;
7201                 goto done;
7202         }
7203
7204         /* IPv6 filter idx must be 4 aligned */
7205         if (t->fs.type == 1 &&
7206             ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
7207                 rc = EINVAL;
7208                 goto done;
7209         }
7210
7211         if (sc->tids.ftid_tab == NULL) {
7212                 KASSERT(sc->tids.ftids_in_use == 0,
7213                     ("%s: no memory allocated but filters_in_use > 0",
7214                     __func__));
7215
7216                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
7217                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
7218                 if (sc->tids.ftid_tab == NULL) {
7219                         rc = ENOMEM;
7220                         goto done;
7221                 }
7222                 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
7223         }
7224
7225         for (i = 0; i < 4; i++) {
7226                 f = &sc->tids.ftid_tab[t->idx + i];
7227
7228                 if (f->pending || f->valid) {
7229                         rc = EBUSY;
7230                         goto done;
7231                 }
7232                 if (f->locked) {
7233                         rc = EPERM;
7234                         goto done;
7235                 }
7236
7237                 if (t->fs.type == 0)
7238                         break;
7239         }
7240
7241         f = &sc->tids.ftid_tab[t->idx];
7242         f->fs = t->fs;
7243
7244         rc = set_filter_wr(sc, t->idx);
7245 done:
7246         end_synchronized_op(sc, 0);
7247
7248         if (rc == 0) {
7249                 mtx_lock(&sc->tids.ftid_lock);
7250                 for (;;) {
7251                         if (f->pending == 0) {
7252                                 rc = f->valid ? 0 : EIO;
7253                                 break;
7254                         }
7255
7256                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7257                             PCATCH, "t4setfw", 0)) {
7258                                 rc = EINPROGRESS;
7259                                 break;
7260                         }
7261                 }
7262                 mtx_unlock(&sc->tids.ftid_lock);
7263         }
7264         return (rc);
7265 }
7266
7267 static int
7268 del_filter(struct adapter *sc, struct t4_filter *t)
7269 {
7270         unsigned int nfilters;
7271         struct filter_entry *f;
7272         int rc;
7273
7274         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
7275         if (rc)
7276                 return (rc);
7277
7278         nfilters = sc->tids.nftids;
7279
7280         if (nfilters == 0) {
7281                 rc = ENOTSUP;
7282                 goto done;
7283         }
7284
7285         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
7286             t->idx >= nfilters) {
7287                 rc = EINVAL;
7288                 goto done;
7289         }
7290
7291         if (!(sc->flags & FULL_INIT_DONE)) {
7292                 rc = EAGAIN;
7293                 goto done;
7294         }
7295
7296         f = &sc->tids.ftid_tab[t->idx];
7297
7298         if (f->pending) {
7299                 rc = EBUSY;
7300                 goto done;
7301         }
7302         if (f->locked) {
7303                 rc = EPERM;
7304                 goto done;
7305         }
7306
7307         if (f->valid) {
7308                 t->fs = f->fs;  /* extra info for the caller */
7309                 rc = del_filter_wr(sc, t->idx);
7310         }
7311
7312 done:
7313         end_synchronized_op(sc, 0);
7314
7315         if (rc == 0) {
7316                 mtx_lock(&sc->tids.ftid_lock);
7317                 for (;;) {
7318                         if (f->pending == 0) {
7319                                 rc = f->valid ? EIO : 0;
7320                                 break;
7321                         }
7322
7323                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7324                             PCATCH, "t4delfw", 0)) {
7325                                 rc = EINPROGRESS;
7326                                 break;
7327                         }
7328                 }
7329                 mtx_unlock(&sc->tids.ftid_lock);
7330         }
7331
7332         return (rc);
7333 }
7334
7335 static void
7336 clear_filter(struct filter_entry *f)
7337 {
7338         if (f->l2t)
7339                 t4_l2t_release(f->l2t);
7340
7341         bzero(f, sizeof (*f));
7342 }
7343
7344 static int
7345 set_filter_wr(struct adapter *sc, int fidx)
7346 {
7347         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7348         struct wrqe *wr;
7349         struct fw_filter_wr *fwr;
7350         unsigned int ftid;
7351
7352         ASSERT_SYNCHRONIZED_OP(sc);
7353
7354         if (f->fs.newdmac || f->fs.newvlan) {
7355                 /* This filter needs an L2T entry; allocate one. */
7356                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
7357                 if (f->l2t == NULL)
7358                         return (EAGAIN);
7359                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
7360                     f->fs.dmac)) {
7361                         t4_l2t_release(f->l2t);
7362                         f->l2t = NULL;
7363                         return (ENOMEM);
7364                 }
7365         }
7366
7367         ftid = sc->tids.ftid_base + fidx;
7368
7369         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7370         if (wr == NULL)
7371                 return (ENOMEM);
7372
7373         fwr = wrtod(wr);
7374         bzero(fwr, sizeof (*fwr));
7375
7376         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
7377         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
7378         fwr->tid_to_iq =
7379             htobe32(V_FW_FILTER_WR_TID(ftid) |
7380                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
7381                 V_FW_FILTER_WR_NOREPLY(0) |
7382                 V_FW_FILTER_WR_IQ(f->fs.iq));
7383         fwr->del_filter_to_l2tix =
7384             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
7385                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
7386                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
7387                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
7388                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
7389                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
7390                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
7391                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
7392                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
7393                     f->fs.newvlan == VLAN_REWRITE) |
7394                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
7395                     f->fs.newvlan == VLAN_REWRITE) |
7396                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
7397                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
7398                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
7399                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
7400         fwr->ethtype = htobe16(f->fs.val.ethtype);
7401         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
7402         fwr->frag_to_ovlan_vldm =
7403             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
7404                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
7405                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
7406                 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
7407                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
7408                 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
7409         fwr->smac_sel = 0;
7410         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
7411             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
7412         fwr->maci_to_matchtypem =
7413             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
7414                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
7415                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
7416                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
7417                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
7418                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
7419                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
7420                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
7421         fwr->ptcl = f->fs.val.proto;
7422         fwr->ptclm = f->fs.mask.proto;
7423         fwr->ttyp = f->fs.val.tos;
7424         fwr->ttypm = f->fs.mask.tos;
7425         fwr->ivlan = htobe16(f->fs.val.vlan);
7426         fwr->ivlanm = htobe16(f->fs.mask.vlan);
7427         fwr->ovlan = htobe16(f->fs.val.vnic);
7428         fwr->ovlanm = htobe16(f->fs.mask.vnic);
7429         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
7430         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
7431         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
7432         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
7433         fwr->lp = htobe16(f->fs.val.dport);
7434         fwr->lpm = htobe16(f->fs.mask.dport);
7435         fwr->fp = htobe16(f->fs.val.sport);
7436         fwr->fpm = htobe16(f->fs.mask.sport);
7437         if (f->fs.newsmac)
7438                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
7439
7440         f->pending = 1;
7441         sc->tids.ftids_in_use++;
7442
7443         t4_wrq_tx(sc, wr);
7444         return (0);
7445 }
7446
7447 static int
7448 del_filter_wr(struct adapter *sc, int fidx)
7449 {
7450         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7451         struct wrqe *wr;
7452         struct fw_filter_wr *fwr;
7453         unsigned int ftid;
7454
7455         ftid = sc->tids.ftid_base + fidx;
7456
7457         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7458         if (wr == NULL)
7459                 return (ENOMEM);
7460         fwr = wrtod(wr);
7461         bzero(fwr, sizeof (*fwr));
7462
7463         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
7464
7465         f->pending = 1;
7466         t4_wrq_tx(sc, wr);
7467         return (0);
7468 }
7469
7470 int
7471 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
7472 {
7473         struct adapter *sc = iq->adapter;
7474         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
7475         unsigned int idx = GET_TID(rpl);
7476         unsigned int rc;
7477         struct filter_entry *f;
7478
7479         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
7480             rss->opcode));
7481
7482         if (is_ftid(sc, idx)) {
7483
7484                 idx -= sc->tids.ftid_base;
7485                 f = &sc->tids.ftid_tab[idx];
7486                 rc = G_COOKIE(rpl->cookie);
7487
7488                 mtx_lock(&sc->tids.ftid_lock);
7489                 if (rc == FW_FILTER_WR_FLT_ADDED) {
7490                         KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
7491                             __func__, idx));
7492                         f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
7493                         f->pending = 0;  /* asynchronous setup completed */
7494                         f->valid = 1;
7495                 } else {
7496                         if (rc != FW_FILTER_WR_FLT_DELETED) {
7497                                 /* Add or delete failed, display an error */
7498                                 log(LOG_ERR,
7499                                     "filter %u setup failed with error %u\n",
7500                                     idx, rc);
7501                         }
7502
7503                         clear_filter(f);
7504                         sc->tids.ftids_in_use--;
7505                 }
7506                 wakeup(&sc->tids.ftid_tab);
7507                 mtx_unlock(&sc->tids.ftid_lock);
7508         }
7509
7510         return (0);
7511 }
7512
7513 static int
7514 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
7515 {
7516         int rc;
7517
7518         if (cntxt->cid > M_CTXTQID)
7519                 return (EINVAL);
7520
7521         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
7522             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
7523                 return (EINVAL);
7524
7525         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
7526         if (rc)
7527                 return (rc);
7528
7529         if (sc->flags & FW_OK) {
7530                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
7531                     &cntxt->data[0]);
7532                 if (rc == 0)
7533                         goto done;
7534         }
7535
7536         /*
7537          * Read via firmware failed or wasn't even attempted.  Read directly via
7538          * the backdoor.
7539          */
7540         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
7541 done:
7542         end_synchronized_op(sc, 0);
7543         return (rc);
7544 }
7545
7546 static int
7547 load_fw(struct adapter *sc, struct t4_data *fw)
7548 {
7549         int rc;
7550         uint8_t *fw_data;
7551
7552         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
7553         if (rc)
7554                 return (rc);
7555
7556         if (sc->flags & FULL_INIT_DONE) {
7557                 rc = EBUSY;
7558                 goto done;
7559         }
7560
7561         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
7562         if (fw_data == NULL) {
7563                 rc = ENOMEM;
7564                 goto done;
7565         }
7566
7567         rc = copyin(fw->data, fw_data, fw->len);
7568         if (rc == 0)
7569                 rc = -t4_load_fw(sc, fw_data, fw->len);
7570
7571         free(fw_data, M_CXGBE);
7572 done:
7573         end_synchronized_op(sc, 0);
7574         return (rc);
7575 }
7576
7577 static int
7578 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
7579 {
7580         uint32_t addr, off, remaining, i, n;
7581         uint32_t *buf, *b;
7582         uint32_t mw_base, mw_aperture;
7583         int rc;
7584         uint8_t *dst;
7585
7586         rc = validate_mem_range(sc, mr->addr, mr->len);
7587         if (rc != 0)
7588                 return (rc);
7589
7590         memwin_info(sc, win, &mw_base, &mw_aperture);
7591         buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
7592         addr = mr->addr;
7593         remaining = mr->len;
7594         dst = (void *)mr->data;
7595
7596         while (remaining) {
7597                 off = position_memwin(sc, win, addr);
7598
7599                 /* number of bytes that we'll copy in the inner loop */
7600                 n = min(remaining, mw_aperture - off);
7601                 for (i = 0; i < n; i += 4)
7602                         *b++ = t4_read_reg(sc, mw_base + off + i);
7603
7604                 rc = copyout(buf, dst, n);
7605                 if (rc != 0)
7606                         break;
7607
7608                 b = buf;
7609                 dst += n;
7610                 remaining -= n;
7611                 addr += n;
7612         }
7613
7614         free(buf, M_CXGBE);
7615         return (rc);
7616 }
7617
7618 static int
7619 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
7620 {
7621         int rc;
7622
7623         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
7624                 return (EINVAL);
7625
7626         if (i2cd->len > sizeof(i2cd->data))
7627                 return (EFBIG);
7628
7629         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
7630         if (rc)
7631                 return (rc);
7632         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
7633             i2cd->offset, i2cd->len, &i2cd->data[0]);
7634         end_synchronized_op(sc, 0);
7635
7636         return (rc);
7637 }
7638
7639 static int
7640 in_range(int val, int lo, int hi)
7641 {
7642
7643         return (val < 0 || (val <= hi && val >= lo));
7644 }
7645
7646 static int
7647 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
7648 {
7649         int fw_subcmd, fw_type, rc;
7650
7651         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
7652         if (rc)
7653                 return (rc);
7654
7655         if (!(sc->flags & FULL_INIT_DONE)) {
7656                 rc = EAGAIN;
7657                 goto done;
7658         }
7659
7660         /*
7661          * Translate the cxgbetool parameters into T4 firmware parameters.  (The
7662          * sub-command and type are in common locations.)
7663          */
7664         if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
7665                 fw_subcmd = FW_SCHED_SC_CONFIG;
7666         else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
7667                 fw_subcmd = FW_SCHED_SC_PARAMS;
7668         else {
7669                 rc = EINVAL;
7670                 goto done;
7671         }
7672         if (p->type == SCHED_CLASS_TYPE_PACKET)
7673                 fw_type = FW_SCHED_TYPE_PKTSCHED;
7674         else {
7675                 rc = EINVAL;
7676                 goto done;
7677         }
7678
7679         if (fw_subcmd == FW_SCHED_SC_CONFIG) {
7680                 /* Vet our parameters ..*/
7681                 if (p->u.config.minmax < 0) {
7682                         rc = EINVAL;
7683                         goto done;
7684                 }
7685
7686                 /* And pass the request to the firmware ...*/
7687                 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1);
7688                 goto done;
7689         }
7690
7691         if (fw_subcmd == FW_SCHED_SC_PARAMS) {
7692                 int fw_level;
7693                 int fw_mode;
7694                 int fw_rateunit;
7695                 int fw_ratemode;
7696
7697                 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
7698                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
7699                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
7700                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
7701                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
7702                         fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
7703                 else {
7704                         rc = EINVAL;
7705                         goto done;
7706                 }
7707
7708                 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
7709                         fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
7710                 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
7711                         fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
7712                 else {
7713                         rc = EINVAL;
7714                         goto done;
7715                 }
7716
7717                 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
7718                         fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
7719                 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
7720                         fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
7721                 else {
7722                         rc = EINVAL;
7723                         goto done;
7724                 }
7725
7726                 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
7727                         fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
7728                 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
7729                         fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
7730                 else {
7731                         rc = EINVAL;
7732                         goto done;
7733                 }
7734
7735                 /* Vet our parameters ... */
7736                 if (!in_range(p->u.params.channel, 0, 3) ||
7737                     !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
7738                     !in_range(p->u.params.minrate, 0, 10000000) ||
7739                     !in_range(p->u.params.maxrate, 0, 10000000) ||
7740                     !in_range(p->u.params.weight, 0, 100)) {
7741                         rc = ERANGE;
7742                         goto done;
7743                 }
7744
7745                 /*
7746                  * Translate any unset parameters into the firmware's
7747                  * nomenclature and/or fail the call if the parameters
7748                  * are required ...
7749                  */
7750                 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
7751                     p->u.params.channel < 0 || p->u.params.cl < 0) {
7752                         rc = EINVAL;
7753                         goto done;
7754                 }
7755                 if (p->u.params.minrate < 0)
7756                         p->u.params.minrate = 0;
7757                 if (p->u.params.maxrate < 0) {
7758                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7759                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7760                                 rc = EINVAL;
7761                                 goto done;
7762                         } else
7763                                 p->u.params.maxrate = 0;
7764                 }
7765                 if (p->u.params.weight < 0) {
7766                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
7767                                 rc = EINVAL;
7768                                 goto done;
7769                         } else
7770                                 p->u.params.weight = 0;
7771                 }
7772                 if (p->u.params.pktsize < 0) {
7773                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7774                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7775                                 rc = EINVAL;
7776                                 goto done;
7777                         } else
7778                                 p->u.params.pktsize = 0;
7779                 }
7780
7781                 /* See what the firmware thinks of the request ... */
7782                 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
7783                     fw_rateunit, fw_ratemode, p->u.params.channel,
7784                     p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
7785                     p->u.params.weight, p->u.params.pktsize, 1);
7786                 goto done;
7787         }
7788
7789         rc = EINVAL;
7790 done:
7791         end_synchronized_op(sc, 0);
7792         return (rc);
7793 }
7794
7795 static int
7796 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
7797 {
7798         struct port_info *pi = NULL;
7799         struct sge_txq *txq;
7800         uint32_t fw_mnem, fw_queue, fw_class;
7801         int i, rc;
7802
7803         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
7804         if (rc)
7805                 return (rc);
7806
7807         if (!(sc->flags & FULL_INIT_DONE)) {
7808                 rc = EAGAIN;
7809                 goto done;
7810         }
7811
7812         if (p->port >= sc->params.nports) {
7813                 rc = EINVAL;
7814                 goto done;
7815         }
7816
7817         pi = sc->port[p->port];
7818         if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
7819                 rc = EINVAL;
7820                 goto done;
7821         }
7822
7823         /*
7824          * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
7825          * Scheduling Class in this case).
7826          */
7827         fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
7828             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
7829         fw_class = p->cl < 0 ? 0xffffffff : p->cl;
7830
7831         /*
7832          * If op.queue is non-negative, then we're only changing the scheduling
7833          * on a single specified TX queue.
7834          */
7835         if (p->queue >= 0) {
7836                 txq = &sc->sge.txq[pi->first_txq + p->queue];
7837                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7838                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7839                     &fw_class);
7840                 goto done;
7841         }
7842
7843         /*
7844          * Change the scheduling on all the TX queues for the
7845          * interface.
7846          */
7847         for_each_txq(pi, i, txq) {
7848                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7849                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7850                     &fw_class);
7851                 if (rc)
7852                         goto done;
7853         }
7854
7855         rc = 0;
7856 done:
7857         end_synchronized_op(sc, 0);
7858         return (rc);
7859 }
7860
7861 int
7862 t4_os_find_pci_capability(struct adapter *sc, int cap)
7863 {
7864         int i;
7865
7866         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
7867 }
7868
7869 int
7870 t4_os_pci_save_state(struct adapter *sc)
7871 {
7872         device_t dev;
7873         struct pci_devinfo *dinfo;
7874
7875         dev = sc->dev;
7876         dinfo = device_get_ivars(dev);
7877
7878         pci_cfg_save(dev, dinfo, 0);
7879         return (0);
7880 }
7881
7882 int
7883 t4_os_pci_restore_state(struct adapter *sc)
7884 {
7885         device_t dev;
7886         struct pci_devinfo *dinfo;
7887
7888         dev = sc->dev;
7889         dinfo = device_get_ivars(dev);
7890
7891         pci_cfg_restore(dev, dinfo);
7892         return (0);
7893 }
7894
7895 void
7896 t4_os_portmod_changed(const struct adapter *sc, int idx)
7897 {
7898         struct port_info *pi = sc->port[idx];
7899         static const char *mod_str[] = {
7900                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
7901         };
7902
7903         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
7904                 if_printf(pi->ifp, "transceiver unplugged.\n");
7905         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
7906                 if_printf(pi->ifp, "unknown transceiver inserted.\n");
7907         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
7908                 if_printf(pi->ifp, "unsupported transceiver inserted.\n");
7909         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
7910                 if_printf(pi->ifp, "%s transceiver inserted.\n",
7911                     mod_str[pi->mod_type]);
7912         } else {
7913                 if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
7914                     pi->mod_type);
7915         }
7916 }
7917
7918 void
7919 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
7920 {
7921         struct port_info *pi = sc->port[idx];
7922         struct ifnet *ifp = pi->ifp;
7923
7924         if (link_stat) {
7925                 pi->linkdnrc = -1;
7926                 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
7927                 if_link_state_change(ifp, LINK_STATE_UP);
7928         } else {
7929                 if (reason >= 0)
7930                         pi->linkdnrc = reason;
7931                 if_link_state_change(ifp, LINK_STATE_DOWN);
7932         }
7933 }
7934
7935 void
7936 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
7937 {
7938         struct adapter *sc;
7939
7940         sx_slock(&t4_list_lock);
7941         SLIST_FOREACH(sc, &t4_list, link) {
7942                 /*
7943                  * func should not make any assumptions about what state sc is
7944                  * in - the only guarantee is that sc->sc_lock is a valid lock.
7945                  */
7946                 func(sc, arg);
7947         }
7948         sx_sunlock(&t4_list_lock);
7949 }
7950
7951 static int
7952 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
7953 {
7954        return (0);
7955 }
7956
7957 static int
7958 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
7959 {
7960        return (0);
7961 }
7962
7963 static int
7964 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
7965     struct thread *td)
7966 {
7967         int rc;
7968         struct adapter *sc = dev->si_drv1;
7969
7970         rc = priv_check(td, PRIV_DRIVER);
7971         if (rc != 0)
7972                 return (rc);
7973
7974         switch (cmd) {
7975         case CHELSIO_T4_GETREG: {
7976                 struct t4_reg *edata = (struct t4_reg *)data;
7977
7978                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
7979                         return (EFAULT);
7980
7981                 if (edata->size == 4)
7982                         edata->val = t4_read_reg(sc, edata->addr);
7983                 else if (edata->size == 8)
7984                         edata->val = t4_read_reg64(sc, edata->addr);
7985                 else
7986                         return (EINVAL);
7987
7988                 break;
7989         }
7990         case CHELSIO_T4_SETREG: {
7991                 struct t4_reg *edata = (struct t4_reg *)data;
7992
7993                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
7994                         return (EFAULT);
7995
7996                 if (edata->size == 4) {
7997                         if (edata->val & 0xffffffff00000000)
7998                                 return (EINVAL);
7999                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
8000                 } else if (edata->size == 8)
8001                         t4_write_reg64(sc, edata->addr, edata->val);
8002                 else
8003                         return (EINVAL);
8004                 break;
8005         }
8006         case CHELSIO_T4_REGDUMP: {
8007                 struct t4_regdump *regs = (struct t4_regdump *)data;
8008                 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
8009                 uint8_t *buf;
8010
8011                 if (regs->len < reglen) {
8012                         regs->len = reglen; /* hint to the caller */
8013                         return (ENOBUFS);
8014                 }
8015
8016                 regs->len = reglen;
8017                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
8018                 t4_get_regs(sc, regs, buf);
8019                 rc = copyout(buf, regs->data, reglen);
8020                 free(buf, M_CXGBE);
8021                 break;
8022         }
8023         case CHELSIO_T4_GET_FILTER_MODE:
8024                 rc = get_filter_mode(sc, (uint32_t *)data);
8025                 break;
8026         case CHELSIO_T4_SET_FILTER_MODE:
8027                 rc = set_filter_mode(sc, *(uint32_t *)data);
8028                 break;
8029         case CHELSIO_T4_GET_FILTER:
8030                 rc = get_filter(sc, (struct t4_filter *)data);
8031                 break;
8032         case CHELSIO_T4_SET_FILTER:
8033                 rc = set_filter(sc, (struct t4_filter *)data);
8034                 break;
8035         case CHELSIO_T4_DEL_FILTER:
8036                 rc = del_filter(sc, (struct t4_filter *)data);
8037                 break;
8038         case CHELSIO_T4_GET_SGE_CONTEXT:
8039                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
8040                 break;
8041         case CHELSIO_T4_LOAD_FW:
8042                 rc = load_fw(sc, (struct t4_data *)data);
8043                 break;
8044         case CHELSIO_T4_GET_MEM:
8045                 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
8046                 break;
8047         case CHELSIO_T4_GET_I2C:
8048                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
8049                 break;
8050         case CHELSIO_T4_CLEAR_STATS: {
8051                 int i;
8052                 u_int port_id = *(uint32_t *)data;
8053                 struct port_info *pi;
8054
8055                 if (port_id >= sc->params.nports)
8056                         return (EINVAL);
8057                 pi = sc->port[port_id];
8058
8059                 /* MAC stats */
8060                 t4_clr_port_stats(sc, pi->tx_chan);
8061
8062                 if (pi->flags & PORT_INIT_DONE) {
8063                         struct sge_rxq *rxq;
8064                         struct sge_txq *txq;
8065                         struct sge_wrq *wrq;
8066
8067                         for_each_rxq(pi, i, rxq) {
8068 #if defined(INET) || defined(INET6)
8069                                 rxq->lro.lro_queued = 0;
8070                                 rxq->lro.lro_flushed = 0;
8071 #endif
8072                                 rxq->rxcsum = 0;
8073                                 rxq->vlan_extraction = 0;
8074                         }
8075
8076                         for_each_txq(pi, i, txq) {
8077                                 txq->txcsum = 0;
8078                                 txq->tso_wrs = 0;
8079                                 txq->vlan_insertion = 0;
8080                                 txq->imm_wrs = 0;
8081                                 txq->sgl_wrs = 0;
8082                                 txq->txpkt_wrs = 0;
8083                                 txq->txpkts_wrs = 0;
8084                                 txq->txpkts_pkts = 0;
8085                                 txq->br->br_drops = 0;
8086                                 txq->no_dmamap = 0;
8087                                 txq->no_desc = 0;
8088                         }
8089
8090 #ifdef TCP_OFFLOAD
8091                         /* nothing to clear for each ofld_rxq */
8092
8093                         for_each_ofld_txq(pi, i, wrq) {
8094                                 wrq->tx_wrs = 0;
8095                                 wrq->no_desc = 0;
8096                         }
8097 #endif
8098                         wrq = &sc->sge.ctrlq[pi->port_id];
8099                         wrq->tx_wrs = 0;
8100                         wrq->no_desc = 0;
8101                 }
8102                 break;
8103         }
8104         case CHELSIO_T4_SCHED_CLASS:
8105                 rc = set_sched_class(sc, (struct t4_sched_params *)data);
8106                 break;
8107         case CHELSIO_T4_SCHED_QUEUE:
8108                 rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
8109                 break;
8110         case CHELSIO_T4_GET_TRACER:
8111                 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
8112                 break;
8113         case CHELSIO_T4_SET_TRACER:
8114                 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
8115                 break;
8116         default:
8117                 rc = EINVAL;
8118         }
8119
8120         return (rc);
8121 }
8122
8123 #ifdef TCP_OFFLOAD
8124 void
8125 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask,
8126     const unsigned int *pgsz_order)
8127 {
8128         struct port_info *pi = ifp->if_softc;
8129         struct adapter *sc = pi->adapter;
8130
8131         t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
8132         t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |
8133                 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) |
8134                 V_HPZ3(pgsz_order[3]));
8135 }
8136
8137 static int
8138 toe_capability(struct port_info *pi, int enable)
8139 {
8140         int rc;
8141         struct adapter *sc = pi->adapter;
8142
8143         ASSERT_SYNCHRONIZED_OP(sc);
8144
8145         if (!is_offload(sc))
8146                 return (ENODEV);
8147
8148         if (enable) {
8149                 if (!(sc->flags & FULL_INIT_DONE)) {
8150                         rc = cxgbe_init_synchronized(pi);
8151                         if (rc)
8152                                 return (rc);
8153                 }
8154
8155                 if (isset(&sc->offload_map, pi->port_id))
8156                         return (0);
8157
8158                 if (!(sc->flags & TOM_INIT_DONE)) {
8159                         rc = t4_activate_uld(sc, ULD_TOM);
8160                         if (rc == EAGAIN) {
8161                                 log(LOG_WARNING,
8162                                     "You must kldload t4_tom.ko before trying "
8163                                     "to enable TOE on a cxgbe interface.\n");
8164                         }
8165                         if (rc != 0)
8166                                 return (rc);
8167                         KASSERT(sc->tom_softc != NULL,
8168                             ("%s: TOM activated but softc NULL", __func__));
8169                         KASSERT(sc->flags & TOM_INIT_DONE,
8170                             ("%s: TOM activated but flag not set", __func__));
8171                 }
8172
8173                 setbit(&sc->offload_map, pi->port_id);
8174         } else {
8175                 if (!isset(&sc->offload_map, pi->port_id))
8176                         return (0);
8177
8178                 KASSERT(sc->flags & TOM_INIT_DONE,
8179                     ("%s: TOM never initialized?", __func__));
8180                 clrbit(&sc->offload_map, pi->port_id);
8181         }
8182
8183         return (0);
8184 }
8185
8186 /*
8187  * Add an upper layer driver to the global list.
8188  */
8189 int
8190 t4_register_uld(struct uld_info *ui)
8191 {
8192         int rc = 0;
8193         struct uld_info *u;
8194
8195         sx_xlock(&t4_uld_list_lock);
8196         SLIST_FOREACH(u, &t4_uld_list, link) {
8197             if (u->uld_id == ui->uld_id) {
8198                     rc = EEXIST;
8199                     goto done;
8200             }
8201         }
8202
8203         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
8204         ui->refcount = 0;
8205 done:
8206         sx_xunlock(&t4_uld_list_lock);
8207         return (rc);
8208 }
8209
8210 int
8211 t4_unregister_uld(struct uld_info *ui)
8212 {
8213         int rc = EINVAL;
8214         struct uld_info *u;
8215
8216         sx_xlock(&t4_uld_list_lock);
8217
8218         SLIST_FOREACH(u, &t4_uld_list, link) {
8219             if (u == ui) {
8220                     if (ui->refcount > 0) {
8221                             rc = EBUSY;
8222                             goto done;
8223                     }
8224
8225                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
8226                     rc = 0;
8227                     goto done;
8228             }
8229         }
8230 done:
8231         sx_xunlock(&t4_uld_list_lock);
8232         return (rc);
8233 }
8234
8235 int
8236 t4_activate_uld(struct adapter *sc, int id)
8237 {
8238         int rc = EAGAIN;
8239         struct uld_info *ui;
8240
8241         ASSERT_SYNCHRONIZED_OP(sc);
8242
8243         sx_slock(&t4_uld_list_lock);
8244
8245         SLIST_FOREACH(ui, &t4_uld_list, link) {
8246                 if (ui->uld_id == id) {
8247                         rc = ui->activate(sc);
8248                         if (rc == 0)
8249                                 ui->refcount++;
8250                         goto done;
8251                 }
8252         }
8253 done:
8254         sx_sunlock(&t4_uld_list_lock);
8255
8256         return (rc);
8257 }
8258
8259 int
8260 t4_deactivate_uld(struct adapter *sc, int id)
8261 {
8262         int rc = EINVAL;
8263         struct uld_info *ui;
8264
8265         ASSERT_SYNCHRONIZED_OP(sc);
8266
8267         sx_slock(&t4_uld_list_lock);
8268
8269         SLIST_FOREACH(ui, &t4_uld_list, link) {
8270                 if (ui->uld_id == id) {
8271                         rc = ui->deactivate(sc);
8272                         if (rc == 0)
8273                                 ui->refcount--;
8274                         goto done;
8275                 }
8276         }
8277 done:
8278         sx_sunlock(&t4_uld_list_lock);
8279
8280         return (rc);
8281 }
8282 #endif
8283
8284 /*
8285  * Come up with reasonable defaults for some of the tunables, provided they're
8286  * not set by the user (in which case we'll use the values as is).
8287  */
8288 static void
8289 tweak_tunables(void)
8290 {
8291         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
8292
8293         if (t4_ntxq10g < 1)
8294                 t4_ntxq10g = min(nc, NTXQ_10G);
8295
8296         if (t4_ntxq1g < 1)
8297                 t4_ntxq1g = min(nc, NTXQ_1G);
8298
8299         if (t4_nrxq10g < 1)
8300                 t4_nrxq10g = min(nc, NRXQ_10G);
8301
8302         if (t4_nrxq1g < 1)
8303                 t4_nrxq1g = min(nc, NRXQ_1G);
8304
8305 #ifdef TCP_OFFLOAD
8306         if (t4_nofldtxq10g < 1)
8307                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
8308
8309         if (t4_nofldtxq1g < 1)
8310                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
8311
8312         if (t4_nofldrxq10g < 1)
8313                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
8314
8315         if (t4_nofldrxq1g < 1)
8316                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
8317
8318         if (t4_toecaps_allowed == -1)
8319                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
8320 #else
8321         if (t4_toecaps_allowed == -1)
8322                 t4_toecaps_allowed = 0;
8323 #endif
8324
8325 #ifdef DEV_NETMAP
8326         if (t4_nnmtxq10g < 1)
8327                 t4_nnmtxq10g = min(nc, NNMTXQ_10G);
8328
8329         if (t4_nnmtxq1g < 1)
8330                 t4_nnmtxq1g = min(nc, NNMTXQ_1G);
8331
8332         if (t4_nnmrxq10g < 1)
8333                 t4_nnmrxq10g = min(nc, NNMRXQ_10G);
8334
8335         if (t4_nnmrxq1g < 1)
8336                 t4_nnmrxq1g = min(nc, NNMRXQ_1G);
8337 #endif
8338
8339         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
8340                 t4_tmr_idx_10g = TMR_IDX_10G;
8341
8342         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
8343                 t4_pktc_idx_10g = PKTC_IDX_10G;
8344
8345         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
8346                 t4_tmr_idx_1g = TMR_IDX_1G;
8347
8348         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
8349                 t4_pktc_idx_1g = PKTC_IDX_1G;
8350
8351         if (t4_qsize_txq < 128)
8352                 t4_qsize_txq = 128;
8353
8354         if (t4_qsize_rxq < 128)
8355                 t4_qsize_rxq = 128;
8356         while (t4_qsize_rxq & 7)
8357                 t4_qsize_rxq++;
8358
8359         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
8360 }
8361
8362 static struct sx mlu;   /* mod load unload */
8363 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
8364
8365 static int
8366 mod_event(module_t mod, int cmd, void *arg)
8367 {
8368         int rc = 0;
8369         static int loaded = 0;
8370
8371         switch (cmd) {
8372         case MOD_LOAD:
8373                 sx_xlock(&mlu);
8374                 if (loaded++ == 0) {
8375                         t4_sge_modload();
8376                         sx_init(&t4_list_lock, "T4/T5 adapters");
8377                         SLIST_INIT(&t4_list);
8378 #ifdef TCP_OFFLOAD
8379                         sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
8380                         SLIST_INIT(&t4_uld_list);
8381 #endif
8382                         t4_tracer_modload();
8383                         tweak_tunables();
8384                 }
8385                 sx_xunlock(&mlu);
8386                 break;
8387
8388         case MOD_UNLOAD:
8389                 sx_xlock(&mlu);
8390                 if (--loaded == 0) {
8391                         int tries;
8392
8393                         sx_slock(&t4_list_lock);
8394                         if (!SLIST_EMPTY(&t4_list)) {
8395                                 rc = EBUSY;
8396                                 sx_sunlock(&t4_list_lock);
8397                                 goto done_unload;
8398                         }
8399 #ifdef TCP_OFFLOAD
8400                         sx_slock(&t4_uld_list_lock);
8401                         if (!SLIST_EMPTY(&t4_uld_list)) {
8402                                 rc = EBUSY;
8403                                 sx_sunlock(&t4_uld_list_lock);
8404                                 sx_sunlock(&t4_list_lock);
8405                                 goto done_unload;
8406                         }
8407 #endif
8408                         tries = 0;
8409                         while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
8410                                 uprintf("%ju clusters with custom free routine "
8411                                     "still is use.\n", t4_sge_extfree_refs());
8412                                 pause("t4unload", 2 * hz);
8413                         }
8414 #ifdef TCP_OFFLOAD
8415                         sx_sunlock(&t4_uld_list_lock);
8416 #endif
8417                         sx_sunlock(&t4_list_lock);
8418
8419                         if (t4_sge_extfree_refs() == 0) {
8420                                 t4_tracer_modunload();
8421 #ifdef TCP_OFFLOAD
8422                                 sx_destroy(&t4_uld_list_lock);
8423 #endif
8424                                 sx_destroy(&t4_list_lock);
8425                                 t4_sge_modunload();
8426                                 loaded = 0;
8427                         } else {
8428                                 rc = EBUSY;
8429                                 loaded++;       /* undo earlier decrement */
8430                         }
8431                 }
8432 done_unload:
8433                 sx_xunlock(&mlu);
8434                 break;
8435         }
8436
8437         return (rc);
8438 }
8439
8440 static devclass_t t4_devclass, t5_devclass;
8441 static devclass_t cxgbe_devclass, cxl_devclass;
8442
8443 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
8444 MODULE_VERSION(t4nex, 1);
8445 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
8446
8447 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
8448 MODULE_VERSION(t5nex, 1);
8449 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
8450
8451 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
8452 MODULE_VERSION(cxgbe, 1);
8453
8454 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
8455 MODULE_VERSION(cxl, 1);