sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/conf.h>
  36 #include <sys/priv.h>
  37 #include <sys/kernel.h>
  38 #include <sys/bus.h>
  39 #include <sys/module.h>
  40 #include <sys/malloc.h>
  41 #include <sys/queue.h>
  42 #include <sys/taskqueue.h>
  43 #include <sys/pciio.h>
  44 #include <dev/pci/pcireg.h>
  45 #include <dev/pci/pcivar.h>
  46 #include <dev/pci/pci_private.h>
  47 #include <sys/firmware.h>
  48 #include <sys/sbuf.h>
  49 #include <sys/smp.h>
  50 #include <sys/socket.h>
  51 #include <sys/sockio.h>
  52 #include <sys/sysctl.h>
  53 #include <net/ethernet.h>
  54 #include <net/if.h>
  55 #include <net/if_types.h>
  56 #include <net/if_dl.h>
  57 #include <net/if_vlan_var.h>
  58 #if defined(__i386__) || defined(__amd64__)
  59 #include <vm/vm.h>
  60 #include <vm/pmap.h>
  61 #endif
  62
  63 #include "common/common.h"
  64 #include "common/t4_msg.h"
  65 #include "common/t4_regs.h"
  66 #include "common/t4_regs_values.h"
  67 #include "t4_ioctl.h"
  68 #include "t4_l2t.h"
  69
  70 /* T4 bus driver interface */
  71 static int t4_probe(device_t);
  72 static int t4_attach(device_t);
  73 static int t4_detach(device_t);
  74 static device_method_t t4_methods[] = {
  75         DEVMETHOD(device_probe,         t4_probe),
  76         DEVMETHOD(device_attach,        t4_attach),
  77         DEVMETHOD(device_detach,        t4_detach),
  78
  79         DEVMETHOD_END
  80 };
  81 static driver_t t4_driver = {
  82         "t4nex",
  83         t4_methods,
  84         sizeof(struct adapter)
  85 };
  86
  87
  88 /* T4 port (cxgbe) interface */
  89 static int cxgbe_probe(device_t);
  90 static int cxgbe_attach(device_t);
  91 static int cxgbe_detach(device_t);
  92 static device_method_t cxgbe_methods[] = {
  93         DEVMETHOD(device_probe,         cxgbe_probe),
  94         DEVMETHOD(device_attach,        cxgbe_attach),
  95         DEVMETHOD(device_detach,        cxgbe_detach),
  96         { 0, 0 }
  97 };
  98 static driver_t cxgbe_driver = {
  99         "cxgbe",
 100         cxgbe_methods,
 101         sizeof(struct port_info)
 102 };
 103
 104 static d_ioctl_t t4_ioctl;
 105 static d_open_t t4_open;
 106 static d_close_t t4_close;
 107
 108 static struct cdevsw t4_cdevsw = {
 109        .d_version = D_VERSION,
 110        .d_flags = 0,
 111        .d_open = t4_open,
 112        .d_close = t4_close,
 113        .d_ioctl = t4_ioctl,
 114        .d_name = "t4nex",
 115 };
 116
 117 /* T5 bus driver interface */
 118 static int t5_probe(device_t);
 119 static device_method_t t5_methods[] = {
 120         DEVMETHOD(device_probe,         t5_probe),
 121         DEVMETHOD(device_attach,        t4_attach),
 122         DEVMETHOD(device_detach,        t4_detach),
 123
 124         DEVMETHOD_END
 125 };
 126 static driver_t t5_driver = {
 127         "t5nex",
 128         t5_methods,
 129         sizeof(struct adapter)
 130 };
 131
 132
 133 /* T5 port (cxl) interface */
 134 static driver_t cxl_driver = {
 135         "cxl",
 136         cxgbe_methods,
 137         sizeof(struct port_info)
 138 };
 139
 140 static struct cdevsw t5_cdevsw = {
 141        .d_version = D_VERSION,
 142        .d_flags = 0,
 143        .d_open = t4_open,
 144        .d_close = t4_close,
 145        .d_ioctl = t4_ioctl,
 146        .d_name = "t5nex",
 147 };
 148
 149 /* ifnet + media interface */
 150 static void cxgbe_init(void *);
 151 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 152 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 153 static void cxgbe_qflush(struct ifnet *);
 154 static uint64_t cxgbe_get_counter(struct ifnet *, ift_counter);
 155 static int cxgbe_media_change(struct ifnet *);
 156 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 157
 158 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
 159
 160 /*
 161  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 162  * then ADAPTER_LOCK, then t4_uld_list_lock.
 163  */
 164 static struct sx t4_list_lock;
 165 SLIST_HEAD(, adapter) t4_list;
 166 #ifdef TCP_OFFLOAD
 167 static struct sx t4_uld_list_lock;
 168 SLIST_HEAD(, uld_info) t4_uld_list;
 169 #endif
 170
 171 /*
 172  * Tunables.  See tweak_tunables() too.
 173  *
 174  * Each tunable is set to a default value here if it's known at compile-time.
 175  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
 176  * provide a reasonable default when the driver is loaded.
 177  *
 178  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
 179  * T5 are under hw.cxl.
 180  */
 181
 182 /*
 183  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 184  */
 185 #define NTXQ_10G 16
 186 static int t4_ntxq10g = -1;
 187 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 188
 189 #define NRXQ_10G 8
 190 static int t4_nrxq10g = -1;
 191 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 192
 193 #define NTXQ_1G 4
 194 static int t4_ntxq1g = -1;
 195 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 196
 197 #define NRXQ_1G 2
 198 static int t4_nrxq1g = -1;
 199 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 200
 201 static int t4_rsrv_noflowq = 0;
 202 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
 203
 204 #ifdef TCP_OFFLOAD
 205 #define NOFLDTXQ_10G 8
 206 static int t4_nofldtxq10g = -1;
 207 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 208
 209 #define NOFLDRXQ_10G 2
 210 static int t4_nofldrxq10g = -1;
 211 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 212
 213 #define NOFLDTXQ_1G 2
 214 static int t4_nofldtxq1g = -1;
 215 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 216
 217 #define NOFLDRXQ_1G 1
 218 static int t4_nofldrxq1g = -1;
 219 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 220 #endif
 221
 222 #ifdef DEV_NETMAP
 223 #define NNMTXQ_10G 2
 224 static int t4_nnmtxq10g = -1;
 225 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g);
 226
 227 #define NNMRXQ_10G 2
 228 static int t4_nnmrxq10g = -1;
 229 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g);
 230
 231 #define NNMTXQ_1G 1
 232 static int t4_nnmtxq1g = -1;
 233 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g);
 234
 235 #define NNMRXQ_1G 1
 236 static int t4_nnmrxq1g = -1;
 237 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g);
 238 #endif
 239
 240 /*
 241  * Holdoff parameters for 10G and 1G ports.
 242  */
 243 #define TMR_IDX_10G 1
 244 static int t4_tmr_idx_10g = TMR_IDX_10G;
 245 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 246
 247 #define PKTC_IDX_10G (-1)
 248 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 249 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 250
 251 #define TMR_IDX_1G 1
 252 static int t4_tmr_idx_1g = TMR_IDX_1G;
 253 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 254
 255 #define PKTC_IDX_1G (-1)
 256 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 257 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 258
 259 /*
 260  * Size (# of entries) of each tx and rx queue.
 261  */
 262 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 263 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 264
 265 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 266 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 267
 268 /*
 269  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 270  */
 271 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 272 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 273
 274 /*
 275  * Configuration file.
 276  */
 277 #define DEFAULT_CF      "default"
 278 #define FLASH_CF        "flash"
 279 #define UWIRE_CF        "uwire"
 280 #define FPGA_CF         "fpga"
 281 static char t4_cfg_file[32] = DEFAULT_CF;
 282 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 283
 284 /*
 285  * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively).
 286  * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
 287  * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
 288  *            mark or when signalled to do so, 0 to never emit PAUSE.
 289  */
 290 static int t4_pause_settings = PAUSE_TX | PAUSE_RX;
 291 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings);
 292
 293 /*
 294  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
 295  * encouraged respectively).
 296  */
 297 static unsigned int t4_fw_install = 1;
 298 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 299
 300 /*
 301  * ASIC features that will be used.  Disable the ones you don't want so that the
 302  * chip resources aren't wasted on features that will not be used.
 303  */
 304 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 305 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 306
 307 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 308 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 309
 310 static int t4_toecaps_allowed = -1;
 311 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 312
 313 static int t4_rdmacaps_allowed = 0;
 314 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 315
 316 static int t4_iscsicaps_allowed = 0;
 317 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 318
 319 static int t4_fcoecaps_allowed = 0;
 320 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 321
 322 static int t5_write_combine = 0;
 323 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 324
 325 struct intrs_and_queues {
 326         uint16_t intr_type;     /* INTx, MSI, or MSI-X */
 327         uint16_t nirq;          /* Total # of vectors */
 328         uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
 329         uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */
 330         uint16_t ntxq10g;       /* # of NIC txq's for each 10G port */
 331         uint16_t nrxq10g;       /* # of NIC rxq's for each 10G port */
 332         uint16_t ntxq1g;        /* # of NIC txq's for each 1G port */
 333         uint16_t nrxq1g;        /* # of NIC rxq's for each 1G port */
 334         uint16_t rsrv_noflowq;  /* Flag whether to reserve queue 0 */
 335 #ifdef TCP_OFFLOAD
 336         uint16_t nofldtxq10g;   /* # of TOE txq's for each 10G port */
 337         uint16_t nofldrxq10g;   /* # of TOE rxq's for each 10G port */
 338         uint16_t nofldtxq1g;    /* # of TOE txq's for each 1G port */
 339         uint16_t nofldrxq1g;    /* # of TOE rxq's for each 1G port */
 340 #endif
 341 #ifdef DEV_NETMAP
 342         uint16_t nnmtxq10g;     /* # of netmap txq's for each 10G port */
 343         uint16_t nnmrxq10g;     /* # of netmap rxq's for each 10G port */
 344         uint16_t nnmtxq1g;      /* # of netmap txq's for each 1G port */
 345         uint16_t nnmrxq1g;      /* # of netmap rxq's for each 1G port */
 346 #endif
 347 };
 348
 349 struct filter_entry {
 350         uint32_t valid:1;       /* filter allocated and valid */
 351         uint32_t locked:1;      /* filter is administratively locked */
 352         uint32_t pending:1;     /* filter action is pending firmware reply */
 353         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 354         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 355
 356         struct t4_filter_specification fs;
 357 };
 358
 359 static int map_bars_0_and_4(struct adapter *);
 360 static int map_bar_2(struct adapter *);
 361 static void setup_memwin(struct adapter *);
 362 static int validate_mem_range(struct adapter *, uint32_t, int);
 363 static int fwmtype_to_hwmtype(int);
 364 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
 365     uint32_t *);
 366 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
 367 static uint32_t position_memwin(struct adapter *, int, uint32_t);
 368 static int cfg_itype_and_nqueues(struct adapter *, int, int,
 369     struct intrs_and_queues *);
 370 static int prep_firmware(struct adapter *);
 371 static int partition_resources(struct adapter *, const struct firmware *,
 372     const char *);
 373 static int get_params__pre_init(struct adapter *);
 374 static int get_params__post_init(struct adapter *);
 375 static int set_params__post_init(struct adapter *);
 376 static void t4_set_desc(struct adapter *);
 377 static void build_medialist(struct port_info *, struct ifmedia *);
 378 static int cxgbe_init_synchronized(struct port_info *);
 379 static int cxgbe_uninit_synchronized(struct port_info *);
 380 static int setup_intr_handlers(struct adapter *);
 381 static void quiesce_eq(struct adapter *, struct sge_eq *);
 382 static void quiesce_iq(struct adapter *, struct sge_iq *);
 383 static void quiesce_fl(struct adapter *, struct sge_fl *);
 384 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 385     driver_intr_t *, void *, char *);
 386 static int t4_free_irq(struct adapter *, struct irq *);
 387 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
 388     unsigned int);
 389 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 390 static void cxgbe_refresh_stats(struct adapter *, struct port_info *);
 391 static void cxgbe_tick(void *);
 392 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 393 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
 394     struct mbuf *);
 395 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 396 static int fw_msg_not_handled(struct adapter *, const __be64 *);
 397 static int t4_sysctls(struct adapter *);
 398 static int cxgbe_sysctls(struct port_info *);
 399 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 400 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 401 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
 402 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 403 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 404 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 405 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 406 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 407 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
 408 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 409 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 410 #ifdef SBUF_DRAIN
 411 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 412 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 413 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 414 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
 415 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
 416 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 417 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 418 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 419 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 420 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 421 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 422 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 423 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
 424 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 425 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
 426 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 427 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 428 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 429 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 430 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 431 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 432 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
 433 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 434 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 435 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 436 #endif
 437 static inline void txq_start(struct ifnet *, struct sge_txq *);
 438 static uint32_t fconf_to_mode(uint32_t);
 439 static uint32_t mode_to_fconf(uint32_t);
 440 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 441 static int get_filter_mode(struct adapter *, uint32_t *);
 442 static int set_filter_mode(struct adapter *, uint32_t);
 443 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 444 static int get_filter(struct adapter *, struct t4_filter *);
 445 static int set_filter(struct adapter *, struct t4_filter *);
 446 static int del_filter(struct adapter *, struct t4_filter *);
 447 static void clear_filter(struct filter_entry *);
 448 static int set_filter_wr(struct adapter *, int);
 449 static int del_filter_wr(struct adapter *, int);
 450 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 451 static int load_fw(struct adapter *, struct t4_data *);
 452 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 453 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 454 static int set_sched_class(struct adapter *, struct t4_sched_params *);
 455 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
 456 #ifdef TCP_OFFLOAD
 457 static int toe_capability(struct port_info *, int);
 458 #endif
 459 static int mod_event(module_t, int, void *);
 460
 461 struct {
 462         uint16_t device;
 463         char *desc;
 464 } t4_pciids[] = {
 465         {0xa000, "Chelsio Terminator 4 FPGA"},
 466         {0x4400, "Chelsio T440-dbg"},
 467         {0x4401, "Chelsio T420-CR"},
 468         {0x4402, "Chelsio T422-CR"},
 469         {0x4403, "Chelsio T440-CR"},
 470         {0x4404, "Chelsio T420-BCH"},
 471         {0x4405, "Chelsio T440-BCH"},
 472         {0x4406, "Chelsio T440-CH"},
 473         {0x4407, "Chelsio T420-SO"},
 474         {0x4408, "Chelsio T420-CX"},
 475         {0x4409, "Chelsio T420-BT"},
 476         {0x440a, "Chelsio T404-BT"},
 477         {0x440e, "Chelsio T440-LP-CR"},
 478 }, t5_pciids[] = {
 479         {0xb000, "Chelsio Terminator 5 FPGA"},
 480         {0x5400, "Chelsio T580-dbg"},
 481         {0x5401,  "Chelsio T520-CR"},           /* 2 x 10G */
 482         {0x5402,  "Chelsio T522-CR"},           /* 2 x 10G, 2 X 1G */
 483         {0x5403,  "Chelsio T540-CR"},           /* 4 x 10G */
 484         {0x5407,  "Chelsio T520-SO"},           /* 2 x 10G, nomem */
 485         {0x5409,  "Chelsio T520-BT"},           /* 2 x 10GBaseT */
 486         {0x540a,  "Chelsio T504-BT"},           /* 4 x 1G */
 487         {0x540d,  "Chelsio T580-CR"},           /* 2 x 40G */
 488         {0x540e,  "Chelsio T540-LP-CR"},        /* 4 x 10G */
 489         {0x5410,  "Chelsio T580-LP-CR"},        /* 2 x 40G */
 490         {0x5411,  "Chelsio T520-LL-CR"},        /* 2 x 10G */
 491         {0x5412,  "Chelsio T560-CR"},           /* 1 x 40G, 2 x 10G */
 492         {0x5414,  "Chelsio T580-LP-SO-CR"},     /* 2 x 40G, nomem */
 493         {0x5415,  "Chelsio T502-BT"},           /* 2 x 1G */
 494 #ifdef notyet
 495         {0x5404,  "Chelsio T520-BCH"},
 496         {0x5405,  "Chelsio T540-BCH"},
 497         {0x5406,  "Chelsio T540-CH"},
 498         {0x5408,  "Chelsio T520-CX"},
 499         {0x540b,  "Chelsio B520-SR"},
 500         {0x540c,  "Chelsio B504-BT"},
 501         {0x540f,  "Chelsio Amsterdam"},
 502         {0x5413,  "Chelsio T580-CHR"},
 503 #endif
 504 };
 505
 506 #ifdef TCP_OFFLOAD
 507 /*
 508  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 509  * exactly the same for both rxq and ofld_rxq.
 510  */
 511 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 512 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 513 #endif
 514
 515 /* No easy way to include t4_msg.h before adapter.h so we check this way */
 516 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
 517 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
 518
 519 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
 520
 521 static int
 522 t4_probe(device_t dev)
 523 {
 524         int i;
 525         uint16_t v = pci_get_vendor(dev);
 526         uint16_t d = pci_get_device(dev);
 527         uint8_t f = pci_get_function(dev);
 528
 529         if (v != PCI_VENDOR_ID_CHELSIO)
 530                 return (ENXIO);
 531
 532         /* Attach only to PF0 of the FPGA */
 533         if (d == 0xa000 && f != 0)
 534                 return (ENXIO);
 535
 536         for (i = 0; i < nitems(t4_pciids); i++) {
 537                 if (d == t4_pciids[i].device) {
 538                         device_set_desc(dev, t4_pciids[i].desc);
 539                         return (BUS_PROBE_DEFAULT);
 540                 }
 541         }
 542
 543         return (ENXIO);
 544 }
 545
 546 static int
 547 t5_probe(device_t dev)
 548 {
 549         int i;
 550         uint16_t v = pci_get_vendor(dev);
 551         uint16_t d = pci_get_device(dev);
 552         uint8_t f = pci_get_function(dev);
 553
 554         if (v != PCI_VENDOR_ID_CHELSIO)
 555                 return (ENXIO);
 556
 557         /* Attach only to PF0 of the FPGA */
 558         if (d == 0xb000 && f != 0)
 559                 return (ENXIO);
 560
 561         for (i = 0; i < nitems(t5_pciids); i++) {
 562                 if (d == t5_pciids[i].device) {
 563                         device_set_desc(dev, t5_pciids[i].desc);
 564                         return (BUS_PROBE_DEFAULT);
 565                 }
 566         }
 567
 568         return (ENXIO);
 569 }
 570
 571 static int
 572 t4_attach(device_t dev)
 573 {
 574         struct adapter *sc;
 575         int rc = 0, i, n10g, n1g, rqidx, tqidx;
 576         struct intrs_and_queues iaq;
 577         struct sge *s;
 578 #ifdef TCP_OFFLOAD
 579         int ofld_rqidx, ofld_tqidx;
 580 #endif
 581 #ifdef DEV_NETMAP
 582         int nm_rqidx, nm_tqidx;
 583 #endif
 584         const char *pcie_ts;
 585
 586         sc = device_get_softc(dev);
 587         sc->dev = dev;
 588
 589         pci_enable_busmaster(dev);
 590         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 591                 uint32_t v;
 592
 593                 pci_set_max_read_req(dev, 4096);
 594                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 595                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 596                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 597
 598                 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
 599         }
 600
 601         sc->traceq = -1;
 602         mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
 603         snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
 604             device_get_nameunit(dev));
 605
 606         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 607             device_get_nameunit(dev));
 608         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 609         sx_xlock(&t4_list_lock);
 610         SLIST_INSERT_HEAD(&t4_list, sc, link);
 611         sx_xunlock(&t4_list_lock);
 612
 613         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 614         TAILQ_INIT(&sc->sfl);
 615         callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
 616
 617         mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF);
 618
 619         rc = map_bars_0_and_4(sc);
 620         if (rc != 0)
 621                 goto done; /* error message displayed already */
 622
 623         /*
 624          * This is the real PF# to which we're attaching.  Works from within PCI
 625          * passthrough environments too, where pci_get_function() could return a
 626          * different PF# depending on the passthrough configuration.  We need to
 627          * use the real PF# in all our communication with the firmware.
 628          */
 629         sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 630         sc->mbox = sc->pf;
 631
 632         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 633         sc->an_handler = an_not_handled;
 634         for (i = 0; i < nitems(sc->cpl_handler); i++)
 635                 sc->cpl_handler[i] = cpl_not_handled;
 636         for (i = 0; i < nitems(sc->fw_msg_handler); i++)
 637                 sc->fw_msg_handler[i] = fw_msg_not_handled;
 638         t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 639         t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
 640         t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt);
 641         t4_init_sge_cpl_handlers(sc);
 642
 643         /* Prepare the adapter for operation */
 644         rc = -t4_prep_adapter(sc);
 645         if (rc != 0) {
 646                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 647                 goto done;
 648         }
 649
 650         /*
 651          * Do this really early, with the memory windows set up even before the
 652          * character device.  The userland tool's register i/o and mem read
 653          * will work even in "recovery mode".
 654          */
 655         setup_memwin(sc);
 656         sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
 657             device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
 658             device_get_nameunit(dev));
 659         if (sc->cdev == NULL)
 660                 device_printf(dev, "failed to create nexus char device.\n");
 661         else
 662                 sc->cdev->si_drv1 = sc;
 663
 664         /* Go no further if recovery mode has been requested. */
 665         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 666                 device_printf(dev, "recovery mode.\n");
 667                 goto done;
 668         }
 669
 670         /* Prepare the firmware for operation */
 671         rc = prep_firmware(sc);
 672         if (rc != 0)
 673                 goto done; /* error message displayed already */
 674
 675         rc = get_params__post_init(sc);
 676         if (rc != 0)
 677                 goto done; /* error message displayed already */
 678
 679         rc = set_params__post_init(sc);
 680         if (rc != 0)
 681                 goto done; /* error message displayed already */
 682
 683         rc = map_bar_2(sc);
 684         if (rc != 0)
 685                 goto done; /* error message displayed already */
 686
 687         rc = t4_create_dma_tag(sc);
 688         if (rc != 0)
 689                 goto done; /* error message displayed already */
 690
 691         /*
 692          * First pass over all the ports - allocate VIs and initialize some
 693          * basic parameters like mac address, port type, etc.  We also figure
 694          * out whether a port is 10G or 1G and use that information when
 695          * calculating how many interrupts to attempt to allocate.
 696          */
 697         n10g = n1g = 0;
 698         for_each_port(sc, i) {
 699                 struct port_info *pi;
 700
 701                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 702                 sc->port[i] = pi;
 703
 704                 /* These must be set before t4_port_init */
 705                 pi->adapter = sc;
 706                 pi->port_id = i;
 707
 708                 /* Allocate the vi and initialize parameters like mac addr */
 709                 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 710                 if (rc != 0) {
 711                         device_printf(dev, "unable to initialize port %d: %d\n",
 712                             i, rc);
 713                         free(pi, M_CXGBE);
 714                         sc->port[i] = NULL;
 715                         goto done;
 716                 }
 717
 718                 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX);
 719                 pi->link_cfg.requested_fc |= t4_pause_settings;
 720                 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX);
 721                 pi->link_cfg.fc |= t4_pause_settings;
 722
 723                 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
 724                 if (rc != 0) {
 725                         device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
 726                         free(pi, M_CXGBE);
 727                         sc->port[i] = NULL;
 728                         goto done;
 729                 }
 730
 731                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 732                     device_get_nameunit(dev), i);
 733                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 734                 sc->chan_map[pi->tx_chan] = i;
 735
 736                 if (is_10G_port(pi) || is_40G_port(pi)) {
 737                         n10g++;
 738                         pi->tmr_idx = t4_tmr_idx_10g;
 739                         pi->pktc_idx = t4_pktc_idx_10g;
 740                 } else {
 741                         n1g++;
 742                         pi->tmr_idx = t4_tmr_idx_1g;
 743                         pi->pktc_idx = t4_pktc_idx_1g;
 744                 }
 745
 746                 pi->xact_addr_filt = -1;
 747                 pi->linkdnrc = -1;
 748
 749                 pi->qsize_rxq = t4_qsize_rxq;
 750                 pi->qsize_txq = t4_qsize_txq;
 751
 752                 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
 753                 if (pi->dev == NULL) {
 754                         device_printf(dev,
 755                             "failed to add device for port %d.\n", i);
 756                         rc = ENXIO;
 757                         goto done;
 758                 }
 759                 device_set_softc(pi->dev, pi);
 760         }
 761
 762         /*
 763          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 764          */
 765         rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
 766         if (rc != 0)
 767                 goto done; /* error message displayed already */
 768
 769         sc->intr_type = iaq.intr_type;
 770         sc->intr_count = iaq.nirq;
 771
 772         s = &sc->sge;
 773         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 774         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 775         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
 776         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 777         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
 778 #ifdef TCP_OFFLOAD
 779         if (is_offload(sc)) {
 780                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 781                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 782                 s->neq += s->nofldtxq + s->nofldrxq;
 783                 s->niq += s->nofldrxq;
 784
 785                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 786                     M_CXGBE, M_ZERO | M_WAITOK);
 787                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 788                     M_CXGBE, M_ZERO | M_WAITOK);
 789         }
 790 #endif
 791 #ifdef DEV_NETMAP
 792         s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g;
 793         s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g;
 794         s->neq += s->nnmtxq + s->nnmrxq;
 795         s->niq += s->nnmrxq;
 796
 797         s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
 798             M_CXGBE, M_ZERO | M_WAITOK);
 799         s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
 800             M_CXGBE, M_ZERO | M_WAITOK);
 801 #endif
 802
 803         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 804             M_ZERO | M_WAITOK);
 805         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 806             M_ZERO | M_WAITOK);
 807         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 808             M_ZERO | M_WAITOK);
 809         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 810             M_ZERO | M_WAITOK);
 811         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 812             M_ZERO | M_WAITOK);
 813
 814         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 815             M_ZERO | M_WAITOK);
 816
 817         t4_init_l2t(sc, M_WAITOK);
 818
 819         /*
 820          * Second pass over the ports.  This time we know the number of rx and
 821          * tx queues that each port should get.
 822          */
 823         rqidx = tqidx = 0;
 824 #ifdef TCP_OFFLOAD
 825         ofld_rqidx = ofld_tqidx = 0;
 826 #endif
 827 #ifdef DEV_NETMAP
 828         nm_rqidx = nm_tqidx = 0;
 829 #endif
 830         for_each_port(sc, i) {
 831                 struct port_info *pi = sc->port[i];
 832
 833                 if (pi == NULL)
 834                         continue;
 835
 836                 pi->first_rxq = rqidx;
 837                 pi->first_txq = tqidx;
 838                 if (is_10G_port(pi) || is_40G_port(pi)) {
 839                         pi->flags |= iaq.intr_flags_10g;
 840                         pi->nrxq = iaq.nrxq10g;
 841                         pi->ntxq = iaq.ntxq10g;
 842                 } else {
 843                         pi->flags |= iaq.intr_flags_1g;
 844                         pi->nrxq = iaq.nrxq1g;
 845                         pi->ntxq = iaq.ntxq1g;
 846                 }
 847
 848                 if (pi->ntxq > 1)
 849                         pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
 850                 else
 851                         pi->rsrv_noflowq = 0;
 852
 853                 rqidx += pi->nrxq;
 854                 tqidx += pi->ntxq;
 855 #ifdef TCP_OFFLOAD
 856                 if (is_offload(sc)) {
 857                         pi->first_ofld_rxq = ofld_rqidx;
 858                         pi->first_ofld_txq = ofld_tqidx;
 859                         if (is_10G_port(pi) || is_40G_port(pi)) {
 860                                 pi->nofldrxq = iaq.nofldrxq10g;
 861                                 pi->nofldtxq = iaq.nofldtxq10g;
 862                         } else {
 863                                 pi->nofldrxq = iaq.nofldrxq1g;
 864                                 pi->nofldtxq = iaq.nofldtxq1g;
 865                         }
 866                         ofld_rqidx += pi->nofldrxq;
 867                         ofld_tqidx += pi->nofldtxq;
 868                 }
 869 #endif
 870 #ifdef DEV_NETMAP
 871                 pi->first_nm_rxq = nm_rqidx;
 872                 pi->first_nm_txq = nm_tqidx;
 873                 if (is_10G_port(pi) || is_40G_port(pi)) {
 874                         pi->nnmrxq = iaq.nnmrxq10g;
 875                         pi->nnmtxq = iaq.nnmtxq10g;
 876                 } else {
 877                         pi->nnmrxq = iaq.nnmrxq1g;
 878                         pi->nnmtxq = iaq.nnmtxq1g;
 879                 }
 880                 nm_rqidx += pi->nnmrxq;
 881                 nm_tqidx += pi->nnmtxq;
 882 #endif
 883         }
 884
 885         rc = setup_intr_handlers(sc);
 886         if (rc != 0) {
 887                 device_printf(dev,
 888                     "failed to setup interrupt handlers: %d\n", rc);
 889                 goto done;
 890         }
 891
 892         rc = bus_generic_attach(dev);
 893         if (rc != 0) {
 894                 device_printf(dev,
 895                     "failed to attach all child ports: %d\n", rc);
 896                 goto done;
 897         }
 898
 899         switch (sc->params.pci.speed) {
 900                 case 0x1:
 901                         pcie_ts = "2.5";
 902                         break;
 903                 case 0x2:
 904                         pcie_ts = "5.0";
 905                         break;
 906                 case 0x3:
 907                         pcie_ts = "8.0";
 908                         break;
 909                 default:
 910                         pcie_ts = "??";
 911                         break;
 912         }
 913         device_printf(dev,
 914             "PCIe x%d (%s GTS/s) (%d), %d ports, %d %s interrupt%s, %d eq, %d iq\n",
 915             sc->params.pci.width, pcie_ts, sc->params.pci.speed,
 916             sc->params.nports, sc->intr_count,
 917             sc->intr_type == INTR_MSIX ? "MSI-X" :
 918             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
 919             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
 920
 921         t4_set_desc(sc);
 922
 923 done:
 924         if (rc != 0 && sc->cdev) {
 925                 /* cdev was created and so cxgbetool works; recover that way. */
 926                 device_printf(dev,
 927                     "error during attach, adapter is now in recovery mode.\n");
 928                 rc = 0;
 929         }
 930
 931         if (rc != 0)
 932                 t4_detach(dev);
 933         else
 934                 t4_sysctls(sc);
 935
 936         return (rc);
 937 }
 938
 939 /*
 940  * Idempotent
 941  */
 942 static int
 943 t4_detach(device_t dev)
 944 {
 945         struct adapter *sc;
 946         struct port_info *pi;
 947         int i, rc;
 948
 949         sc = device_get_softc(dev);
 950
 951         if (sc->flags & FULL_INIT_DONE)
 952                 t4_intr_disable(sc);
 953
 954         if (sc->cdev) {
 955                 destroy_dev(sc->cdev);
 956                 sc->cdev = NULL;
 957         }
 958
 959         rc = bus_generic_detach(dev);
 960         if (rc) {
 961                 device_printf(dev,
 962                     "failed to detach child devices: %d\n", rc);
 963                 return (rc);
 964         }
 965
 966         for (i = 0; i < sc->intr_count; i++)
 967                 t4_free_irq(sc, &sc->irq[i]);
 968
 969         for (i = 0; i < MAX_NPORTS; i++) {
 970                 pi = sc->port[i];
 971                 if (pi) {
 972                         t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid);
 973                         if (pi->dev)
 974                                 device_delete_child(dev, pi->dev);
 975
 976                         mtx_destroy(&pi->pi_lock);
 977                         free(pi, M_CXGBE);
 978                 }
 979         }
 980
 981         if (sc->flags & FULL_INIT_DONE)
 982                 adapter_full_uninit(sc);
 983
 984         if (sc->flags & FW_OK)
 985                 t4_fw_bye(sc, sc->mbox);
 986
 987         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
 988                 pci_release_msi(dev);
 989
 990         if (sc->regs_res)
 991                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
 992                     sc->regs_res);
 993
 994         if (sc->udbs_res)
 995                 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
 996                     sc->udbs_res);
 997
 998         if (sc->msix_res)
 999                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
1000                     sc->msix_res);
1001
1002         if (sc->l2t)
1003                 t4_free_l2t(sc->l2t);
1004
1005 #ifdef TCP_OFFLOAD
1006         free(sc->sge.ofld_rxq, M_CXGBE);
1007         free(sc->sge.ofld_txq, M_CXGBE);
1008 #endif
1009 #ifdef DEV_NETMAP
1010         free(sc->sge.nm_rxq, M_CXGBE);
1011         free(sc->sge.nm_txq, M_CXGBE);
1012 #endif
1013         free(sc->irq, M_CXGBE);
1014         free(sc->sge.rxq, M_CXGBE);
1015         free(sc->sge.txq, M_CXGBE);
1016         free(sc->sge.ctrlq, M_CXGBE);
1017         free(sc->sge.iqmap, M_CXGBE);
1018         free(sc->sge.eqmap, M_CXGBE);
1019         free(sc->tids.ftid_tab, M_CXGBE);
1020         t4_destroy_dma_tag(sc);
1021         if (mtx_initialized(&sc->sc_lock)) {
1022                 sx_xlock(&t4_list_lock);
1023                 SLIST_REMOVE(&t4_list, sc, adapter, link);
1024                 sx_xunlock(&t4_list_lock);
1025                 mtx_destroy(&sc->sc_lock);
1026         }
1027
1028         if (mtx_initialized(&sc->tids.ftid_lock))
1029                 mtx_destroy(&sc->tids.ftid_lock);
1030         if (mtx_initialized(&sc->sfl_lock))
1031                 mtx_destroy(&sc->sfl_lock);
1032         if (mtx_initialized(&sc->ifp_lock))
1033                 mtx_destroy(&sc->ifp_lock);
1034         if (mtx_initialized(&sc->regwin_lock))
1035                 mtx_destroy(&sc->regwin_lock);
1036
1037         bzero(sc, sizeof(*sc));
1038
1039         return (0);
1040 }
1041
1042 static int
1043 cxgbe_probe(device_t dev)
1044 {
1045         char buf[128];
1046         struct port_info *pi = device_get_softc(dev);
1047
1048         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
1049         device_set_desc_copy(dev, buf);
1050
1051         return (BUS_PROBE_DEFAULT);
1052 }
1053
1054 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
1055     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1056     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
1057 #define T4_CAP_ENABLE (T4_CAP)
1058
1059 static int
1060 cxgbe_attach(device_t dev)
1061 {
1062         struct port_info *pi = device_get_softc(dev);
1063         struct ifnet *ifp;
1064         char *s;
1065         int n, o;
1066
1067         /* Allocate an ifnet and set it up */
1068         ifp = if_alloc(IFT_ETHER);
1069         if (ifp == NULL) {
1070                 device_printf(dev, "Cannot allocate ifnet\n");
1071                 return (ENOMEM);
1072         }
1073         pi->ifp = ifp;
1074         ifp->if_softc = pi;
1075
1076         callout_init(&pi->tick, CALLOUT_MPSAFE);
1077
1078         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1079         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1080
1081         ifp->if_init = cxgbe_init;
1082         ifp->if_ioctl = cxgbe_ioctl;
1083         ifp->if_transmit = cxgbe_transmit;
1084         ifp->if_qflush = cxgbe_qflush;
1085         ifp->if_get_counter = cxgbe_get_counter;
1086
1087         ifp->if_capabilities = T4_CAP;
1088 #ifdef TCP_OFFLOAD
1089         if (is_offload(pi->adapter))
1090                 ifp->if_capabilities |= IFCAP_TOE;
1091 #endif
1092         ifp->if_capenable = T4_CAP_ENABLE;
1093         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1094             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1095
1096         ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1097         ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS;
1098         ifp->if_hw_tsomaxsegsize = 65536;
1099
1100         /* Initialize ifmedia for this port */
1101         ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1102             cxgbe_media_status);
1103         build_medialist(pi, &pi->media);
1104
1105         pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1106             EVENTHANDLER_PRI_ANY);
1107
1108         ether_ifattach(ifp, pi->hw_addr);
1109
1110         n = 128;
1111         s = malloc(n, M_CXGBE, M_WAITOK);
1112         o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq);
1113         MPASS(n > o);
1114 #ifdef TCP_OFFLOAD
1115         if (is_offload(pi->adapter)) {
1116                 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)",
1117                     pi->nofldtxq, pi->nofldrxq);
1118                 MPASS(n > o);
1119         }
1120 #endif
1121 #ifdef DEV_NETMAP
1122         o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq,
1123             pi->nnmrxq);
1124         MPASS(n > o);
1125 #endif
1126         device_printf(dev, "%s\n", s);
1127         free(s, M_CXGBE);
1128
1129 #ifdef DEV_NETMAP
1130         /* nm_media handled here to keep implementation private to this file */
1131         ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change,
1132             cxgbe_media_status);
1133         build_medialist(pi, &pi->nm_media);
1134         create_netmap_ifnet(pi);        /* logs errors it something fails */
1135 #endif
1136         cxgbe_sysctls(pi);
1137
1138         return (0);
1139 }
1140
1141 static int
1142 cxgbe_detach(device_t dev)
1143 {
1144         struct port_info *pi = device_get_softc(dev);
1145         struct adapter *sc = pi->adapter;
1146         struct ifnet *ifp = pi->ifp;
1147
1148         /* Tell if_ioctl and if_init that the port is going away */
1149         ADAPTER_LOCK(sc);
1150         SET_DOOMED(pi);
1151         wakeup(&sc->flags);
1152         while (IS_BUSY(sc))
1153                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
1154         SET_BUSY(sc);
1155 #ifdef INVARIANTS
1156         sc->last_op = "t4detach";
1157         sc->last_op_thr = curthread;
1158 #endif
1159         ADAPTER_UNLOCK(sc);
1160
1161         if (pi->flags & HAS_TRACEQ) {
1162                 sc->traceq = -1;        /* cloner should not create ifnet */
1163                 t4_tracer_port_detach(sc);
1164         }
1165
1166         if (pi->vlan_c)
1167                 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
1168
1169         PORT_LOCK(pi);
1170         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1171         callout_stop(&pi->tick);
1172         PORT_UNLOCK(pi);
1173         callout_drain(&pi->tick);
1174
1175         /* Let detach proceed even if these fail. */
1176         cxgbe_uninit_synchronized(pi);
1177         port_full_uninit(pi);
1178
1179         ifmedia_removeall(&pi->media);
1180         ether_ifdetach(pi->ifp);
1181         if_free(pi->ifp);
1182
1183 #ifdef DEV_NETMAP
1184         /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */
1185         destroy_netmap_ifnet(pi);
1186 #endif
1187
1188         ADAPTER_LOCK(sc);
1189         CLR_BUSY(sc);
1190         wakeup(&sc->flags);
1191         ADAPTER_UNLOCK(sc);
1192
1193         return (0);
1194 }
1195
1196 static void
1197 cxgbe_init(void *arg)
1198 {
1199         struct port_info *pi = arg;
1200         struct adapter *sc = pi->adapter;
1201
1202         if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
1203                 return;
1204         cxgbe_init_synchronized(pi);
1205         end_synchronized_op(sc, 0);
1206 }
1207
1208 static int
1209 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1210 {
1211         int rc = 0, mtu, flags, can_sleep;
1212         struct port_info *pi = ifp->if_softc;
1213         struct adapter *sc = pi->adapter;
1214         struct ifreq *ifr = (struct ifreq *)data;
1215         uint32_t mask;
1216
1217         switch (cmd) {
1218         case SIOCSIFMTU:
1219                 mtu = ifr->ifr_mtu;
1220                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1221                         return (EINVAL);
1222
1223                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
1224                 if (rc)
1225                         return (rc);
1226                 ifp->if_mtu = mtu;
1227                 if (pi->flags & PORT_INIT_DONE) {
1228                         t4_update_fl_bufsize(ifp);
1229                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1230                                 rc = update_mac_settings(ifp, XGMAC_MTU);
1231                 }
1232                 end_synchronized_op(sc, 0);
1233                 break;
1234
1235         case SIOCSIFFLAGS:
1236                 can_sleep = 0;
1237 redo_sifflags:
1238                 rc = begin_synchronized_op(sc, pi,
1239                     can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg");
1240                 if (rc)
1241                         return (rc);
1242
1243                 if (ifp->if_flags & IFF_UP) {
1244                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1245                                 flags = pi->if_flags;
1246                                 if ((ifp->if_flags ^ flags) &
1247                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1248                                         if (can_sleep == 1) {
1249                                                 end_synchronized_op(sc, 0);
1250                                                 can_sleep = 0;
1251                                                 goto redo_sifflags;
1252                                         }
1253                                         rc = update_mac_settings(ifp,
1254                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1255                                 }
1256                         } else {
1257                                 if (can_sleep == 0) {
1258                                         end_synchronized_op(sc, LOCK_HELD);
1259                                         can_sleep = 1;
1260                                         goto redo_sifflags;
1261                                 }
1262                                 rc = cxgbe_init_synchronized(pi);
1263                         }
1264                         pi->if_flags = ifp->if_flags;
1265                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1266                         if (can_sleep == 0) {
1267                                 end_synchronized_op(sc, LOCK_HELD);
1268                                 can_sleep = 1;
1269                                 goto redo_sifflags;
1270                         }
1271                         rc = cxgbe_uninit_synchronized(pi);
1272                 }
1273                 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD);
1274                 break;
1275
1276         case SIOCADDMULTI:
1277         case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1278                 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
1279                 if (rc)
1280                         return (rc);
1281                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1282                         rc = update_mac_settings(ifp, XGMAC_MCADDRS);
1283                 end_synchronized_op(sc, LOCK_HELD);
1284                 break;
1285
1286         case SIOCSIFCAP:
1287                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
1288                 if (rc)
1289                         return (rc);
1290
1291                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1292                 if (mask & IFCAP_TXCSUM) {
1293                         ifp->if_capenable ^= IFCAP_TXCSUM;
1294                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1295
1296                         if (IFCAP_TSO4 & ifp->if_capenable &&
1297                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1298                                 ifp->if_capenable &= ~IFCAP_TSO4;
1299                                 if_printf(ifp,
1300                                     "tso4 disabled due to -txcsum.\n");
1301                         }
1302                 }
1303                 if (mask & IFCAP_TXCSUM_IPV6) {
1304                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1305                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1306
1307                         if (IFCAP_TSO6 & ifp->if_capenable &&
1308                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1309                                 ifp->if_capenable &= ~IFCAP_TSO6;
1310                                 if_printf(ifp,
1311                                     "tso6 disabled due to -txcsum6.\n");
1312                         }
1313                 }
1314                 if (mask & IFCAP_RXCSUM)
1315                         ifp->if_capenable ^= IFCAP_RXCSUM;
1316                 if (mask & IFCAP_RXCSUM_IPV6)
1317                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1318
1319                 /*
1320                  * Note that we leave CSUM_TSO alone (it is always set).  The
1321                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1322                  * sending a TSO request our way, so it's sufficient to toggle
1323                  * IFCAP_TSOx only.
1324                  */
1325                 if (mask & IFCAP_TSO4) {
1326                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1327                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1328                                 if_printf(ifp, "enable txcsum first.\n");
1329                                 rc = EAGAIN;
1330                                 goto fail;
1331                         }
1332                         ifp->if_capenable ^= IFCAP_TSO4;
1333                 }
1334                 if (mask & IFCAP_TSO6) {
1335                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1336                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1337                                 if_printf(ifp, "enable txcsum6 first.\n");
1338                                 rc = EAGAIN;
1339                                 goto fail;
1340                         }
1341                         ifp->if_capenable ^= IFCAP_TSO6;
1342                 }
1343                 if (mask & IFCAP_LRO) {
1344 #if defined(INET) || defined(INET6)
1345                         int i;
1346                         struct sge_rxq *rxq;
1347
1348                         ifp->if_capenable ^= IFCAP_LRO;
1349                         for_each_rxq(pi, i, rxq) {
1350                                 if (ifp->if_capenable & IFCAP_LRO)
1351                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1352                                 else
1353                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1354                         }
1355 #endif
1356                 }
1357 #ifdef TCP_OFFLOAD
1358                 if (mask & IFCAP_TOE) {
1359                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1360
1361                         rc = toe_capability(pi, enable);
1362                         if (rc != 0)
1363                                 goto fail;
1364
1365                         ifp->if_capenable ^= mask;
1366                 }
1367 #endif
1368                 if (mask & IFCAP_VLAN_HWTAGGING) {
1369                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1370                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1371                                 rc = update_mac_settings(ifp, XGMAC_VLANEX);
1372                 }
1373                 if (mask & IFCAP_VLAN_MTU) {
1374                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1375
1376                         /* Need to find out how to disable auto-mtu-inflation */
1377                 }
1378                 if (mask & IFCAP_VLAN_HWTSO)
1379                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1380                 if (mask & IFCAP_VLAN_HWCSUM)
1381                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1382
1383 #ifdef VLAN_CAPABILITIES
1384                 VLAN_CAPABILITIES(ifp);
1385 #endif
1386 fail:
1387                 end_synchronized_op(sc, 0);
1388                 break;
1389
1390         case SIOCSIFMEDIA:
1391         case SIOCGIFMEDIA:
1392                 ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1393                 break;
1394
1395         case SIOCGI2C: {
1396                 struct ifi2creq i2c;
1397
1398                 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
1399                 if (rc != 0)
1400                         break;
1401                 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
1402                         rc = EPERM;
1403                         break;
1404                 }
1405                 if (i2c.len > sizeof(i2c.data)) {
1406                         rc = EINVAL;
1407                         break;
1408                 }
1409                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4i2c");
1410                 if (rc)
1411                         return (rc);
1412                 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr,
1413                     i2c.offset, i2c.len, &i2c.data[0]);
1414                 end_synchronized_op(sc, 0);
1415                 if (rc == 0)
1416                         rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
1417                 break;
1418         }
1419
1420         default:
1421                 rc = ether_ioctl(ifp, cmd, data);
1422         }
1423
1424         return (rc);
1425 }
1426
1427 static int
1428 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1429 {
1430         struct port_info *pi = ifp->if_softc;
1431         struct adapter *sc = pi->adapter;
1432         struct sge_txq *txq = &sc->sge.txq[pi->first_txq];
1433         struct buf_ring *br;
1434         int rc;
1435
1436         M_ASSERTPKTHDR(m);
1437
1438         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1439                 m_freem(m);
1440                 return (ENETDOWN);
1441         }
1442
1443         /* check if flowid is set */
1444         if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
1445                 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq))
1446                     + pi->rsrv_noflowq);
1447         br = txq->br;
1448
1449         if (TXQ_TRYLOCK(txq) == 0) {
1450                 struct sge_eq *eq = &txq->eq;
1451
1452                 /*
1453                  * It is possible that t4_eth_tx finishes up and releases the
1454                  * lock between the TRYLOCK above and the drbr_enqueue here.  We
1455                  * need to make sure that this mbuf doesn't just sit there in
1456                  * the drbr.
1457                  */
1458
1459                 rc = drbr_enqueue(ifp, br, m);
1460                 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 &&
1461                     !(eq->flags & EQ_DOOMED))
1462                         callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq);
1463                 return (rc);
1464         }
1465
1466         /*
1467          * txq->m is the mbuf that is held up due to a temporary shortage of
1468          * resources and it should be put on the wire first.  Then what's in
1469          * drbr and finally the mbuf that was just passed in to us.
1470          *
1471          * Return code should indicate the fate of the mbuf that was passed in
1472          * this time.
1473          */
1474
1475         TXQ_LOCK_ASSERT_OWNED(txq);
1476         if (drbr_needs_enqueue(ifp, br) || txq->m) {
1477
1478                 /* Queued for transmission. */
1479
1480                 rc = drbr_enqueue(ifp, br, m);
1481                 m = txq->m ? txq->m : drbr_dequeue(ifp, br);
1482                 (void) t4_eth_tx(ifp, txq, m);
1483                 TXQ_UNLOCK(txq);
1484                 return (rc);
1485         }
1486
1487         /* Direct transmission. */
1488         rc = t4_eth_tx(ifp, txq, m);
1489         if (rc != 0 && txq->m)
1490                 rc = 0; /* held, will be transmitted soon (hopefully) */
1491
1492         TXQ_UNLOCK(txq);
1493         return (rc);
1494 }
1495
1496 static void
1497 cxgbe_qflush(struct ifnet *ifp)
1498 {
1499         struct port_info *pi = ifp->if_softc;
1500         struct sge_txq *txq;
1501         int i;
1502         struct mbuf *m;
1503
1504         /* queues do not exist if !PORT_INIT_DONE. */
1505         if (pi->flags & PORT_INIT_DONE) {
1506                 for_each_txq(pi, i, txq) {
1507                         TXQ_LOCK(txq);
1508                         m_freem(txq->m);
1509                         txq->m = NULL;
1510                         while ((m = buf_ring_dequeue_sc(txq->br)) != NULL)
1511                                 m_freem(m);
1512                         TXQ_UNLOCK(txq);
1513                 }
1514         }
1515         if_qflush(ifp);
1516 }
1517
1518 static uint64_t
1519 cxgbe_get_counter(struct ifnet *ifp, ift_counter c)
1520 {
1521         struct port_info *pi = ifp->if_softc;
1522         struct adapter *sc = pi->adapter;
1523         struct port_stats *s = &pi->stats;
1524
1525         cxgbe_refresh_stats(sc, pi);
1526
1527         switch (c) {
1528         case IFCOUNTER_IPACKETS:
1529                 return (s->rx_frames - s->rx_pause);
1530
1531         case IFCOUNTER_IERRORS:
1532                 return (s->rx_jabber + s->rx_runt + s->rx_too_long +
1533                     s->rx_fcs_err + s->rx_len_err);
1534
1535         case IFCOUNTER_OPACKETS:
1536                 return (s->tx_frames - s->tx_pause);
1537
1538         case IFCOUNTER_OERRORS:
1539                 return (s->tx_error_frames);
1540
1541         case IFCOUNTER_IBYTES:
1542                 return (s->rx_octets - s->rx_pause * 64);
1543
1544         case IFCOUNTER_OBYTES:
1545                 return (s->tx_octets - s->tx_pause * 64);
1546
1547         case IFCOUNTER_IMCASTS:
1548                 return (s->rx_mcast_frames - s->rx_pause);
1549
1550         case IFCOUNTER_OMCASTS:
1551                 return (s->tx_mcast_frames - s->tx_pause);
1552
1553         case IFCOUNTER_IQDROPS:
1554                 return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
1555                     s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
1556                     s->rx_trunc3 + pi->tnl_cong_drops);
1557
1558         case IFCOUNTER_OQDROPS: {
1559                 uint64_t drops;
1560
1561                 drops = s->tx_drop;
1562                 if (pi->flags & PORT_INIT_DONE) {
1563                         int i;
1564                         struct sge_txq *txq;
1565
1566                         for_each_txq(pi, i, txq)
1567                                 drops += txq->br->br_drops;
1568                 }
1569
1570                 return (drops);
1571
1572         }
1573
1574         default:
1575                 return (if_get_counter_default(ifp, c));
1576         }
1577 }
1578
1579 static int
1580 cxgbe_media_change(struct ifnet *ifp)
1581 {
1582         struct port_info *pi = ifp->if_softc;
1583
1584         device_printf(pi->dev, "%s unimplemented.\n", __func__);
1585
1586         return (EOPNOTSUPP);
1587 }
1588
1589 static void
1590 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1591 {
1592         struct port_info *pi = ifp->if_softc;
1593         struct ifmedia *media = NULL;
1594         struct ifmedia_entry *cur;
1595         int speed = pi->link_cfg.speed;
1596 #ifdef INVARIANTS
1597         int data = (pi->port_type << 8) | pi->mod_type;
1598 #endif
1599
1600         if (ifp == pi->ifp)
1601                 media = &pi->media;
1602 #ifdef DEV_NETMAP
1603         else if (ifp == pi->nm_ifp)
1604                 media = &pi->nm_media;
1605 #endif
1606         MPASS(media != NULL);
1607
1608         cur = media->ifm_cur;
1609         MPASS(cur->ifm_data == data);
1610
1611         ifmr->ifm_status = IFM_AVALID;
1612         if (!pi->link_cfg.link_ok)
1613                 return;
1614
1615         ifmr->ifm_status |= IFM_ACTIVE;
1616
1617         /* active and current will differ iff current media is autoselect. */
1618         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1619                 return;
1620
1621         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1622         if (speed == SPEED_10000)
1623                 ifmr->ifm_active |= IFM_10G_T;
1624         else if (speed == SPEED_1000)
1625                 ifmr->ifm_active |= IFM_1000_T;
1626         else if (speed == SPEED_100)
1627                 ifmr->ifm_active |= IFM_100_TX;
1628         else if (speed == SPEED_10)
1629                 ifmr->ifm_active |= IFM_10_T;
1630         else
1631                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1632                             speed));
1633 }
1634
1635 void
1636 t4_fatal_err(struct adapter *sc)
1637 {
1638         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1639         t4_intr_disable(sc);
1640         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1641             device_get_nameunit(sc->dev));
1642 }
1643
1644 static int
1645 map_bars_0_and_4(struct adapter *sc)
1646 {
1647         sc->regs_rid = PCIR_BAR(0);
1648         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1649             &sc->regs_rid, RF_ACTIVE);
1650         if (sc->regs_res == NULL) {
1651                 device_printf(sc->dev, "cannot map registers.\n");
1652                 return (ENXIO);
1653         }
1654         sc->bt = rman_get_bustag(sc->regs_res);
1655         sc->bh = rman_get_bushandle(sc->regs_res);
1656         sc->mmio_len = rman_get_size(sc->regs_res);
1657         setbit(&sc->doorbells, DOORBELL_KDB);
1658
1659         sc->msix_rid = PCIR_BAR(4);
1660         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1661             &sc->msix_rid, RF_ACTIVE);
1662         if (sc->msix_res == NULL) {
1663                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1664                 return (ENXIO);
1665         }
1666
1667         return (0);
1668 }
1669
1670 static int
1671 map_bar_2(struct adapter *sc)
1672 {
1673
1674         /*
1675          * T4: only iWARP driver uses the userspace doorbells.  There is no need
1676          * to map it if RDMA is disabled.
1677          */
1678         if (is_t4(sc) && sc->rdmacaps == 0)
1679                 return (0);
1680
1681         sc->udbs_rid = PCIR_BAR(2);
1682         sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1683             &sc->udbs_rid, RF_ACTIVE);
1684         if (sc->udbs_res == NULL) {
1685                 device_printf(sc->dev, "cannot map doorbell BAR.\n");
1686                 return (ENXIO);
1687         }
1688         sc->udbs_base = rman_get_virtual(sc->udbs_res);
1689
1690         if (is_t5(sc)) {
1691                 setbit(&sc->doorbells, DOORBELL_UDB);
1692 #if defined(__i386__) || defined(__amd64__)
1693                 if (t5_write_combine) {
1694                         int rc;
1695
1696                         /*
1697                          * Enable write combining on BAR2.  This is the
1698                          * userspace doorbell BAR and is split into 128B
1699                          * (UDBS_SEG_SIZE) doorbell regions, each associated
1700                          * with an egress queue.  The first 64B has the doorbell
1701                          * and the second 64B can be used to submit a tx work
1702                          * request with an implicit doorbell.
1703                          */
1704
1705                         rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
1706                             rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
1707                         if (rc == 0) {
1708                                 clrbit(&sc->doorbells, DOORBELL_UDB);
1709                                 setbit(&sc->doorbells, DOORBELL_WCWR);
1710                                 setbit(&sc->doorbells, DOORBELL_UDBWC);
1711                         } else {
1712                                 device_printf(sc->dev,
1713                                     "couldn't enable write combining: %d\n",
1714                                     rc);
1715                         }
1716
1717                         t4_write_reg(sc, A_SGE_STAT_CFG,
1718                             V_STATSOURCE_T5(7) | V_STATMODE(0));
1719                 }
1720 #endif
1721         }
1722
1723         return (0);
1724 }
1725
1726 static const struct memwin t4_memwin[] = {
1727         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1728         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1729         { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
1730 };
1731
1732 static const struct memwin t5_memwin[] = {
1733         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1734         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1735         { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1736 };
1737
1738 static void
1739 setup_memwin(struct adapter *sc)
1740 {
1741         const struct memwin *mw;
1742         int i, n;
1743         uint32_t bar0;
1744
1745         if (is_t4(sc)) {
1746                 /*
1747                  * Read low 32b of bar0 indirectly via the hardware backdoor
1748                  * mechanism.  Works from within PCI passthrough environments
1749                  * too, where rman_get_start() can return a different value.  We
1750                  * need to program the T4 memory window decoders with the actual
1751                  * addresses that will be coming across the PCIe link.
1752                  */
1753                 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1754                 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1755
1756                 mw = &t4_memwin[0];
1757                 n = nitems(t4_memwin);
1758         } else {
1759                 /* T5 uses the relative offset inside the PCIe BAR */
1760                 bar0 = 0;
1761
1762                 mw = &t5_memwin[0];
1763                 n = nitems(t5_memwin);
1764         }
1765
1766         for (i = 0; i < n; i++, mw++) {
1767                 t4_write_reg(sc,
1768                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
1769                     (mw->base + bar0) | V_BIR(0) |
1770                     V_WINDOW(ilog2(mw->aperture) - 10));
1771         }
1772
1773         /* flush */
1774         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1775 }
1776
1777 /*
1778  * Verify that the memory range specified by the addr/len pair is valid and lies
1779  * entirely within a single region (EDCx or MCx).
1780  */
1781 static int
1782 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
1783 {
1784         uint32_t em, addr_len, maddr, mlen;
1785
1786         /* Memory can only be accessed in naturally aligned 4 byte units */
1787         if (addr & 3 || len & 3 || len == 0)
1788                 return (EINVAL);
1789
1790         /* Enabled memories */
1791         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1792         if (em & F_EDRAM0_ENABLE) {
1793                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1794                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1795                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1796                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1797                     addr + len <= maddr + mlen)
1798                         return (0);
1799         }
1800         if (em & F_EDRAM1_ENABLE) {
1801                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1802                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1803                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1804                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1805                     addr + len <= maddr + mlen)
1806                         return (0);
1807         }
1808         if (em & F_EXT_MEM_ENABLE) {
1809                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1810                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1811                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1812                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1813                     addr + len <= maddr + mlen)
1814                         return (0);
1815         }
1816         if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
1817                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1818                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1819                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1820                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1821                     addr + len <= maddr + mlen)
1822                         return (0);
1823         }
1824
1825         return (EFAULT);
1826 }
1827
1828 static int
1829 fwmtype_to_hwmtype(int mtype)
1830 {
1831
1832         switch (mtype) {
1833         case FW_MEMTYPE_EDC0:
1834                 return (MEM_EDC0);
1835         case FW_MEMTYPE_EDC1:
1836                 return (MEM_EDC1);
1837         case FW_MEMTYPE_EXTMEM:
1838                 return (MEM_MC0);
1839         case FW_MEMTYPE_EXTMEM1:
1840                 return (MEM_MC1);
1841         default:
1842                 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
1843         }
1844 }
1845
1846 /*
1847  * Verify that the memory range specified by the memtype/offset/len pair is
1848  * valid and lies entirely within the memtype specified.  The global address of
1849  * the start of the range is returned in addr.
1850  */
1851 static int
1852 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1853     uint32_t *addr)
1854 {
1855         uint32_t em, addr_len, maddr, mlen;
1856
1857         /* Memory can only be accessed in naturally aligned 4 byte units */
1858         if (off & 3 || len & 3 || len == 0)
1859                 return (EINVAL);
1860
1861         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1862         switch (fwmtype_to_hwmtype(mtype)) {
1863         case MEM_EDC0:
1864                 if (!(em & F_EDRAM0_ENABLE))
1865                         return (EINVAL);
1866                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1867                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1868                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1869                 break;
1870         case MEM_EDC1:
1871                 if (!(em & F_EDRAM1_ENABLE))
1872                         return (EINVAL);
1873                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1874                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1875                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1876                 break;
1877         case MEM_MC:
1878                 if (!(em & F_EXT_MEM_ENABLE))
1879                         return (EINVAL);
1880                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1881                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1882                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1883                 break;
1884         case MEM_MC1:
1885                 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
1886                         return (EINVAL);
1887                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1888                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1889                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1890                 break;
1891         default:
1892                 return (EINVAL);
1893         }
1894
1895         if (mlen > 0 && off < mlen && off + len <= mlen) {
1896                 *addr = maddr + off;    /* global address */
1897                 return (0);
1898         }
1899
1900         return (EFAULT);
1901 }
1902
1903 static void
1904 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1905 {
1906         const struct memwin *mw;
1907
1908         if (is_t4(sc)) {
1909                 KASSERT(win >= 0 && win < nitems(t4_memwin),
1910                     ("%s: incorrect memwin# (%d)", __func__, win));
1911                 mw = &t4_memwin[win];
1912         } else {
1913                 KASSERT(win >= 0 && win < nitems(t5_memwin),
1914                     ("%s: incorrect memwin# (%d)", __func__, win));
1915                 mw = &t5_memwin[win];
1916         }
1917
1918         if (base != NULL)
1919                 *base = mw->base;
1920         if (aperture != NULL)
1921                 *aperture = mw->aperture;
1922 }
1923
1924 /*
1925  * Positions the memory window such that it can be used to access the specified
1926  * address in the chip's address space.  The return value is the offset of addr
1927  * from the start of the window.
1928  */
1929 static uint32_t
1930 position_memwin(struct adapter *sc, int n, uint32_t addr)
1931 {
1932         uint32_t start, pf;
1933         uint32_t reg;
1934
1935         KASSERT(n >= 0 && n <= 3,
1936             ("%s: invalid window %d.", __func__, n));
1937         KASSERT((addr & 3) == 0,
1938             ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
1939
1940         if (is_t4(sc)) {
1941                 pf = 0;
1942                 start = addr & ~0xf;    /* start must be 16B aligned */
1943         } else {
1944                 pf = V_PFNUM(sc->pf);
1945                 start = addr & ~0x7f;   /* start must be 128B aligned */
1946         }
1947         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1948
1949         t4_write_reg(sc, reg, start | pf);
1950         t4_read_reg(sc, reg);
1951
1952         return (addr - start);
1953 }
1954
1955 static int
1956 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1957     struct intrs_and_queues *iaq)
1958 {
1959         int rc, itype, navail, nrxq10g, nrxq1g, n;
1960         int nofldrxq10g = 0, nofldrxq1g = 0;
1961         int nnmrxq10g = 0, nnmrxq1g = 0;
1962
1963         bzero(iaq, sizeof(*iaq));
1964
1965         iaq->ntxq10g = t4_ntxq10g;
1966         iaq->ntxq1g = t4_ntxq1g;
1967         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1968         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1969         iaq->rsrv_noflowq = t4_rsrv_noflowq;
1970 #ifdef TCP_OFFLOAD
1971         if (is_offload(sc)) {
1972                 iaq->nofldtxq10g = t4_nofldtxq10g;
1973                 iaq->nofldtxq1g = t4_nofldtxq1g;
1974                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1975                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1976         }
1977 #endif
1978 #ifdef DEV_NETMAP
1979         iaq->nnmtxq10g = t4_nnmtxq10g;
1980         iaq->nnmtxq1g = t4_nnmtxq1g;
1981         iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g;
1982         iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g;
1983 #endif
1984
1985         for (itype = INTR_MSIX; itype; itype >>= 1) {
1986
1987                 if ((itype & t4_intr_types) == 0)
1988                         continue;       /* not allowed */
1989
1990                 if (itype == INTR_MSIX)
1991                         navail = pci_msix_count(sc->dev);
1992                 else if (itype == INTR_MSI)
1993                         navail = pci_msi_count(sc->dev);
1994                 else
1995                         navail = 1;
1996 restart:
1997                 if (navail == 0)
1998                         continue;
1999
2000                 iaq->intr_type = itype;
2001                 iaq->intr_flags_10g = 0;
2002                 iaq->intr_flags_1g = 0;
2003
2004                 /*
2005                  * Best option: an interrupt vector for errors, one for the
2006                  * firmware event queue, and one for every rxq (NIC, TOE, and
2007                  * netmap).
2008                  */
2009                 iaq->nirq = T4_EXTRA_INTR;
2010                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g);
2011                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g);
2012                 if (iaq->nirq <= navail &&
2013                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
2014                         iaq->intr_flags_10g = INTR_ALL;
2015                         iaq->intr_flags_1g = INTR_ALL;
2016                         goto allocate;
2017                 }
2018
2019                 /*
2020                  * Second best option: a vector for errors, one for the firmware
2021                  * event queue, and vectors for either all the NIC rx queues or
2022                  * all the TOE rx queues.  The queues that don't get vectors
2023                  * will forward their interrupts to those that do.
2024                  *
2025                  * Note: netmap rx queues cannot be created early and so they
2026                  * can't be setup to receive forwarded interrupts for others.
2027                  */
2028                 iaq->nirq = T4_EXTRA_INTR;
2029                 if (nrxq10g >= nofldrxq10g) {
2030                         iaq->intr_flags_10g = INTR_RXQ;
2031                         iaq->nirq += n10g * nrxq10g;
2032 #ifdef DEV_NETMAP
2033                         iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g);
2034 #endif
2035                 } else {
2036                         iaq->intr_flags_10g = INTR_OFLD_RXQ;
2037                         iaq->nirq += n10g * nofldrxq10g;
2038 #ifdef DEV_NETMAP
2039                         iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g);
2040 #endif
2041                 }
2042                 if (nrxq1g >= nofldrxq1g) {
2043                         iaq->intr_flags_1g = INTR_RXQ;
2044                         iaq->nirq += n1g * nrxq1g;
2045 #ifdef DEV_NETMAP
2046                         iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g);
2047 #endif
2048                 } else {
2049                         iaq->intr_flags_1g = INTR_OFLD_RXQ;
2050                         iaq->nirq += n1g * nofldrxq1g;
2051 #ifdef DEV_NETMAP
2052                         iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g);
2053 #endif
2054                 }
2055                 if (iaq->nirq <= navail &&
2056                     (itype != INTR_MSI || powerof2(iaq->nirq)))
2057                         goto allocate;
2058
2059                 /*
2060                  * Next best option: an interrupt vector for errors, one for the
2061                  * firmware event queue, and at least one per port.  At this
2062                  * point we know we'll have to downsize nrxq and/or nofldrxq
2063                  * and/or nnmrxq to fit what's available to us.
2064                  */
2065                 iaq->nirq = T4_EXTRA_INTR;
2066                 iaq->nirq += n10g + n1g;
2067                 if (iaq->nirq <= navail) {
2068                         int leftover = navail - iaq->nirq;
2069
2070                         if (n10g > 0) {
2071                                 int target = max(nrxq10g, nofldrxq10g);
2072
2073                                 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
2074                                     INTR_RXQ : INTR_OFLD_RXQ;
2075
2076                                 n = 1;
2077                                 while (n < target && leftover >= n10g) {
2078                                         leftover -= n10g;
2079                                         iaq->nirq += n10g;
2080                                         n++;
2081                                 }
2082                                 iaq->nrxq10g = min(n, nrxq10g);
2083 #ifdef TCP_OFFLOAD
2084                                 iaq->nofldrxq10g = min(n, nofldrxq10g);
2085 #endif
2086 #ifdef DEV_NETMAP
2087                                 iaq->nnmrxq10g = min(n, nnmrxq10g);
2088 #endif
2089                         }
2090
2091                         if (n1g > 0) {
2092                                 int target = max(nrxq1g, nofldrxq1g);
2093
2094                                 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
2095                                     INTR_RXQ : INTR_OFLD_RXQ;
2096
2097                                 n = 1;
2098                                 while (n < target && leftover >= n1g) {
2099                                         leftover -= n1g;
2100                                         iaq->nirq += n1g;
2101                                         n++;
2102                                 }
2103                                 iaq->nrxq1g = min(n, nrxq1g);
2104 #ifdef TCP_OFFLOAD
2105                                 iaq->nofldrxq1g = min(n, nofldrxq1g);
2106 #endif
2107 #ifdef DEV_NETMAP
2108                                 iaq->nnmrxq1g = min(n, nnmrxq1g);
2109 #endif
2110                         }
2111
2112                         if (itype != INTR_MSI || powerof2(iaq->nirq))
2113                                 goto allocate;
2114                 }
2115
2116                 /*
2117                  * Least desirable option: one interrupt vector for everything.
2118                  */
2119                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
2120                 iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
2121 #ifdef TCP_OFFLOAD
2122                 if (is_offload(sc))
2123                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2124 #endif
2125 #ifdef DEV_NETMAP
2126                 iaq->nnmrxq10g = iaq->nnmrxq1g = 1;
2127 #endif
2128
2129 allocate:
2130                 navail = iaq->nirq;
2131                 rc = 0;
2132                 if (itype == INTR_MSIX)
2133                         rc = pci_alloc_msix(sc->dev, &navail);
2134                 else if (itype == INTR_MSI)
2135                         rc = pci_alloc_msi(sc->dev, &navail);
2136
2137                 if (rc == 0) {
2138                         if (navail == iaq->nirq)
2139                                 return (0);
2140
2141                         /*
2142                          * Didn't get the number requested.  Use whatever number
2143                          * the kernel is willing to allocate (it's in navail).
2144                          */
2145                         device_printf(sc->dev, "fewer vectors than requested, "
2146                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
2147                             itype, iaq->nirq, navail);
2148                         pci_release_msi(sc->dev);
2149                         goto restart;
2150                 }
2151
2152                 device_printf(sc->dev,
2153                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
2154                     itype, rc, iaq->nirq, navail);
2155         }
2156
2157         device_printf(sc->dev,
2158             "failed to find a usable interrupt type.  "
2159             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
2160             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
2161
2162         return (ENXIO);
2163 }
2164
2165 #define FW_VERSION(chip) ( \
2166     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
2167     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
2168     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
2169     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
2170 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
2171
2172 struct fw_info {
2173         uint8_t chip;
2174         char *kld_name;
2175         char *fw_mod_name;
2176         struct fw_hdr fw_hdr;   /* XXX: waste of space, need a sparse struct */
2177 } fw_info[] = {
2178         {
2179                 .chip = CHELSIO_T4,
2180                 .kld_name = "t4fw_cfg",
2181                 .fw_mod_name = "t4fw",
2182                 .fw_hdr = {
2183                         .chip = FW_HDR_CHIP_T4,
2184                         .fw_ver = htobe32_const(FW_VERSION(T4)),
2185                         .intfver_nic = FW_INTFVER(T4, NIC),
2186                         .intfver_vnic = FW_INTFVER(T4, VNIC),
2187                         .intfver_ofld = FW_INTFVER(T4, OFLD),
2188                         .intfver_ri = FW_INTFVER(T4, RI),
2189                         .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
2190                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
2191                         .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
2192                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
2193                 },
2194         }, {
2195                 .chip = CHELSIO_T5,
2196                 .kld_name = "t5fw_cfg",
2197                 .fw_mod_name = "t5fw",
2198                 .fw_hdr = {
2199                         .chip = FW_HDR_CHIP_T5,
2200                         .fw_ver = htobe32_const(FW_VERSION(T5)),
2201                         .intfver_nic = FW_INTFVER(T5, NIC),
2202                         .intfver_vnic = FW_INTFVER(T5, VNIC),
2203                         .intfver_ofld = FW_INTFVER(T5, OFLD),
2204                         .intfver_ri = FW_INTFVER(T5, RI),
2205                         .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
2206                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
2207                         .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
2208                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
2209                 },
2210         }
2211 };
2212
2213 static struct fw_info *
2214 find_fw_info(int chip)
2215 {
2216         int i;
2217
2218         for (i = 0; i < nitems(fw_info); i++) {
2219                 if (fw_info[i].chip == chip)
2220                         return (&fw_info[i]);
2221         }
2222         return (NULL);
2223 }
2224
2225 /*
2226  * Is the given firmware API compatible with the one the driver was compiled
2227  * with?
2228  */
2229 static int
2230 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
2231 {
2232
2233         /* short circuit if it's the exact same firmware version */
2234         if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
2235                 return (1);
2236
2237         /*
2238          * XXX: Is this too conservative?  Perhaps I should limit this to the
2239          * features that are supported in the driver.
2240          */
2241 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
2242         if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
2243             SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
2244             SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
2245                 return (1);
2246 #undef SAME_INTF
2247
2248         return (0);
2249 }
2250
2251 /*
2252  * The firmware in the KLD is usable, but should it be installed?  This routine
2253  * explains itself in detail if it indicates the KLD firmware should be
2254  * installed.
2255  */
2256 static int
2257 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
2258 {
2259         const char *reason;
2260
2261         if (!card_fw_usable) {
2262                 reason = "incompatible or unusable";
2263                 goto install;
2264         }
2265
2266         if (k > c) {
2267                 reason = "older than the version bundled with this driver";
2268                 goto install;
2269         }
2270
2271         if (t4_fw_install == 2 && k != c) {
2272                 reason = "different than the version bundled with this driver";
2273                 goto install;
2274         }
2275
2276         return (0);
2277
2278 install:
2279         if (t4_fw_install == 0) {
2280                 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2281                     "but the driver is prohibited from installing a different "
2282                     "firmware on the card.\n",
2283                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2284                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2285
2286                 return (0);
2287         }
2288
2289         device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2290             "installing firmware %u.%u.%u.%u on card.\n",
2291             G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2292             G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2293             G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2294             G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2295
2296         return (1);
2297 }
2298 /*
2299  * Establish contact with the firmware and determine if we are the master driver
2300  * or not, and whether we are responsible for chip initialization.
2301  */
2302 static int
2303 prep_firmware(struct adapter *sc)
2304 {
2305         const struct firmware *fw = NULL, *default_cfg;
2306         int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2307         enum dev_state state;
2308         struct fw_info *fw_info;
2309         struct fw_hdr *card_fw;         /* fw on the card */
2310         const struct fw_hdr *kld_fw;    /* fw in the KLD */
2311         const struct fw_hdr *drv_fw;    /* fw header the driver was compiled
2312                                            against */
2313
2314         /* Contact firmware. */
2315         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2316         if (rc < 0 || state == DEV_STATE_ERR) {
2317                 rc = -rc;
2318                 device_printf(sc->dev,
2319                     "failed to connect to the firmware: %d, %d.\n", rc, state);
2320                 return (rc);
2321         }
2322         pf = rc;
2323         if (pf == sc->mbox)
2324                 sc->flags |= MASTER_PF;
2325         else if (state == DEV_STATE_UNINIT) {
2326                 /*
2327                  * We didn't get to be the master so we definitely won't be
2328                  * configuring the chip.  It's a bug if someone else hasn't
2329                  * configured it already.
2330                  */
2331                 device_printf(sc->dev, "couldn't be master(%d), "
2332                     "device not already initialized either(%d).\n", rc, state);
2333                 return (EDOOFUS);
2334         }
2335
2336         /* This is the firmware whose headers the driver was compiled against */
2337         fw_info = find_fw_info(chip_id(sc));
2338         if (fw_info == NULL) {
2339                 device_printf(sc->dev,
2340                     "unable to look up firmware information for chip %d.\n",
2341                     chip_id(sc));
2342                 return (EINVAL);
2343         }
2344         drv_fw = &fw_info->fw_hdr;
2345
2346         /*
2347          * The firmware KLD contains many modules.  The KLD name is also the
2348          * name of the module that contains the default config file.
2349          */
2350         default_cfg = firmware_get(fw_info->kld_name);
2351
2352         /* Read the header of the firmware on the card */
2353         card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2354         rc = -t4_read_flash(sc, FLASH_FW_START,
2355             sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2356         if (rc == 0)
2357                 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2358         else {
2359                 device_printf(sc->dev,
2360                     "Unable to read card's firmware header: %d\n", rc);
2361                 card_fw_usable = 0;
2362         }
2363
2364         /* This is the firmware in the KLD */
2365         fw = firmware_get(fw_info->fw_mod_name);
2366         if (fw != NULL) {
2367                 kld_fw = (const void *)fw->data;
2368                 kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2369         } else {
2370                 kld_fw = NULL;
2371                 kld_fw_usable = 0;
2372         }
2373
2374         if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2375             (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2376                 /*
2377                  * Common case: the firmware on the card is an exact match and
2378                  * the KLD is an exact match too, or the KLD is
2379                  * absent/incompatible.  Note that t4_fw_install = 2 is ignored
2380                  * here -- use cxgbetool loadfw if you want to reinstall the
2381                  * same firmware as the one on the card.
2382                  */
2383         } else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2384             should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2385             be32toh(card_fw->fw_ver))) {
2386
2387                 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
2388                 if (rc != 0) {
2389                         device_printf(sc->dev,
2390                             "failed to install firmware: %d\n", rc);
2391                         goto done;
2392                 }
2393
2394                 /* Installed successfully, update the cached header too. */
2395                 memcpy(card_fw, kld_fw, sizeof(*card_fw));
2396                 card_fw_usable = 1;
2397                 need_fw_reset = 0;      /* already reset as part of load_fw */
2398         }
2399
2400         if (!card_fw_usable) {
2401                 uint32_t d, c, k;
2402
2403                 d = ntohl(drv_fw->fw_ver);
2404                 c = ntohl(card_fw->fw_ver);
2405                 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
2406
2407                 device_printf(sc->dev, "Cannot find a usable firmware: "
2408                     "fw_install %d, chip state %d, "
2409                     "driver compiled with %d.%d.%d.%d, "
2410                     "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
2411                     t4_fw_install, state,
2412                     G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
2413                     G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
2414                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2415                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
2416                     G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2417                     G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2418                 rc = EINVAL;
2419                 goto done;
2420         }
2421
2422         /* We're using whatever's on the card and it's known to be good. */
2423         sc->params.fw_vers = ntohl(card_fw->fw_ver);
2424         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
2425             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2426             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2427             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2428             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2429         t4_get_tp_version(sc, &sc->params.tp_vers);
2430
2431         /* Reset device */
2432         if (need_fw_reset &&
2433             (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
2434                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
2435                 if (rc != ETIMEDOUT && rc != EIO)
2436                         t4_fw_bye(sc, sc->mbox);
2437                 goto done;
2438         }
2439         sc->flags |= FW_OK;
2440
2441         rc = get_params__pre_init(sc);
2442         if (rc != 0)
2443                 goto done; /* error message displayed already */
2444
2445         /* Partition adapter resources as specified in the config file. */
2446         if (state == DEV_STATE_UNINIT) {
2447
2448                 KASSERT(sc->flags & MASTER_PF,
2449                     ("%s: trying to change chip settings when not master.",
2450                     __func__));
2451
2452                 rc = partition_resources(sc, default_cfg, fw_info->kld_name);
2453                 if (rc != 0)
2454                         goto done;      /* error message displayed already */
2455
2456                 t4_tweak_chip_settings(sc);
2457
2458                 /* get basic stuff going */
2459                 rc = -t4_fw_initialize(sc, sc->mbox);
2460                 if (rc != 0) {
2461                         device_printf(sc->dev, "fw init failed: %d.\n", rc);
2462                         goto done;
2463                 }
2464         } else {
2465                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
2466                 sc->cfcsum = 0;
2467         }
2468
2469 done:
2470         free(card_fw, M_CXGBE);
2471         if (fw != NULL)
2472                 firmware_put(fw, FIRMWARE_UNLOAD);
2473         if (default_cfg != NULL)
2474                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
2475
2476         return (rc);
2477 }
2478
2479 #define FW_PARAM_DEV(param) \
2480         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2481          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2482 #define FW_PARAM_PFVF(param) \
2483         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2484          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2485
2486 /*
2487  * Partition chip resources for use between various PFs, VFs, etc.
2488  */
2489 static int
2490 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
2491     const char *name_prefix)
2492 {
2493         const struct firmware *cfg = NULL;
2494         int rc = 0;
2495         struct fw_caps_config_cmd caps;
2496         uint32_t mtype, moff, finicsum, cfcsum;
2497
2498         /*
2499          * Figure out what configuration file to use.  Pick the default config
2500          * file for the card if the user hasn't specified one explicitly.
2501          */
2502         snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
2503         if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
2504                 /* Card specific overrides go here. */
2505                 if (pci_get_device(sc->dev) == 0x440a)
2506                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
2507                 if (is_fpga(sc))
2508                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
2509         }
2510
2511         /*
2512          * We need to load another module if the profile is anything except
2513          * "default" or "flash".
2514          */
2515         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
2516             strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2517                 char s[32];
2518
2519                 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
2520                 cfg = firmware_get(s);
2521                 if (cfg == NULL) {
2522                         if (default_cfg != NULL) {
2523                                 device_printf(sc->dev,
2524                                     "unable to load module \"%s\" for "
2525                                     "configuration profile \"%s\", will use "
2526                                     "the default config file instead.\n",
2527                                     s, sc->cfg_file);
2528                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2529                                     "%s", DEFAULT_CF);
2530                         } else {
2531                                 device_printf(sc->dev,
2532                                     "unable to load module \"%s\" for "
2533                                     "configuration profile \"%s\", will use "
2534                                     "the config file on the card's flash "
2535                                     "instead.\n", s, sc->cfg_file);
2536                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2537                                     "%s", FLASH_CF);
2538                         }
2539                 }
2540         }
2541
2542         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
2543             default_cfg == NULL) {
2544                 device_printf(sc->dev,
2545                     "default config file not available, will use the config "
2546                     "file on the card's flash instead.\n");
2547                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
2548         }
2549
2550         if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2551                 u_int cflen, i, n;
2552                 const uint32_t *cfdata;
2553                 uint32_t param, val, addr, off, mw_base, mw_aperture;
2554
2555                 KASSERT(cfg != NULL || default_cfg != NULL,
2556                     ("%s: no config to upload", __func__));
2557
2558                 /*
2559                  * Ask the firmware where it wants us to upload the config file.
2560                  */
2561                 param = FW_PARAM_DEV(CF);
2562                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2563                 if (rc != 0) {
2564                         /* No support for config file?  Shouldn't happen. */
2565                         device_printf(sc->dev,
2566                             "failed to query config file location: %d.\n", rc);
2567                         goto done;
2568                 }
2569                 mtype = G_FW_PARAMS_PARAM_Y(val);
2570                 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
2571
2572                 /*
2573                  * XXX: sheer laziness.  We deliberately added 4 bytes of
2574                  * useless stuffing/comments at the end of the config file so
2575                  * it's ok to simply throw away the last remaining bytes when
2576                  * the config file is not an exact multiple of 4.  This also
2577                  * helps with the validate_mt_off_len check.
2578                  */
2579                 if (cfg != NULL) {
2580                         cflen = cfg->datasize & ~3;
2581                         cfdata = cfg->data;
2582                 } else {
2583                         cflen = default_cfg->datasize & ~3;
2584                         cfdata = default_cfg->data;
2585                 }
2586
2587                 if (cflen > FLASH_CFG_MAX_SIZE) {
2588                         device_printf(sc->dev,
2589                             "config file too long (%d, max allowed is %d).  "
2590                             "Will try to use the config on the card, if any.\n",
2591                             cflen, FLASH_CFG_MAX_SIZE);
2592                         goto use_config_on_flash;
2593                 }
2594
2595                 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
2596                 if (rc != 0) {
2597                         device_printf(sc->dev,
2598                             "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
2599                             "Will try to use the config on the card, if any.\n",
2600                             __func__, mtype, moff, cflen, rc);
2601                         goto use_config_on_flash;
2602                 }
2603
2604                 memwin_info(sc, 2, &mw_base, &mw_aperture);
2605                 while (cflen) {
2606                         off = position_memwin(sc, 2, addr);
2607                         n = min(cflen, mw_aperture - off);
2608                         for (i = 0; i < n; i += 4)
2609                                 t4_write_reg(sc, mw_base + off + i, *cfdata++);
2610                         cflen -= n;
2611                         addr += n;
2612                 }
2613         } else {
2614 use_config_on_flash:
2615                 mtype = FW_MEMTYPE_FLASH;
2616                 moff = t4_flash_cfg_addr(sc);
2617         }
2618
2619         bzero(&caps, sizeof(caps));
2620         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2621             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2622         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
2623             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
2624             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
2625         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2626         if (rc != 0) {
2627                 device_printf(sc->dev,
2628                     "failed to pre-process config file: %d "
2629                     "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
2630                 goto done;
2631         }
2632
2633         finicsum = be32toh(caps.finicsum);
2634         cfcsum = be32toh(caps.cfcsum);
2635         if (finicsum != cfcsum) {
2636                 device_printf(sc->dev,
2637                     "WARNING: config file checksum mismatch: %08x %08x\n",
2638                     finicsum, cfcsum);
2639         }
2640         sc->cfcsum = cfcsum;
2641
2642 #define LIMIT_CAPS(x) do { \
2643         caps.x &= htobe16(t4_##x##_allowed); \
2644 } while (0)
2645
2646         /*
2647          * Let the firmware know what features will (not) be used so it can tune
2648          * things accordingly.
2649          */
2650         LIMIT_CAPS(linkcaps);
2651         LIMIT_CAPS(niccaps);
2652         LIMIT_CAPS(toecaps);
2653         LIMIT_CAPS(rdmacaps);
2654         LIMIT_CAPS(iscsicaps);
2655         LIMIT_CAPS(fcoecaps);
2656 #undef LIMIT_CAPS
2657
2658         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2659             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
2660         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2661         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
2662         if (rc != 0) {
2663                 device_printf(sc->dev,
2664                     "failed to process config file: %d.\n", rc);
2665         }
2666 done:
2667         if (cfg != NULL)
2668                 firmware_put(cfg, FIRMWARE_UNLOAD);
2669         return (rc);
2670 }
2671
2672 /*
2673  * Retrieve parameters that are needed (or nice to have) very early.
2674  */
2675 static int
2676 get_params__pre_init(struct adapter *sc)
2677 {
2678         int rc;
2679         uint32_t param[2], val[2];
2680         struct fw_devlog_cmd cmd;
2681         struct devlog_params *dlog = &sc->params.devlog;
2682
2683         param[0] = FW_PARAM_DEV(PORTVEC);
2684         param[1] = FW_PARAM_DEV(CCLK);
2685         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2686         if (rc != 0) {
2687                 device_printf(sc->dev,
2688                     "failed to query parameters (pre_init): %d.\n", rc);
2689                 return (rc);
2690         }
2691
2692         sc->params.portvec = val[0];
2693         sc->params.nports = bitcount32(val[0]);
2694         sc->params.vpd.cclk = val[1];
2695
2696         /* Read device log parameters. */
2697         bzero(&cmd, sizeof(cmd));
2698         cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
2699             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2700         cmd.retval_len16 = htobe32(FW_LEN16(cmd));
2701         rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
2702         if (rc != 0) {
2703                 device_printf(sc->dev,
2704                     "failed to get devlog parameters: %d.\n", rc);
2705                 bzero(dlog, sizeof (*dlog));
2706                 rc = 0; /* devlog isn't critical for device operation */
2707         } else {
2708                 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
2709                 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
2710                 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
2711                 dlog->size = be32toh(cmd.memsize_devlog);
2712         }
2713
2714         return (rc);
2715 }
2716
2717 /*
2718  * Retrieve various parameters that are of interest to the driver.  The device
2719  * has been initialized by the firmware at this point.
2720  */
2721 static int
2722 get_params__post_init(struct adapter *sc)
2723 {
2724         int rc;
2725         uint32_t param[7], val[7];
2726         struct fw_caps_config_cmd caps;
2727
2728         param[0] = FW_PARAM_PFVF(IQFLINT_START);
2729         param[1] = FW_PARAM_PFVF(EQ_START);
2730         param[2] = FW_PARAM_PFVF(FILTER_START);
2731         param[3] = FW_PARAM_PFVF(FILTER_END);
2732         param[4] = FW_PARAM_PFVF(L2T_START);
2733         param[5] = FW_PARAM_PFVF(L2T_END);
2734         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2735         if (rc != 0) {
2736                 device_printf(sc->dev,
2737                     "failed to query parameters (post_init): %d.\n", rc);
2738                 return (rc);
2739         }
2740
2741         sc->sge.iq_start = val[0];
2742         sc->sge.eq_start = val[1];
2743         sc->tids.ftid_base = val[2];
2744         sc->tids.nftids = val[3] - val[2] + 1;
2745         sc->params.ftid_min = val[2];
2746         sc->params.ftid_max = val[3];
2747         sc->vres.l2t.start = val[4];
2748         sc->vres.l2t.size = val[5] - val[4] + 1;
2749         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
2750             ("%s: L2 table size (%u) larger than expected (%u)",
2751             __func__, sc->vres.l2t.size, L2T_SIZE));
2752
2753         /* get capabilites */
2754         bzero(&caps, sizeof(caps));
2755         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2756             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2757         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2758         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2759         if (rc != 0) {
2760                 device_printf(sc->dev,
2761                     "failed to get card capabilities: %d.\n", rc);
2762                 return (rc);
2763         }
2764
2765 #define READ_CAPS(x) do { \
2766         sc->x = htobe16(caps.x); \
2767 } while (0)
2768         READ_CAPS(linkcaps);
2769         READ_CAPS(niccaps);
2770         READ_CAPS(toecaps);
2771         READ_CAPS(rdmacaps);
2772         READ_CAPS(iscsicaps);
2773         READ_CAPS(fcoecaps);
2774
2775         if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
2776                 param[0] = FW_PARAM_PFVF(ETHOFLD_START);
2777                 param[1] = FW_PARAM_PFVF(ETHOFLD_END);
2778                 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2779                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
2780                 if (rc != 0) {
2781                         device_printf(sc->dev,
2782                             "failed to query NIC parameters: %d.\n", rc);
2783                         return (rc);
2784                 }
2785                 sc->tids.etid_base = val[0];
2786                 sc->params.etid_min = val[0];
2787                 sc->tids.netids = val[1] - val[0] + 1;
2788                 sc->params.netids = sc->tids.netids;
2789                 sc->params.eo_wr_cred = val[2];
2790                 sc->params.ethoffload = 1;
2791         }
2792
2793         if (sc->toecaps) {
2794                 /* query offload-related parameters */
2795                 param[0] = FW_PARAM_DEV(NTID);
2796                 param[1] = FW_PARAM_PFVF(SERVER_START);
2797                 param[2] = FW_PARAM_PFVF(SERVER_END);
2798                 param[3] = FW_PARAM_PFVF(TDDP_START);
2799                 param[4] = FW_PARAM_PFVF(TDDP_END);
2800                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2801                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2802                 if (rc != 0) {
2803                         device_printf(sc->dev,
2804                             "failed to query TOE parameters: %d.\n", rc);
2805                         return (rc);
2806                 }
2807                 sc->tids.ntids = val[0];
2808                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
2809                 sc->tids.stid_base = val[1];
2810                 sc->tids.nstids = val[2] - val[1] + 1;
2811                 sc->vres.ddp.start = val[3];
2812                 sc->vres.ddp.size = val[4] - val[3] + 1;
2813                 sc->params.ofldq_wr_cred = val[5];
2814                 sc->params.offload = 1;
2815         }
2816         if (sc->rdmacaps) {
2817                 param[0] = FW_PARAM_PFVF(STAG_START);
2818                 param[1] = FW_PARAM_PFVF(STAG_END);
2819                 param[2] = FW_PARAM_PFVF(RQ_START);
2820                 param[3] = FW_PARAM_PFVF(RQ_END);
2821                 param[4] = FW_PARAM_PFVF(PBL_START);
2822                 param[5] = FW_PARAM_PFVF(PBL_END);
2823                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2824                 if (rc != 0) {
2825                         device_printf(sc->dev,
2826                             "failed to query RDMA parameters(1): %d.\n", rc);
2827                         return (rc);
2828                 }
2829                 sc->vres.stag.start = val[0];
2830                 sc->vres.stag.size = val[1] - val[0] + 1;
2831                 sc->vres.rq.start = val[2];
2832                 sc->vres.rq.size = val[3] - val[2] + 1;
2833                 sc->vres.pbl.start = val[4];
2834                 sc->vres.pbl.size = val[5] - val[4] + 1;
2835
2836                 param[0] = FW_PARAM_PFVF(SQRQ_START);
2837                 param[1] = FW_PARAM_PFVF(SQRQ_END);
2838                 param[2] = FW_PARAM_PFVF(CQ_START);
2839                 param[3] = FW_PARAM_PFVF(CQ_END);
2840                 param[4] = FW_PARAM_PFVF(OCQ_START);
2841                 param[5] = FW_PARAM_PFVF(OCQ_END);
2842                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2843                 if (rc != 0) {
2844                         device_printf(sc->dev,
2845                             "failed to query RDMA parameters(2): %d.\n", rc);
2846                         return (rc);
2847                 }
2848                 sc->vres.qp.start = val[0];
2849                 sc->vres.qp.size = val[1] - val[0] + 1;
2850                 sc->vres.cq.start = val[2];
2851                 sc->vres.cq.size = val[3] - val[2] + 1;
2852                 sc->vres.ocq.start = val[4];
2853                 sc->vres.ocq.size = val[5] - val[4] + 1;
2854         }
2855         if (sc->iscsicaps) {
2856                 param[0] = FW_PARAM_PFVF(ISCSI_START);
2857                 param[1] = FW_PARAM_PFVF(ISCSI_END);
2858                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2859                 if (rc != 0) {
2860                         device_printf(sc->dev,
2861                             "failed to query iSCSI parameters: %d.\n", rc);
2862                         return (rc);
2863                 }
2864                 sc->vres.iscsi.start = val[0];
2865                 sc->vres.iscsi.size = val[1] - val[0] + 1;
2866         }
2867
2868         /*
2869          * We've got the params we wanted to query via the firmware.  Now grab
2870          * some others directly from the chip.
2871          */
2872         rc = t4_read_chip_settings(sc);
2873
2874         return (rc);
2875 }
2876
2877 static int
2878 set_params__post_init(struct adapter *sc)
2879 {
2880         uint32_t param, val;
2881
2882         /* ask for encapsulated CPLs */
2883         param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
2884         val = 1;
2885         (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2886
2887         return (0);
2888 }
2889
2890 #undef FW_PARAM_PFVF
2891 #undef FW_PARAM_DEV
2892
2893 static void
2894 t4_set_desc(struct adapter *sc)
2895 {
2896         char buf[128];
2897         struct adapter_params *p = &sc->params;
2898
2899         snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
2900             "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
2901             chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
2902
2903         device_set_desc_copy(sc->dev, buf);
2904 }
2905
2906 static void
2907 build_medialist(struct port_info *pi, struct ifmedia *media)
2908 {
2909         int data, m;
2910
2911         PORT_LOCK(pi);
2912
2913         ifmedia_removeall(media);
2914
2915         m = IFM_ETHER | IFM_FDX;
2916         data = (pi->port_type << 8) | pi->mod_type;
2917
2918         switch(pi->port_type) {
2919         case FW_PORT_TYPE_BT_XFI:
2920                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2921                 break;
2922
2923         case FW_PORT_TYPE_BT_XAUI:
2924                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2925                 /* fall through */
2926
2927         case FW_PORT_TYPE_BT_SGMII:
2928                 ifmedia_add(media, m | IFM_1000_T, data, NULL);
2929                 ifmedia_add(media, m | IFM_100_TX, data, NULL);
2930                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL);
2931                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2932                 break;
2933
2934         case FW_PORT_TYPE_CX4:
2935                 ifmedia_add(media, m | IFM_10G_CX4, data, NULL);
2936                 ifmedia_set(media, m | IFM_10G_CX4);
2937                 break;
2938
2939         case FW_PORT_TYPE_QSFP_10G:
2940         case FW_PORT_TYPE_SFP:
2941         case FW_PORT_TYPE_FIBER_XFI:
2942         case FW_PORT_TYPE_FIBER_XAUI:
2943                 switch (pi->mod_type) {
2944
2945                 case FW_PORT_MOD_TYPE_LR:
2946                         ifmedia_add(media, m | IFM_10G_LR, data, NULL);
2947                         ifmedia_set(media, m | IFM_10G_LR);
2948                         break;
2949
2950                 case FW_PORT_MOD_TYPE_SR:
2951                         ifmedia_add(media, m | IFM_10G_SR, data, NULL);
2952                         ifmedia_set(media, m | IFM_10G_SR);
2953                         break;
2954
2955                 case FW_PORT_MOD_TYPE_LRM:
2956                         ifmedia_add(media, m | IFM_10G_LRM, data, NULL);
2957                         ifmedia_set(media, m | IFM_10G_LRM);
2958                         break;
2959
2960                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2961                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2962                         ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL);
2963                         ifmedia_set(media, m | IFM_10G_TWINAX);
2964                         break;
2965
2966                 case FW_PORT_MOD_TYPE_NONE:
2967                         m &= ~IFM_FDX;
2968                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2969                         ifmedia_set(media, m | IFM_NONE);
2970                         break;
2971
2972                 case FW_PORT_MOD_TYPE_NA:
2973                 case FW_PORT_MOD_TYPE_ER:
2974                 default:
2975                         device_printf(pi->dev,
2976                             "unknown port_type (%d), mod_type (%d)\n",
2977                             pi->port_type, pi->mod_type);
2978                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2979                         ifmedia_set(media, m | IFM_UNKNOWN);
2980                         break;
2981                 }
2982                 break;
2983
2984         case FW_PORT_TYPE_QSFP:
2985                 switch (pi->mod_type) {
2986
2987                 case FW_PORT_MOD_TYPE_LR:
2988                         ifmedia_add(media, m | IFM_40G_LR4, data, NULL);
2989                         ifmedia_set(media, m | IFM_40G_LR4);
2990                         break;
2991
2992                 case FW_PORT_MOD_TYPE_SR:
2993                         ifmedia_add(media, m | IFM_40G_SR4, data, NULL);
2994                         ifmedia_set(media, m | IFM_40G_SR4);
2995                         break;
2996
2997                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2998                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2999                         ifmedia_add(media, m | IFM_40G_CR4, data, NULL);
3000                         ifmedia_set(media, m | IFM_40G_CR4);
3001                         break;
3002
3003                 case FW_PORT_MOD_TYPE_NONE:
3004                         m &= ~IFM_FDX;
3005                         ifmedia_add(media, m | IFM_NONE, data, NULL);
3006                         ifmedia_set(media, m | IFM_NONE);
3007                         break;
3008
3009                 default:
3010                         device_printf(pi->dev,
3011                             "unknown port_type (%d), mod_type (%d)\n",
3012                             pi->port_type, pi->mod_type);
3013                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
3014                         ifmedia_set(media, m | IFM_UNKNOWN);
3015                         break;
3016                 }
3017                 break;
3018
3019         default:
3020                 device_printf(pi->dev,
3021                     "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
3022                     pi->mod_type);
3023                 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
3024                 ifmedia_set(media, m | IFM_UNKNOWN);
3025                 break;
3026         }
3027
3028         PORT_UNLOCK(pi);
3029 }
3030
3031 #define FW_MAC_EXACT_CHUNK      7
3032
3033 /*
3034  * Program the port's XGMAC based on parameters in ifnet.  The caller also
3035  * indicates which parameters should be programmed (the rest are left alone).
3036  */
3037 int
3038 update_mac_settings(struct ifnet *ifp, int flags)
3039 {
3040         int rc = 0;
3041         struct port_info *pi = ifp->if_softc;
3042         struct adapter *sc = pi->adapter;
3043         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
3044         uint16_t viid = 0xffff;
3045         int16_t *xact_addr_filt = NULL;
3046
3047         ASSERT_SYNCHRONIZED_OP(sc);
3048         KASSERT(flags, ("%s: not told what to update.", __func__));
3049
3050         if (ifp == pi->ifp) {
3051                 viid = pi->viid;
3052                 xact_addr_filt = &pi->xact_addr_filt;
3053         }
3054 #ifdef DEV_NETMAP
3055         else if (ifp == pi->nm_ifp) {
3056                 viid = pi->nm_viid;
3057                 xact_addr_filt = &pi->nm_xact_addr_filt;
3058         }
3059 #endif
3060         if (flags & XGMAC_MTU)
3061                 mtu = ifp->if_mtu;
3062
3063         if (flags & XGMAC_PROMISC)
3064                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
3065
3066         if (flags & XGMAC_ALLMULTI)
3067                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
3068
3069         if (flags & XGMAC_VLANEX)
3070                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
3071
3072         if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
3073                 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti,
3074                     1, vlanex, false);
3075                 if (rc) {
3076                         if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
3077                             rc);
3078                         return (rc);
3079                 }
3080         }
3081
3082         if (flags & XGMAC_UCADDR) {
3083                 uint8_t ucaddr[ETHER_ADDR_LEN];
3084
3085                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
3086                 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr,
3087                     true, true);
3088                 if (rc < 0) {
3089                         rc = -rc;
3090                         if_printf(ifp, "change_mac failed: %d\n", rc);
3091                         return (rc);
3092                 } else {
3093                         *xact_addr_filt = rc;
3094                         rc = 0;
3095                 }
3096         }
3097
3098         if (flags & XGMAC_MCADDRS) {
3099                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
3100                 int del = 1;
3101                 uint64_t hash = 0;
3102                 struct ifmultiaddr *ifma;
3103                 int i = 0, j;
3104
3105                 if_maddr_rlock(ifp);
3106                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
3107                         if (ifma->ifma_addr->sa_family != AF_LINK)
3108                                 continue;
3109                         mcaddr[i] =
3110                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
3111                         MPASS(ETHER_IS_MULTICAST(mcaddr[i]));
3112                         i++;
3113
3114                         if (i == FW_MAC_EXACT_CHUNK) {
3115                                 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del,
3116                                     i, mcaddr, NULL, &hash, 0);
3117                                 if (rc < 0) {
3118                                         rc = -rc;
3119                                         for (j = 0; j < i; j++) {
3120                                                 if_printf(ifp,
3121                                                     "failed to add mc address"
3122                                                     " %02x:%02x:%02x:"
3123                                                     "%02x:%02x:%02x rc=%d\n",
3124                                                     mcaddr[j][0], mcaddr[j][1],
3125                                                     mcaddr[j][2], mcaddr[j][3],
3126                                                     mcaddr[j][4], mcaddr[j][5],
3127                                                     rc);
3128                                         }
3129                                         goto mcfail;
3130                                 }
3131                                 del = 0;
3132                                 i = 0;
3133                         }
3134                 }
3135                 if (i > 0) {
3136                         rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i,
3137                             mcaddr, NULL, &hash, 0);
3138                         if (rc < 0) {
3139                                 rc = -rc;
3140                                 for (j = 0; j < i; j++) {
3141                                         if_printf(ifp,
3142                                             "failed to add mc address"
3143                                             " %02x:%02x:%02x:"
3144                                             "%02x:%02x:%02x rc=%d\n",
3145                                             mcaddr[j][0], mcaddr[j][1],
3146                                             mcaddr[j][2], mcaddr[j][3],
3147                                             mcaddr[j][4], mcaddr[j][5],
3148                                             rc);
3149                                 }
3150                                 goto mcfail;
3151                         }
3152                 }
3153
3154                 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0);
3155                 if (rc != 0)
3156                         if_printf(ifp, "failed to set mc address hash: %d", rc);
3157 mcfail:
3158                 if_maddr_runlock(ifp);
3159         }
3160
3161         return (rc);
3162 }
3163
3164 int
3165 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
3166     char *wmesg)
3167 {
3168         int rc, pri;
3169
3170 #ifdef WITNESS
3171         /* the caller thinks it's ok to sleep, but is it really? */
3172         if (flags & SLEEP_OK)
3173                 pause("t4slptst", 1);
3174 #endif
3175
3176         if (INTR_OK)
3177                 pri = PCATCH;
3178         else
3179                 pri = 0;
3180
3181         ADAPTER_LOCK(sc);
3182         for (;;) {
3183
3184                 if (pi && IS_DOOMED(pi)) {
3185                         rc = ENXIO;
3186                         goto done;
3187                 }
3188
3189                 if (!IS_BUSY(sc)) {
3190                         rc = 0;
3191                         break;
3192                 }
3193
3194                 if (!(flags & SLEEP_OK)) {
3195                         rc = EBUSY;
3196                         goto done;
3197                 }
3198
3199                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
3200                         rc = EINTR;
3201                         goto done;
3202                 }
3203         }
3204
3205         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
3206         SET_BUSY(sc);
3207 #ifdef INVARIANTS
3208         sc->last_op = wmesg;
3209         sc->last_op_thr = curthread;
3210 #endif
3211
3212 done:
3213         if (!(flags & HOLD_LOCK) || rc)
3214                 ADAPTER_UNLOCK(sc);
3215
3216         return (rc);
3217 }
3218
3219 void
3220 end_synchronized_op(struct adapter *sc, int flags)
3221 {
3222
3223         if (flags & LOCK_HELD)
3224                 ADAPTER_LOCK_ASSERT_OWNED(sc);
3225         else
3226                 ADAPTER_LOCK(sc);
3227
3228         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
3229         CLR_BUSY(sc);
3230         wakeup(&sc->flags);
3231         ADAPTER_UNLOCK(sc);
3232 }
3233
3234 static int
3235 cxgbe_init_synchronized(struct port_info *pi)
3236 {
3237         struct adapter *sc = pi->adapter;
3238         struct ifnet *ifp = pi->ifp;
3239         int rc = 0;
3240
3241         ASSERT_SYNCHRONIZED_OP(sc);
3242
3243         if (isset(&sc->open_device_map, pi->port_id)) {
3244                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
3245                     ("mismatch between open_device_map and if_drv_flags"));
3246                 return (0);     /* already running */
3247         }
3248
3249         if (!(sc->flags & FULL_INIT_DONE) &&
3250             ((rc = adapter_full_init(sc)) != 0))
3251                 return (rc);    /* error message displayed already */
3252
3253         if (!(pi->flags & PORT_INIT_DONE) &&
3254             ((rc = port_full_init(pi)) != 0))
3255                 return (rc); /* error message displayed already */
3256
3257         rc = update_mac_settings(ifp, XGMAC_ALL);
3258         if (rc)
3259                 goto done;      /* error message displayed already */
3260
3261         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
3262         if (rc != 0) {
3263                 if_printf(ifp, "enable_vi failed: %d\n", rc);
3264                 goto done;
3265         }
3266
3267         /*
3268          * The first iq of the first port to come up is used for tracing.
3269          */
3270         if (sc->traceq < 0) {
3271                 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
3272                 t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
3273                     A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
3274                     V_QUEUENUMBER(sc->traceq));
3275                 pi->flags |= HAS_TRACEQ;
3276         }
3277
3278         /* all ok */
3279         setbit(&sc->open_device_map, pi->port_id);
3280         PORT_LOCK(pi);
3281         ifp->if_drv_flags |= IFF_DRV_RUNNING;
3282         PORT_UNLOCK(pi);
3283
3284         callout_reset(&pi->tick, hz, cxgbe_tick, pi);
3285 done:
3286         if (rc != 0)
3287                 cxgbe_uninit_synchronized(pi);
3288
3289         return (rc);
3290 }
3291
3292 /*
3293  * Idempotent.
3294  */
3295 static int
3296 cxgbe_uninit_synchronized(struct port_info *pi)
3297 {
3298         struct adapter *sc = pi->adapter;
3299         struct ifnet *ifp = pi->ifp;
3300         int rc;
3301
3302         ASSERT_SYNCHRONIZED_OP(sc);
3303
3304         /*
3305          * Disable the VI so that all its data in either direction is discarded
3306          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
3307          * tick) intact as the TP can deliver negative advice or data that it's
3308          * holding in its RAM (for an offloaded connection) even after the VI is
3309          * disabled.
3310          */
3311         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
3312         if (rc) {
3313                 if_printf(ifp, "disable_vi failed: %d\n", rc);
3314                 return (rc);
3315         }
3316
3317         clrbit(&sc->open_device_map, pi->port_id);
3318         PORT_LOCK(pi);
3319         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3320         PORT_UNLOCK(pi);
3321
3322         pi->link_cfg.link_ok = 0;
3323         pi->link_cfg.speed = 0;
3324         pi->linkdnrc = -1;
3325         t4_os_link_changed(sc, pi->port_id, 0, -1);
3326
3327         return (0);
3328 }
3329
3330 /*
3331  * It is ok for this function to fail midway and return right away.  t4_detach
3332  * will walk the entire sc->irq list and clean up whatever is valid.
3333  */
3334 static int
3335 setup_intr_handlers(struct adapter *sc)
3336 {
3337         int rc, rid, p, q;
3338         char s[8];
3339         struct irq *irq;
3340         struct port_info *pi;
3341         struct sge_rxq *rxq;
3342 #ifdef TCP_OFFLOAD
3343         struct sge_ofld_rxq *ofld_rxq;
3344 #endif
3345 #ifdef DEV_NETMAP
3346         struct sge_nm_rxq *nm_rxq;
3347 #endif
3348
3349         /*
3350          * Setup interrupts.
3351          */
3352         irq = &sc->irq[0];
3353         rid = sc->intr_type == INTR_INTX ? 0 : 1;
3354         if (sc->intr_count == 1)
3355                 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
3356
3357         /* Multiple interrupts. */
3358         KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
3359             ("%s: too few intr.", __func__));
3360
3361         /* The first one is always error intr */
3362         rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
3363         if (rc != 0)
3364                 return (rc);
3365         irq++;
3366         rid++;
3367
3368         /* The second one is always the firmware event queue */
3369         rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
3370         if (rc != 0)
3371                 return (rc);
3372         irq++;
3373         rid++;
3374
3375         for_each_port(sc, p) {
3376                 pi = sc->port[p];
3377
3378                 if (pi->flags & INTR_RXQ) {
3379                         for_each_rxq(pi, q, rxq) {
3380                                 snprintf(s, sizeof(s), "%d.%d", p, q);
3381                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
3382                                     s);
3383                                 if (rc != 0)
3384                                         return (rc);
3385                                 irq++;
3386                                 rid++;
3387                         }
3388                 }
3389 #ifdef TCP_OFFLOAD
3390                 if (pi->flags & INTR_OFLD_RXQ) {
3391                         for_each_ofld_rxq(pi, q, ofld_rxq) {
3392                                 snprintf(s, sizeof(s), "%d,%d", p, q);
3393                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr,
3394                                     ofld_rxq, s);
3395                                 if (rc != 0)
3396                                         return (rc);
3397                                 irq++;
3398                                 rid++;
3399                         }
3400                 }
3401 #endif
3402 #ifdef DEV_NETMAP
3403                 if (pi->flags & INTR_NM_RXQ) {
3404                         for_each_nm_rxq(pi, q, nm_rxq) {
3405                                 snprintf(s, sizeof(s), "%d-%d", p, q);
3406                                 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr,
3407                                     nm_rxq, s);
3408                                 if (rc != 0)
3409                                         return (rc);
3410                                 irq++;
3411                                 rid++;
3412                         }
3413                 }
3414 #endif
3415         }
3416         MPASS(irq == &sc->irq[sc->intr_count]);
3417
3418         return (0);
3419 }
3420
3421 int
3422 adapter_full_init(struct adapter *sc)
3423 {
3424         int rc, i;
3425
3426         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3427         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
3428             ("%s: FULL_INIT_DONE already", __func__));
3429
3430         /*
3431          * queues that belong to the adapter (not any particular port).
3432          */
3433         rc = t4_setup_adapter_queues(sc);
3434         if (rc != 0)
3435                 goto done;
3436
3437         for (i = 0; i < nitems(sc->tq); i++) {
3438                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
3439                     taskqueue_thread_enqueue, &sc->tq[i]);
3440                 if (sc->tq[i] == NULL) {
3441                         device_printf(sc->dev,
3442                             "failed to allocate task queue %d\n", i);
3443                         rc = ENOMEM;
3444                         goto done;
3445                 }
3446                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
3447                     device_get_nameunit(sc->dev), i);
3448         }
3449
3450         t4_intr_enable(sc);
3451         sc->flags |= FULL_INIT_DONE;
3452 done:
3453         if (rc != 0)
3454                 adapter_full_uninit(sc);
3455
3456         return (rc);
3457 }
3458
3459 int
3460 adapter_full_uninit(struct adapter *sc)
3461 {
3462         int i;
3463
3464         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3465
3466         t4_teardown_adapter_queues(sc);
3467
3468         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
3469                 taskqueue_free(sc->tq[i]);
3470                 sc->tq[i] = NULL;
3471         }
3472
3473         sc->flags &= ~FULL_INIT_DONE;
3474
3475         return (0);
3476 }
3477
3478 int
3479 port_full_init(struct port_info *pi)
3480 {
3481         struct adapter *sc = pi->adapter;
3482         struct ifnet *ifp = pi->ifp;
3483         uint16_t *rss;
3484         struct sge_rxq *rxq;
3485         int rc, i, j;
3486
3487         ASSERT_SYNCHRONIZED_OP(sc);
3488         KASSERT((pi->flags & PORT_INIT_DONE) == 0,
3489             ("%s: PORT_INIT_DONE already", __func__));
3490
3491         sysctl_ctx_init(&pi->ctx);
3492         pi->flags |= PORT_SYSCTL_CTX;
3493
3494         /*
3495          * Allocate tx/rx/fl queues for this port.
3496          */
3497         rc = t4_setup_port_queues(pi);
3498         if (rc != 0)
3499                 goto done;      /* error message displayed already */
3500
3501         /*
3502          * Setup RSS for this port.  Save a copy of the RSS table for later use.
3503          */
3504         rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
3505         for (i = 0; i < pi->rss_size;) {
3506                 for_each_rxq(pi, j, rxq) {
3507                         rss[i++] = rxq->iq.abs_id;
3508                         if (i == pi->rss_size)
3509                                 break;
3510                 }
3511         }
3512
3513         rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
3514             pi->rss_size);
3515         if (rc != 0) {
3516                 if_printf(ifp, "rss_config failed: %d\n", rc);
3517                 goto done;
3518         }
3519
3520         pi->rss = rss;
3521         pi->flags |= PORT_INIT_DONE;
3522 done:
3523         if (rc != 0)
3524                 port_full_uninit(pi);
3525
3526         return (rc);
3527 }
3528
3529 /*
3530  * Idempotent.
3531  */
3532 int
3533 port_full_uninit(struct port_info *pi)
3534 {
3535         struct adapter *sc = pi->adapter;
3536         int i;
3537         struct sge_rxq *rxq;
3538         struct sge_txq *txq;
3539 #ifdef TCP_OFFLOAD
3540         struct sge_ofld_rxq *ofld_rxq;
3541         struct sge_wrq *ofld_txq;
3542 #endif
3543
3544         if (pi->flags & PORT_INIT_DONE) {
3545
3546                 /* Need to quiesce queues.  XXX: ctrl queues? */
3547
3548                 for_each_txq(pi, i, txq) {
3549                         quiesce_eq(sc, &txq->eq);
3550                 }
3551
3552 #ifdef TCP_OFFLOAD
3553                 for_each_ofld_txq(pi, i, ofld_txq) {
3554                         quiesce_eq(sc, &ofld_txq->eq);
3555                 }
3556 #endif
3557
3558                 for_each_rxq(pi, i, rxq) {
3559                         quiesce_iq(sc, &rxq->iq);
3560                         quiesce_fl(sc, &rxq->fl);
3561                 }
3562
3563 #ifdef TCP_OFFLOAD
3564                 for_each_ofld_rxq(pi, i, ofld_rxq) {
3565                         quiesce_iq(sc, &ofld_rxq->iq);
3566                         quiesce_fl(sc, &ofld_rxq->fl);
3567                 }
3568 #endif
3569                 free(pi->rss, M_CXGBE);
3570         }
3571
3572         t4_teardown_port_queues(pi);
3573         pi->flags &= ~PORT_INIT_DONE;
3574
3575         return (0);
3576 }
3577
3578 static void
3579 quiesce_eq(struct adapter *sc, struct sge_eq *eq)
3580 {
3581         EQ_LOCK(eq);
3582         eq->flags |= EQ_DOOMED;
3583
3584         /*
3585          * Wait for the response to a credit flush if one's
3586          * pending.
3587          */
3588         while (eq->flags & EQ_CRFLUSHED)
3589                 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0);
3590         EQ_UNLOCK(eq);
3591
3592         callout_drain(&eq->tx_callout); /* XXX: iffy */
3593         pause("callout", 10);           /* Still iffy */
3594
3595         taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task);
3596 }
3597
3598 static void
3599 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
3600 {
3601         (void) sc;      /* unused */
3602
3603         /* Synchronize with the interrupt handler */
3604         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
3605                 pause("iqfree", 1);
3606 }
3607
3608 static void
3609 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
3610 {
3611         mtx_lock(&sc->sfl_lock);
3612         FL_LOCK(fl);
3613         fl->flags |= FL_DOOMED;
3614         FL_UNLOCK(fl);
3615         mtx_unlock(&sc->sfl_lock);
3616
3617         callout_drain(&sc->sfl_callout);
3618         KASSERT((fl->flags & FL_STARVING) == 0,
3619             ("%s: still starving", __func__));
3620 }
3621
3622 static int
3623 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
3624     driver_intr_t *handler, void *arg, char *name)
3625 {
3626         int rc;
3627
3628         irq->rid = rid;
3629         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
3630             RF_SHAREABLE | RF_ACTIVE);
3631         if (irq->res == NULL) {
3632                 device_printf(sc->dev,
3633                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
3634                 return (ENOMEM);
3635         }
3636
3637         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
3638             NULL, handler, arg, &irq->tag);
3639         if (rc != 0) {
3640                 device_printf(sc->dev,
3641                     "failed to setup interrupt for rid %d, name %s: %d\n",
3642                     rid, name, rc);
3643         } else if (name)
3644                 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
3645
3646         return (rc);
3647 }
3648
3649 static int
3650 t4_free_irq(struct adapter *sc, struct irq *irq)
3651 {
3652         if (irq->tag)
3653                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
3654         if (irq->res)
3655                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
3656
3657         bzero(irq, sizeof(*irq));
3658
3659         return (0);
3660 }
3661
3662 static void
3663 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
3664     unsigned int end)
3665 {
3666         uint32_t *p = (uint32_t *)(buf + start);
3667
3668         for ( ; start <= end; start += sizeof(uint32_t))
3669                 *p++ = t4_read_reg(sc, start);
3670 }
3671
3672 static void
3673 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
3674 {
3675         int i, n;
3676         const unsigned int *reg_ranges;
3677         static const unsigned int t4_reg_ranges[] = {
3678                 0x1008, 0x1108,
3679                 0x1180, 0x11b4,
3680                 0x11fc, 0x123c,
3681                 0x1300, 0x173c,
3682                 0x1800, 0x18fc,
3683                 0x3000, 0x30d8,
3684                 0x30e0, 0x5924,
3685                 0x5960, 0x59d4,
3686                 0x5a00, 0x5af8,
3687                 0x6000, 0x6098,
3688                 0x6100, 0x6150,
3689                 0x6200, 0x6208,
3690                 0x6240, 0x6248,
3691                 0x6280, 0x6338,
3692                 0x6370, 0x638c,
3693                 0x6400, 0x643c,
3694                 0x6500, 0x6524,
3695                 0x6a00, 0x6a38,
3696                 0x6a60, 0x6a78,
3697                 0x6b00, 0x6b84,
3698                 0x6bf0, 0x6c84,
3699                 0x6cf0, 0x6d84,
3700                 0x6df0, 0x6e84,
3701                 0x6ef0, 0x6f84,
3702                 0x6ff0, 0x7084,
3703                 0x70f0, 0x7184,
3704                 0x71f0, 0x7284,
3705                 0x72f0, 0x7384,
3706                 0x73f0, 0x7450,
3707                 0x7500, 0x7530,
3708                 0x7600, 0x761c,
3709                 0x7680, 0x76cc,
3710                 0x7700, 0x7798,
3711                 0x77c0, 0x77fc,
3712                 0x7900, 0x79fc,
3713                 0x7b00, 0x7c38,
3714                 0x7d00, 0x7efc,
3715                 0x8dc0, 0x8e1c,
3716                 0x8e30, 0x8e78,
3717                 0x8ea0, 0x8f6c,
3718                 0x8fc0, 0x9074,
3719                 0x90fc, 0x90fc,
3720                 0x9400, 0x9458,
3721                 0x9600, 0x96bc,
3722                 0x9800, 0x9808,
3723                 0x9820, 0x983c,
3724                 0x9850, 0x9864,
3725                 0x9c00, 0x9c6c,
3726                 0x9c80, 0x9cec,
3727                 0x9d00, 0x9d6c,
3728                 0x9d80, 0x9dec,
3729                 0x9e00, 0x9e6c,
3730                 0x9e80, 0x9eec,
3731                 0x9f00, 0x9f6c,
3732                 0x9f80, 0x9fec,
3733                 0xd004, 0xd03c,
3734                 0xdfc0, 0xdfe0,
3735                 0xe000, 0xea7c,
3736                 0xf000, 0x11110,
3737                 0x11118, 0x11190,
3738                 0x19040, 0x1906c,
3739                 0x19078, 0x19080,
3740                 0x1908c, 0x19124,
3741                 0x19150, 0x191b0,
3742                 0x191d0, 0x191e8,
3743                 0x19238, 0x1924c,
3744                 0x193f8, 0x19474,
3745                 0x19490, 0x194f8,
3746                 0x19800, 0x19f30,
3747                 0x1a000, 0x1a06c,
3748                 0x1a0b0, 0x1a120,
3749                 0x1a128, 0x1a138,
3750                 0x1a190, 0x1a1c4,
3751                 0x1a1fc, 0x1a1fc,
3752                 0x1e040, 0x1e04c,
3753                 0x1e284, 0x1e28c,
3754                 0x1e2c0, 0x1e2c0,
3755                 0x1e2e0, 0x1e2e0,
3756                 0x1e300, 0x1e384,
3757                 0x1e3c0, 0x1e3c8,
3758                 0x1e440, 0x1e44c,
3759                 0x1e684, 0x1e68c,
3760                 0x1e6c0, 0x1e6c0,
3761                 0x1e6e0, 0x1e6e0,
3762                 0x1e700, 0x1e784,
3763                 0x1e7c0, 0x1e7c8,
3764                 0x1e840, 0x1e84c,
3765                 0x1ea84, 0x1ea8c,
3766                 0x1eac0, 0x1eac0,
3767                 0x1eae0, 0x1eae0,
3768                 0x1eb00, 0x1eb84,
3769                 0x1ebc0, 0x1ebc8,
3770                 0x1ec40, 0x1ec4c,
3771                 0x1ee84, 0x1ee8c,
3772                 0x1eec0, 0x1eec0,
3773                 0x1eee0, 0x1eee0,
3774                 0x1ef00, 0x1ef84,
3775                 0x1efc0, 0x1efc8,
3776                 0x1f040, 0x1f04c,
3777                 0x1f284, 0x1f28c,
3778                 0x1f2c0, 0x1f2c0,
3779                 0x1f2e0, 0x1f2e0,
3780                 0x1f300, 0x1f384,
3781                 0x1f3c0, 0x1f3c8,
3782                 0x1f440, 0x1f44c,
3783                 0x1f684, 0x1f68c,
3784                 0x1f6c0, 0x1f6c0,
3785                 0x1f6e0, 0x1f6e0,
3786                 0x1f700, 0x1f784,
3787                 0x1f7c0, 0x1f7c8,
3788                 0x1f840, 0x1f84c,
3789                 0x1fa84, 0x1fa8c,
3790                 0x1fac0, 0x1fac0,
3791                 0x1fae0, 0x1fae0,
3792                 0x1fb00, 0x1fb84,
3793                 0x1fbc0, 0x1fbc8,
3794                 0x1fc40, 0x1fc4c,
3795                 0x1fe84, 0x1fe8c,
3796                 0x1fec0, 0x1fec0,
3797                 0x1fee0, 0x1fee0,
3798                 0x1ff00, 0x1ff84,
3799                 0x1ffc0, 0x1ffc8,
3800                 0x20000, 0x2002c,
3801                 0x20100, 0x2013c,
3802                 0x20190, 0x201c8,
3803                 0x20200, 0x20318,
3804                 0x20400, 0x20528,
3805                 0x20540, 0x20614,
3806                 0x21000, 0x21040,
3807                 0x2104c, 0x21060,
3808                 0x210c0, 0x210ec,
3809                 0x21200, 0x21268,
3810                 0x21270, 0x21284,
3811                 0x212fc, 0x21388,
3812                 0x21400, 0x21404,
3813                 0x21500, 0x21518,
3814                 0x2152c, 0x2153c,
3815                 0x21550, 0x21554,
3816                 0x21600, 0x21600,
3817                 0x21608, 0x21628,
3818                 0x21630, 0x2163c,
3819                 0x21700, 0x2171c,
3820                 0x21780, 0x2178c,
3821                 0x21800, 0x21c38,
3822                 0x21c80, 0x21d7c,
3823                 0x21e00, 0x21e04,
3824                 0x22000, 0x2202c,
3825                 0x22100, 0x2213c,
3826                 0x22190, 0x221c8,
3827                 0x22200, 0x22318,
3828                 0x22400, 0x22528,
3829                 0x22540, 0x22614,
3830                 0x23000, 0x23040,
3831                 0x2304c, 0x23060,
3832                 0x230c0, 0x230ec,
3833                 0x23200, 0x23268,
3834                 0x23270, 0x23284,
3835                 0x232fc, 0x23388,
3836                 0x23400, 0x23404,
3837                 0x23500, 0x23518,
3838                 0x2352c, 0x2353c,
3839                 0x23550, 0x23554,
3840                 0x23600, 0x23600,
3841                 0x23608, 0x23628,
3842                 0x23630, 0x2363c,
3843                 0x23700, 0x2371c,
3844                 0x23780, 0x2378c,
3845                 0x23800, 0x23c38,
3846                 0x23c80, 0x23d7c,
3847                 0x23e00, 0x23e04,
3848                 0x24000, 0x2402c,
3849                 0x24100, 0x2413c,
3850                 0x24190, 0x241c8,
3851                 0x24200, 0x24318,
3852                 0x24400, 0x24528,
3853                 0x24540, 0x24614,
3854                 0x25000, 0x25040,
3855                 0x2504c, 0x25060,
3856                 0x250c0, 0x250ec,
3857                 0x25200, 0x25268,
3858                 0x25270, 0x25284,
3859                 0x252fc, 0x25388,
3860                 0x25400, 0x25404,
3861                 0x25500, 0x25518,
3862                 0x2552c, 0x2553c,
3863                 0x25550, 0x25554,
3864                 0x25600, 0x25600,
3865                 0x25608, 0x25628,
3866                 0x25630, 0x2563c,
3867                 0x25700, 0x2571c,
3868                 0x25780, 0x2578c,
3869                 0x25800, 0x25c38,
3870                 0x25c80, 0x25d7c,
3871                 0x25e00, 0x25e04,
3872                 0x26000, 0x2602c,
3873                 0x26100, 0x2613c,
3874                 0x26190, 0x261c8,
3875                 0x26200, 0x26318,
3876                 0x26400, 0x26528,
3877                 0x26540, 0x26614,
3878                 0x27000, 0x27040,
3879                 0x2704c, 0x27060,
3880                 0x270c0, 0x270ec,
3881                 0x27200, 0x27268,
3882                 0x27270, 0x27284,
3883                 0x272fc, 0x27388,
3884                 0x27400, 0x27404,
3885                 0x27500, 0x27518,
3886                 0x2752c, 0x2753c,
3887                 0x27550, 0x27554,
3888                 0x27600, 0x27600,
3889                 0x27608, 0x27628,
3890                 0x27630, 0x2763c,
3891                 0x27700, 0x2771c,
3892                 0x27780, 0x2778c,
3893                 0x27800, 0x27c38,
3894                 0x27c80, 0x27d7c,
3895                 0x27e00, 0x27e04
3896         };
3897         static const unsigned int t5_reg_ranges[] = {
3898                 0x1008, 0x1148,
3899                 0x1180, 0x11b4,
3900                 0x11fc, 0x123c,
3901                 0x1280, 0x173c,
3902                 0x1800, 0x18fc,
3903                 0x3000, 0x3028,
3904                 0x3060, 0x30d8,
3905                 0x30e0, 0x30fc,
3906                 0x3140, 0x357c,
3907                 0x35a8, 0x35cc,
3908                 0x35ec, 0x35ec,
3909                 0x3600, 0x5624,
3910                 0x56cc, 0x575c,
3911                 0x580c, 0x5814,
3912                 0x5890, 0x58bc,
3913                 0x5940, 0x59dc,
3914                 0x59fc, 0x5a18,
3915                 0x5a60, 0x5a9c,
3916                 0x5b94, 0x5bfc,
3917                 0x6000, 0x6040,
3918                 0x6058, 0x614c,
3919                 0x7700, 0x7798,
3920                 0x77c0, 0x78fc,
3921                 0x7b00, 0x7c54,
3922                 0x7d00, 0x7efc,
3923                 0x8dc0, 0x8de0,
3924                 0x8df8, 0x8e84,
3925                 0x8ea0, 0x8f84,
3926                 0x8fc0, 0x90f8,
3927                 0x9400, 0x9470,
3928                 0x9600, 0x96f4,
3929                 0x9800, 0x9808,
3930                 0x9820, 0x983c,
3931                 0x9850, 0x9864,
3932                 0x9c00, 0x9c6c,
3933                 0x9c80, 0x9cec,
3934                 0x9d00, 0x9d6c,
3935                 0x9d80, 0x9dec,
3936                 0x9e00, 0x9e6c,
3937                 0x9e80, 0x9eec,
3938                 0x9f00, 0x9f6c,
3939                 0x9f80, 0xa020,
3940                 0xd004, 0xd03c,
3941                 0xdfc0, 0xdfe0,
3942                 0xe000, 0x11088,
3943                 0x1109c, 0x11110,
3944                 0x11118, 0x1117c,
3945                 0x11190, 0x11204,
3946                 0x19040, 0x1906c,
3947                 0x19078, 0x19080,
3948                 0x1908c, 0x19124,
3949                 0x19150, 0x191b0,
3950                 0x191d0, 0x191e8,
3951                 0x19238, 0x19290,
3952                 0x193f8, 0x19474,
3953                 0x19490, 0x194cc,
3954                 0x194f0, 0x194f8,
3955                 0x19c00, 0x19c60,
3956                 0x19c94, 0x19e10,
3957                 0x19e50, 0x19f34,
3958                 0x19f40, 0x19f50,
3959                 0x19f90, 0x19fe4,
3960                 0x1a000, 0x1a06c,
3961                 0x1a0b0, 0x1a120,
3962                 0x1a128, 0x1a138,
3963                 0x1a190, 0x1a1c4,
3964                 0x1a1fc, 0x1a1fc,
3965                 0x1e008, 0x1e00c,
3966                 0x1e040, 0x1e04c,
3967                 0x1e284, 0x1e290,
3968                 0x1e2c0, 0x1e2c0,
3969                 0x1e2e0, 0x1e2e0,
3970                 0x1e300, 0x1e384,
3971                 0x1e3c0, 0x1e3c8,
3972                 0x1e408, 0x1e40c,
3973                 0x1e440, 0x1e44c,
3974                 0x1e684, 0x1e690,
3975                 0x1e6c0, 0x1e6c0,
3976                 0x1e6e0, 0x1e6e0,
3977                 0x1e700, 0x1e784,
3978                 0x1e7c0, 0x1e7c8,
3979                 0x1e808, 0x1e80c,
3980                 0x1e840, 0x1e84c,
3981                 0x1ea84, 0x1ea90,
3982                 0x1eac0, 0x1eac0,
3983                 0x1eae0, 0x1eae0,
3984                 0x1eb00, 0x1eb84,
3985                 0x1ebc0, 0x1ebc8,
3986                 0x1ec08, 0x1ec0c,
3987                 0x1ec40, 0x1ec4c,
3988                 0x1ee84, 0x1ee90,
3989                 0x1eec0, 0x1eec0,
3990                 0x1eee0, 0x1eee0,
3991                 0x1ef00, 0x1ef84,
3992                 0x1efc0, 0x1efc8,
3993                 0x1f008, 0x1f00c,
3994                 0x1f040, 0x1f04c,
3995                 0x1f284, 0x1f290,
3996                 0x1f2c0, 0x1f2c0,
3997                 0x1f2e0, 0x1f2e0,
3998                 0x1f300, 0x1f384,
3999                 0x1f3c0, 0x1f3c8,
4000                 0x1f408, 0x1f40c,
4001                 0x1f440, 0x1f44c,
4002                 0x1f684, 0x1f690,
4003                 0x1f6c0, 0x1f6c0,
4004                 0x1f6e0, 0x1f6e0,
4005                 0x1f700, 0x1f784,
4006                 0x1f7c0, 0x1f7c8,
4007                 0x1f808, 0x1f80c,
4008                 0x1f840, 0x1f84c,
4009                 0x1fa84, 0x1fa90,
4010                 0x1fac0, 0x1fac0,
4011                 0x1fae0, 0x1fae0,
4012                 0x1fb00, 0x1fb84,
4013                 0x1fbc0, 0x1fbc8,
4014                 0x1fc08, 0x1fc0c,
4015                 0x1fc40, 0x1fc4c,
4016                 0x1fe84, 0x1fe90,
4017                 0x1fec0, 0x1fec0,
4018                 0x1fee0, 0x1fee0,
4019                 0x1ff00, 0x1ff84,
4020                 0x1ffc0, 0x1ffc8,
4021                 0x30000, 0x30030,
4022                 0x30100, 0x30144,
4023                 0x30190, 0x301d0,
4024                 0x30200, 0x30318,
4025                 0x30400, 0x3052c,
4026                 0x30540, 0x3061c,
4027                 0x30800, 0x30834,
4028                 0x308c0, 0x30908,
4029                 0x30910, 0x309ac,
4030                 0x30a00, 0x30a2c,
4031                 0x30a44, 0x30a50,
4032                 0x30a74, 0x30c24,
4033                 0x30d00, 0x30d00,
4034                 0x30d08, 0x30d14,
4035                 0x30d1c, 0x30d20,
4036                 0x30d3c, 0x30d50,
4037                 0x31200, 0x3120c,
4038                 0x31220, 0x31220,
4039                 0x31240, 0x31240,
4040                 0x31600, 0x3160c,
4041                 0x31a00, 0x31a1c,
4042                 0x31e00, 0x31e20,
4043                 0x31e38, 0x31e3c,
4044                 0x31e80, 0x31e80,
4045                 0x31e88, 0x31ea8,
4046                 0x31eb0, 0x31eb4,
4047                 0x31ec8, 0x31ed4,
4048                 0x31fb8, 0x32004,
4049                 0x32200, 0x32200,
4050                 0x32208, 0x32240,
4051                 0x32248, 0x32280,
4052                 0x32288, 0x322c0,
4053                 0x322c8, 0x322fc,
4054                 0x32600, 0x32630,
4055                 0x32a00, 0x32abc,
4056                 0x32b00, 0x32b70,
4057                 0x33000, 0x33048,
4058                 0x33060, 0x3309c,
4059                 0x330f0, 0x33148,
4060                 0x33160, 0x3319c,
4061                 0x331f0, 0x332e4,
4062                 0x332f8, 0x333e4,
4063                 0x333f8, 0x33448,
4064                 0x33460, 0x3349c,
4065                 0x334f0, 0x33548,
4066                 0x33560, 0x3359c,
4067                 0x335f0, 0x336e4,
4068                 0x336f8, 0x337e4,
4069                 0x337f8, 0x337fc,
4070                 0x33814, 0x33814,
4071                 0x3382c, 0x3382c,
4072                 0x33880, 0x3388c,
4073                 0x338e8, 0x338ec,
4074                 0x33900, 0x33948,
4075                 0x33960, 0x3399c,
4076                 0x339f0, 0x33ae4,
4077                 0x33af8, 0x33b10,
4078                 0x33b28, 0x33b28,
4079                 0x33b3c, 0x33b50,
4080                 0x33bf0, 0x33c10,
4081                 0x33c28, 0x33c28,
4082                 0x33c3c, 0x33c50,
4083                 0x33cf0, 0x33cfc,
4084                 0x34000, 0x34030,
4085                 0x34100, 0x34144,
4086                 0x34190, 0x341d0,
4087                 0x34200, 0x34318,
4088                 0x34400, 0x3452c,
4089                 0x34540, 0x3461c,
4090                 0x34800, 0x34834,
4091                 0x348c0, 0x34908,
4092                 0x34910, 0x349ac,
4093                 0x34a00, 0x34a2c,
4094                 0x34a44, 0x34a50,
4095                 0x34a74, 0x34c24,
4096                 0x34d00, 0x34d00,
4097                 0x34d08, 0x34d14,
4098                 0x34d1c, 0x34d20,
4099                 0x34d3c, 0x34d50,
4100                 0x35200, 0x3520c,
4101                 0x35220, 0x35220,
4102                 0x35240, 0x35240,
4103                 0x35600, 0x3560c,
4104                 0x35a00, 0x35a1c,
4105                 0x35e00, 0x35e20,
4106                 0x35e38, 0x35e3c,
4107                 0x35e80, 0x35e80,
4108                 0x35e88, 0x35ea8,
4109                 0x35eb0, 0x35eb4,
4110                 0x35ec8, 0x35ed4,
4111                 0x35fb8, 0x36004,
4112                 0x36200, 0x36200,
4113                 0x36208, 0x36240,
4114                 0x36248, 0x36280,
4115                 0x36288, 0x362c0,
4116                 0x362c8, 0x362fc,
4117                 0x36600, 0x36630,
4118                 0x36a00, 0x36abc,
4119                 0x36b00, 0x36b70,
4120                 0x37000, 0x37048,
4121                 0x37060, 0x3709c,
4122                 0x370f0, 0x37148,
4123                 0x37160, 0x3719c,
4124                 0x371f0, 0x372e4,
4125                 0x372f8, 0x373e4,
4126                 0x373f8, 0x37448,
4127                 0x37460, 0x3749c,
4128                 0x374f0, 0x37548,
4129                 0x37560, 0x3759c,
4130                 0x375f0, 0x376e4,
4131                 0x376f8, 0x377e4,
4132                 0x377f8, 0x377fc,
4133                 0x37814, 0x37814,
4134                 0x3782c, 0x3782c,
4135                 0x37880, 0x3788c,
4136                 0x378e8, 0x378ec,
4137                 0x37900, 0x37948,
4138                 0x37960, 0x3799c,
4139                 0x379f0, 0x37ae4,
4140                 0x37af8, 0x37b10,
4141                 0x37b28, 0x37b28,
4142                 0x37b3c, 0x37b50,
4143                 0x37bf0, 0x37c10,
4144                 0x37c28, 0x37c28,
4145                 0x37c3c, 0x37c50,
4146                 0x37cf0, 0x37cfc,
4147                 0x38000, 0x38030,
4148                 0x38100, 0x38144,
4149                 0x38190, 0x381d0,
4150                 0x38200, 0x38318,
4151                 0x38400, 0x3852c,
4152                 0x38540, 0x3861c,
4153                 0x38800, 0x38834,
4154                 0x388c0, 0x38908,
4155                 0x38910, 0x389ac,
4156                 0x38a00, 0x38a2c,
4157                 0x38a44, 0x38a50,
4158                 0x38a74, 0x38c24,
4159                 0x38d00, 0x38d00,
4160                 0x38d08, 0x38d14,
4161                 0x38d1c, 0x38d20,
4162                 0x38d3c, 0x38d50,
4163                 0x39200, 0x3920c,
4164                 0x39220, 0x39220,
4165                 0x39240, 0x39240,
4166                 0x39600, 0x3960c,
4167                 0x39a00, 0x39a1c,
4168                 0x39e00, 0x39e20,
4169                 0x39e38, 0x39e3c,
4170                 0x39e80, 0x39e80,
4171                 0x39e88, 0x39ea8,
4172                 0x39eb0, 0x39eb4,
4173                 0x39ec8, 0x39ed4,
4174                 0x39fb8, 0x3a004,
4175                 0x3a200, 0x3a200,
4176                 0x3a208, 0x3a240,
4177                 0x3a248, 0x3a280,
4178                 0x3a288, 0x3a2c0,
4179                 0x3a2c8, 0x3a2fc,
4180                 0x3a600, 0x3a630,
4181                 0x3aa00, 0x3aabc,
4182                 0x3ab00, 0x3ab70,
4183                 0x3b000, 0x3b048,
4184                 0x3b060, 0x3b09c,
4185                 0x3b0f0, 0x3b148,
4186                 0x3b160, 0x3b19c,
4187                 0x3b1f0, 0x3b2e4,
4188                 0x3b2f8, 0x3b3e4,
4189                 0x3b3f8, 0x3b448,
4190                 0x3b460, 0x3b49c,
4191                 0x3b4f0, 0x3b548,
4192                 0x3b560, 0x3b59c,
4193                 0x3b5f0, 0x3b6e4,
4194                 0x3b6f8, 0x3b7e4,
4195                 0x3b7f8, 0x3b7fc,
4196                 0x3b814, 0x3b814,
4197                 0x3b82c, 0x3b82c,
4198                 0x3b880, 0x3b88c,
4199                 0x3b8e8, 0x3b8ec,
4200                 0x3b900, 0x3b948,
4201                 0x3b960, 0x3b99c,
4202                 0x3b9f0, 0x3bae4,
4203                 0x3baf8, 0x3bb10,
4204                 0x3bb28, 0x3bb28,
4205                 0x3bb3c, 0x3bb50,
4206                 0x3bbf0, 0x3bc10,
4207                 0x3bc28, 0x3bc28,
4208                 0x3bc3c, 0x3bc50,
4209                 0x3bcf0, 0x3bcfc,
4210                 0x3c000, 0x3c030,
4211                 0x3c100, 0x3c144,
4212                 0x3c190, 0x3c1d0,
4213                 0x3c200, 0x3c318,
4214                 0x3c400, 0x3c52c,
4215                 0x3c540, 0x3c61c,
4216                 0x3c800, 0x3c834,
4217                 0x3c8c0, 0x3c908,
4218                 0x3c910, 0x3c9ac,
4219                 0x3ca00, 0x3ca2c,
4220                 0x3ca44, 0x3ca50,
4221                 0x3ca74, 0x3cc24,
4222                 0x3cd00, 0x3cd00,
4223                 0x3cd08, 0x3cd14,
4224                 0x3cd1c, 0x3cd20,
4225                 0x3cd3c, 0x3cd50,
4226                 0x3d200, 0x3d20c,
4227                 0x3d220, 0x3d220,
4228                 0x3d240, 0x3d240,
4229                 0x3d600, 0x3d60c,
4230                 0x3da00, 0x3da1c,
4231                 0x3de00, 0x3de20,
4232                 0x3de38, 0x3de3c,
4233                 0x3de80, 0x3de80,
4234                 0x3de88, 0x3dea8,
4235                 0x3deb0, 0x3deb4,
4236                 0x3dec8, 0x3ded4,
4237                 0x3dfb8, 0x3e004,
4238                 0x3e200, 0x3e200,
4239                 0x3e208, 0x3e240,
4240                 0x3e248, 0x3e280,
4241                 0x3e288, 0x3e2c0,
4242                 0x3e2c8, 0x3e2fc,
4243                 0x3e600, 0x3e630,
4244                 0x3ea00, 0x3eabc,
4245                 0x3eb00, 0x3eb70,
4246                 0x3f000, 0x3f048,
4247                 0x3f060, 0x3f09c,
4248                 0x3f0f0, 0x3f148,
4249                 0x3f160, 0x3f19c,
4250                 0x3f1f0, 0x3f2e4,
4251                 0x3f2f8, 0x3f3e4,
4252                 0x3f3f8, 0x3f448,
4253                 0x3f460, 0x3f49c,
4254                 0x3f4f0, 0x3f548,
4255                 0x3f560, 0x3f59c,
4256                 0x3f5f0, 0x3f6e4,
4257                 0x3f6f8, 0x3f7e4,
4258                 0x3f7f8, 0x3f7fc,
4259                 0x3f814, 0x3f814,
4260                 0x3f82c, 0x3f82c,
4261                 0x3f880, 0x3f88c,
4262                 0x3f8e8, 0x3f8ec,
4263                 0x3f900, 0x3f948,
4264                 0x3f960, 0x3f99c,
4265                 0x3f9f0, 0x3fae4,
4266                 0x3faf8, 0x3fb10,
4267                 0x3fb28, 0x3fb28,
4268                 0x3fb3c, 0x3fb50,
4269                 0x3fbf0, 0x3fc10,
4270                 0x3fc28, 0x3fc28,
4271                 0x3fc3c, 0x3fc50,
4272                 0x3fcf0, 0x3fcfc,
4273                 0x40000, 0x4000c,
4274                 0x40040, 0x40068,
4275                 0x4007c, 0x40144,
4276                 0x40180, 0x4018c,
4277                 0x40200, 0x40298,
4278                 0x402ac, 0x4033c,
4279                 0x403f8, 0x403fc,
4280                 0x41304, 0x413c4,
4281                 0x41400, 0x4141c,
4282                 0x41480, 0x414d0,
4283                 0x44000, 0x44078,
4284                 0x440c0, 0x44278,
4285                 0x442c0, 0x44478,
4286                 0x444c0, 0x44678,
4287                 0x446c0, 0x44878,
4288                 0x448c0, 0x449fc,
4289                 0x45000, 0x45068,
4290                 0x45080, 0x45084,
4291                 0x450a0, 0x450b0,
4292                 0x45200, 0x45268,
4293                 0x45280, 0x45284,
4294                 0x452a0, 0x452b0,
4295                 0x460c0, 0x460e4,
4296                 0x47000, 0x4708c,
4297                 0x47200, 0x47250,
4298                 0x47400, 0x47420,
4299                 0x47600, 0x47618,
4300                 0x47800, 0x47814,
4301                 0x48000, 0x4800c,
4302                 0x48040, 0x48068,
4303                 0x4807c, 0x48144,
4304                 0x48180, 0x4818c,
4305                 0x48200, 0x48298,
4306                 0x482ac, 0x4833c,
4307                 0x483f8, 0x483fc,
4308                 0x49304, 0x493c4,
4309                 0x49400, 0x4941c,
4310                 0x49480, 0x494d0,
4311                 0x4c000, 0x4c078,
4312                 0x4c0c0, 0x4c278,
4313                 0x4c2c0, 0x4c478,
4314                 0x4c4c0, 0x4c678,
4315                 0x4c6c0, 0x4c878,
4316                 0x4c8c0, 0x4c9fc,
4317                 0x4d000, 0x4d068,
4318                 0x4d080, 0x4d084,
4319                 0x4d0a0, 0x4d0b0,
4320                 0x4d200, 0x4d268,
4321                 0x4d280, 0x4d284,
4322                 0x4d2a0, 0x4d2b0,
4323                 0x4e0c0, 0x4e0e4,
4324                 0x4f000, 0x4f08c,
4325                 0x4f200, 0x4f250,
4326                 0x4f400, 0x4f420,
4327                 0x4f600, 0x4f618,
4328                 0x4f800, 0x4f814,
4329                 0x50000, 0x500cc,
4330                 0x50400, 0x50400,
4331                 0x50800, 0x508cc,
4332                 0x50c00, 0x50c00,
4333                 0x51000, 0x5101c,
4334                 0x51300, 0x51308,
4335         };
4336
4337         if (is_t4(sc)) {
4338                 reg_ranges = &t4_reg_ranges[0];
4339                 n = nitems(t4_reg_ranges);
4340         } else {
4341                 reg_ranges = &t5_reg_ranges[0];
4342                 n = nitems(t5_reg_ranges);
4343         }
4344
4345         regs->version = chip_id(sc) | chip_rev(sc) << 10;
4346         for (i = 0; i < n; i += 2)
4347                 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
4348 }
4349
4350 static void
4351 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi)
4352 {
4353         int i;
4354         u_int v, tnl_cong_drops;
4355         struct timeval tv;
4356         const struct timeval interval = {0, 250000};    /* 250ms */
4357
4358         getmicrotime(&tv);
4359         timevalsub(&tv, &interval);
4360         if (timevalcmp(&tv, &pi->last_refreshed, <))
4361                 return;
4362
4363         tnl_cong_drops = 0;
4364         t4_get_port_stats(sc, pi->tx_chan, &pi->stats);
4365         for (i = 0; i < NCHAN; i++) {
4366                 if (pi->rx_chan_map & (1 << i)) {
4367                         mtx_lock(&sc->regwin_lock);
4368                         t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
4369                             1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4370                         mtx_unlock(&sc->regwin_lock);
4371                         tnl_cong_drops += v;
4372                 }
4373         }
4374         pi->tnl_cong_drops = tnl_cong_drops;
4375         getmicrotime(&pi->last_refreshed);
4376 }
4377
4378 static void
4379 cxgbe_tick(void *arg)
4380 {
4381         struct port_info *pi = arg;
4382         struct adapter *sc = pi->adapter;
4383         struct ifnet *ifp = pi->ifp;
4384
4385         PORT_LOCK(pi);
4386         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4387                 PORT_UNLOCK(pi);
4388                 return; /* without scheduling another callout */
4389         }
4390
4391         cxgbe_refresh_stats(sc, pi);
4392
4393         callout_schedule(&pi->tick, hz);
4394         PORT_UNLOCK(pi);
4395 }
4396
4397 static void
4398 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4399 {
4400         struct ifnet *vlan;
4401
4402         if (arg != ifp || ifp->if_type != IFT_ETHER)
4403                 return;
4404
4405         vlan = VLAN_DEVAT(ifp, vid);
4406         VLAN_SETCOOKIE(vlan, ifp);
4407 }
4408
4409 static int
4410 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4411 {
4412
4413 #ifdef INVARIANTS
4414         panic("%s: opcode 0x%02x on iq %p with payload %p",
4415             __func__, rss->opcode, iq, m);
4416 #else
4417         log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
4418             __func__, rss->opcode, iq, m);
4419         m_freem(m);
4420 #endif
4421         return (EDOOFUS);
4422 }
4423
4424 int
4425 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
4426 {
4427         uintptr_t *loc, new;
4428
4429         if (opcode >= nitems(sc->cpl_handler))
4430                 return (EINVAL);
4431
4432         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
4433         loc = (uintptr_t *) &sc->cpl_handler[opcode];
4434         atomic_store_rel_ptr(loc, new);
4435
4436         return (0);
4437 }
4438
4439 static int
4440 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
4441 {
4442
4443 #ifdef INVARIANTS
4444         panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
4445 #else
4446         log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
4447             __func__, iq, ctrl);
4448 #endif
4449         return (EDOOFUS);
4450 }
4451
4452 int
4453 t4_register_an_handler(struct adapter *sc, an_handler_t h)
4454 {
4455         uintptr_t *loc, new;
4456
4457         new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
4458         loc = (uintptr_t *) &sc->an_handler;
4459         atomic_store_rel_ptr(loc, new);
4460
4461         return (0);
4462 }
4463
4464 static int
4465 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
4466 {
4467         const struct cpl_fw6_msg *cpl =
4468             __containerof(rpl, struct cpl_fw6_msg, data[0]);
4469
4470 #ifdef INVARIANTS
4471         panic("%s: fw_msg type %d", __func__, cpl->type);
4472 #else
4473         log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
4474 #endif
4475         return (EDOOFUS);
4476 }
4477
4478 int
4479 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
4480 {
4481         uintptr_t *loc, new;
4482
4483         if (type >= nitems(sc->fw_msg_handler))
4484                 return (EINVAL);
4485
4486         /*
4487          * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
4488          * handler dispatch table.  Reject any attempt to install a handler for
4489          * this subtype.
4490          */
4491         if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
4492                 return (EINVAL);
4493
4494         new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
4495         loc = (uintptr_t *) &sc->fw_msg_handler[type];
4496         atomic_store_rel_ptr(loc, new);
4497
4498         return (0);
4499 }
4500
4501 static int
4502 t4_sysctls(struct adapter *sc)
4503 {
4504         struct sysctl_ctx_list *ctx;
4505         struct sysctl_oid *oid;
4506         struct sysctl_oid_list *children, *c0;
4507         static char *caps[] = {
4508                 "\20\1PPP\2QFC\3DCBX",                  /* caps[0] linkcaps */
4509                 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL"        /* caps[1] niccaps */
4510                     "\6HASHFILTER\7ETHOFLD",
4511                 "\20\1TOE",                             /* caps[2] toecaps */
4512                 "\20\1RDDP\2RDMAC",                     /* caps[3] rdmacaps */
4513                 "\20\1INITIATOR_PDU\2TARGET_PDU"        /* caps[4] iscsicaps */
4514                     "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
4515                     "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
4516                 "\20\1INITIATOR\2TARGET\3CTRL_OFLD"     /* caps[5] fcoecaps */
4517                     "\4PO_INITIAOR\5PO_TARGET"
4518         };
4519         static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
4520
4521         ctx = device_get_sysctl_ctx(sc->dev);
4522
4523         /*
4524          * dev.t4nex.X.
4525          */
4526         oid = device_get_sysctl_tree(sc->dev);
4527         c0 = children = SYSCTL_CHILDREN(oid);
4528
4529         sc->sc_do_rxcopy = 1;
4530         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
4531             &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
4532
4533         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
4534             sc->params.nports, "# of ports");
4535
4536         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
4537             NULL, chip_rev(sc), "chip hardware revision");
4538
4539         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
4540             CTLFLAG_RD, sc->fw_version, 0, "firmware version");
4541
4542         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
4543             CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
4544
4545         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
4546             sc->cfcsum, "config file checksum");
4547
4548         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
4549             CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
4550             sysctl_bitfield, "A", "available doorbells");
4551
4552         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
4553             CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
4554             sysctl_bitfield, "A", "available link capabilities");
4555
4556         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
4557             CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
4558             sysctl_bitfield, "A", "available NIC capabilities");
4559
4560         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
4561             CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
4562             sysctl_bitfield, "A", "available TCP offload capabilities");
4563
4564         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
4565             CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
4566             sysctl_bitfield, "A", "available RDMA capabilities");
4567
4568         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
4569             CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
4570             sysctl_bitfield, "A", "available iSCSI capabilities");
4571
4572         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
4573             CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
4574             sysctl_bitfield, "A", "available FCoE capabilities");
4575
4576         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
4577             sc->params.vpd.cclk, "core clock frequency (in KHz)");
4578
4579         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
4580             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
4581             sizeof(sc->sge.timer_val), sysctl_int_array, "A",
4582             "interrupt holdoff timer values (us)");
4583
4584         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
4585             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
4586             sizeof(sc->sge.counter_val), sysctl_int_array, "A",
4587             "interrupt holdoff packet counter values");
4588
4589         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
4590             NULL, sc->tids.nftids, "number of filters");
4591
4592         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
4593             CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
4594             "chip temperature (in Celsius)");
4595
4596         t4_sge_sysctls(sc, ctx, children);
4597
4598         sc->lro_timeout = 100;
4599         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
4600             &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
4601
4602 #ifdef SBUF_DRAIN
4603         /*
4604          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
4605          */
4606         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
4607             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
4608             "logs and miscellaneous information");
4609         children = SYSCTL_CHILDREN(oid);
4610
4611         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
4612             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4613             sysctl_cctrl, "A", "congestion control");
4614
4615         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
4616             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4617             sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
4618
4619         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
4620             CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
4621             sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
4622
4623         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
4624             CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
4625             sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
4626
4627         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
4628             CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
4629             sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
4630
4631         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
4632             CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
4633             sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
4634
4635         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
4636             CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
4637             sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
4638
4639         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
4640             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4641             sysctl_cim_la, "A", "CIM logic analyzer");
4642
4643         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
4644             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4645             sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
4646
4647         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
4648             CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
4649             sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
4650
4651         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
4652             CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
4653             sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
4654
4655         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
4656             CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
4657             sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
4658
4659         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
4660             CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
4661             sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
4662
4663         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
4664             CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
4665             sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
4666
4667         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
4668             CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
4669             sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
4670
4671         if (is_t5(sc)) {
4672                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
4673                     CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
4674                     sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
4675
4676                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
4677                     CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
4678                     sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
4679         }
4680
4681         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
4682             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4683             sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
4684
4685         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
4686             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4687             sysctl_cim_qcfg, "A", "CIM queue configuration");
4688
4689         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
4690             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4691             sysctl_cpl_stats, "A", "CPL statistics");
4692
4693         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
4694             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4695             sysctl_ddp_stats, "A", "DDP statistics");
4696
4697         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
4698             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4699             sysctl_devlog, "A", "firmware's device log");
4700
4701         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
4702             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4703             sysctl_fcoe_stats, "A", "FCoE statistics");
4704
4705         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
4706             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4707             sysctl_hw_sched, "A", "hardware scheduler ");
4708
4709         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
4710             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4711             sysctl_l2t, "A", "hardware L2 table");
4712
4713         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
4714             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4715             sysctl_lb_stats, "A", "loopback statistics");
4716
4717         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
4718             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4719             sysctl_meminfo, "A", "memory regions");
4720
4721         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
4722             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4723             sysctl_mps_tcam, "A", "MPS TCAM entries");
4724
4725         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
4726             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4727             sysctl_path_mtus, "A", "path MTUs");
4728
4729         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
4730             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4731             sysctl_pm_stats, "A", "PM statistics");
4732
4733         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
4734             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4735             sysctl_rdma_stats, "A", "RDMA statistics");
4736
4737         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
4738             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4739             sysctl_tcp_stats, "A", "TCP statistics");
4740
4741         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
4742             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4743             sysctl_tids, "A", "TID information");
4744
4745         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
4746             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4747             sysctl_tp_err_stats, "A", "TP error statistics");
4748
4749         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
4750             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4751             sysctl_tp_la, "A", "TP logic analyzer");
4752
4753         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
4754             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4755             sysctl_tx_rate, "A", "Tx rate");
4756
4757         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
4758             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4759             sysctl_ulprx_la, "A", "ULPRX logic analyzer");
4760
4761         if (is_t5(sc)) {
4762                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
4763                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4764                     sysctl_wcwr_stats, "A", "write combined work requests");
4765         }
4766 #endif
4767
4768 #ifdef TCP_OFFLOAD
4769         if (is_offload(sc)) {
4770                 /*
4771                  * dev.t4nex.X.toe.
4772                  */
4773                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
4774                     NULL, "TOE parameters");
4775                 children = SYSCTL_CHILDREN(oid);
4776
4777                 sc->tt.sndbuf = 256 * 1024;
4778                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
4779                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
4780
4781                 sc->tt.ddp = 0;
4782                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
4783                     &sc->tt.ddp, 0, "DDP allowed");
4784
4785                 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
4786                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
4787                     &sc->tt.indsz, 0, "DDP max indicate size allowed");
4788
4789                 sc->tt.ddp_thres =
4790                     G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
4791                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
4792                     &sc->tt.ddp_thres, 0, "DDP threshold");
4793
4794                 sc->tt.rx_coalesce = 1;
4795                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
4796                     CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
4797         }
4798 #endif
4799
4800
4801         return (0);
4802 }
4803
4804 static int
4805 cxgbe_sysctls(struct port_info *pi)
4806 {
4807         struct sysctl_ctx_list *ctx;
4808         struct sysctl_oid *oid;
4809         struct sysctl_oid_list *children;
4810         struct adapter *sc = pi->adapter;
4811
4812         ctx = device_get_sysctl_ctx(pi->dev);
4813
4814         /*
4815          * dev.cxgbe.X.
4816          */
4817         oid = device_get_sysctl_tree(pi->dev);
4818         children = SYSCTL_CHILDREN(oid);
4819
4820         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
4821            CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
4822         if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
4823                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
4824                     CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
4825                     "PHY temperature (in Celsius)");
4826                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
4827                     CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
4828                     "PHY firmware version");
4829         }
4830         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
4831             &pi->nrxq, 0, "# of rx queues");
4832         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
4833             &pi->ntxq, 0, "# of tx queues");
4834         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
4835             &pi->first_rxq, 0, "index of first rx queue");
4836         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
4837             &pi->first_txq, 0, "index of first tx queue");
4838         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
4839             CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU",
4840             "Reserve queue 0 for non-flowid packets");
4841
4842 #ifdef TCP_OFFLOAD
4843         if (is_offload(sc)) {
4844                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
4845                     &pi->nofldrxq, 0,
4846                     "# of rx queues for offloaded TCP connections");
4847                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
4848                     &pi->nofldtxq, 0,
4849                     "# of tx queues for offloaded TCP connections");
4850                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
4851                     CTLFLAG_RD, &pi->first_ofld_rxq, 0,
4852                     "index of first TOE rx queue");
4853                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
4854                     CTLFLAG_RD, &pi->first_ofld_txq, 0,
4855                     "index of first TOE tx queue");
4856         }
4857 #endif
4858 #ifdef DEV_NETMAP
4859         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
4860             &pi->nnmrxq, 0, "# of rx queues for netmap");
4861         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
4862             &pi->nnmtxq, 0, "# of tx queues for netmap");
4863         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
4864             CTLFLAG_RD, &pi->first_nm_rxq, 0,
4865             "index of first netmap rx queue");
4866         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
4867             CTLFLAG_RD, &pi->first_nm_txq, 0,
4868             "index of first netmap tx queue");
4869 #endif
4870
4871         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
4872             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
4873             "holdoff timer index");
4874         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
4875             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
4876             "holdoff packet counter index");
4877
4878         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
4879             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
4880             "rx queue size");
4881         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
4882             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
4883             "tx queue size");
4884
4885         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
4886             CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings,
4887             "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
4888
4889         /*
4890          * dev.cxgbe.X.stats.
4891          */
4892         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
4893             NULL, "port statistics");
4894         children = SYSCTL_CHILDREN(oid);
4895
4896 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
4897         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
4898             CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
4899             sysctl_handle_t4_reg64, "QU", desc)
4900
4901         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
4902             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
4903         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
4904             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
4905         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
4906             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
4907         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
4908             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
4909         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
4910             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
4911         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
4912             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
4913         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
4914             "# of tx frames in this range",
4915             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
4916         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
4917             "# of tx frames in this range",
4918             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
4919         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
4920             "# of tx frames in this range",
4921             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
4922         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
4923             "# of tx frames in this range",
4924             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
4925         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
4926             "# of tx frames in this range",
4927             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
4928         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
4929             "# of tx frames in this range",
4930             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
4931         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
4932             "# of tx frames in this range",
4933             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
4934         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
4935             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
4936         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
4937             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
4938         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
4939             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
4940         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
4941             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
4942         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
4943             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
4944         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
4945             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
4946         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
4947             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
4948         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
4949             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
4950         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
4951             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
4952         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
4953             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
4954
4955         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
4956             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
4957         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
4958             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
4959         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
4960             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
4961         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
4962             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
4963         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
4964             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
4965         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
4966             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
4967         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
4968             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
4969         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
4970             "# of frames received with bad FCS",
4971             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
4972         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
4973             "# of frames received with length error",
4974             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
4975         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
4976             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
4977         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
4978             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
4979         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
4980             "# of rx frames in this range",
4981             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
4982         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
4983             "# of rx frames in this range",
4984             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
4985         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
4986             "# of rx frames in this range",
4987             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
4988         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
4989             "# of rx frames in this range",
4990             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
4991         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
4992             "# of rx frames in this range",
4993             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
4994         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
4995             "# of rx frames in this range",
4996             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
4997         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
4998             "# of rx frames in this range",
4999             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
5000         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
5001             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
5002         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
5003             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
5004         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
5005             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
5006         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
5007             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
5008         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
5009             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
5010         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
5011             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
5012         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
5013             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
5014         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
5015             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
5016         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
5017             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
5018
5019 #undef SYSCTL_ADD_T4_REG64
5020
5021 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
5022         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
5023             &pi->stats.name, desc)
5024
5025         /* We get these from port_stats and they may be stale by upto 1s */
5026         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
5027             "# drops due to buffer-group 0 overflows");
5028         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
5029             "# drops due to buffer-group 1 overflows");
5030         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
5031             "# drops due to buffer-group 2 overflows");
5032         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
5033             "# drops due to buffer-group 3 overflows");
5034         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
5035             "# of buffer-group 0 truncated packets");
5036         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
5037             "# of buffer-group 1 truncated packets");
5038         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
5039             "# of buffer-group 2 truncated packets");
5040         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
5041             "# of buffer-group 3 truncated packets");
5042
5043 #undef SYSCTL_ADD_T4_PORTSTAT
5044
5045         return (0);
5046 }
5047
5048 static int
5049 sysctl_int_array(SYSCTL_HANDLER_ARGS)
5050 {
5051         int rc, *i;
5052         struct sbuf sb;
5053
5054         sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
5055         for (i = arg1; arg2; arg2 -= sizeof(int), i++)
5056                 sbuf_printf(&sb, "%d ", *i);
5057         sbuf_trim(&sb);
5058         sbuf_finish(&sb);
5059         rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
5060         sbuf_delete(&sb);
5061         return (rc);
5062 }
5063
5064 static int
5065 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
5066 {
5067         int rc;
5068         struct sbuf *sb;
5069
5070         rc = sysctl_wire_old_buffer(req, 0);
5071         if (rc != 0)
5072                 return(rc);
5073
5074         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5075         if (sb == NULL)
5076                 return (ENOMEM);
5077
5078         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
5079         rc = sbuf_finish(sb);
5080         sbuf_delete(sb);
5081
5082         return (rc);
5083 }
5084
5085 static int
5086 sysctl_btphy(SYSCTL_HANDLER_ARGS)
5087 {
5088         struct port_info *pi = arg1;
5089         int op = arg2;
5090         struct adapter *sc = pi->adapter;
5091         u_int v;
5092         int rc;
5093
5094         rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt");
5095         if (rc)
5096                 return (rc);
5097         /* XXX: magic numbers */
5098         rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
5099             &v);
5100         end_synchronized_op(sc, 0);
5101         if (rc)
5102                 return (rc);
5103         if (op == 0)
5104                 v /= 256;
5105
5106         rc = sysctl_handle_int(oidp, &v, 0, req);
5107         return (rc);
5108 }
5109
5110 static int
5111 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
5112 {
5113         struct port_info *pi = arg1;
5114         int rc, val;
5115
5116         val = pi->rsrv_noflowq;
5117         rc = sysctl_handle_int(oidp, &val, 0, req);
5118         if (rc != 0 || req->newptr == NULL)
5119                 return (rc);
5120
5121         if ((val >= 1) && (pi->ntxq > 1))
5122                 pi->rsrv_noflowq = 1;
5123         else
5124                 pi->rsrv_noflowq = 0;
5125
5126         return (rc);
5127 }
5128
5129 static int
5130 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
5131 {
5132         struct port_info *pi = arg1;
5133         struct adapter *sc = pi->adapter;
5134         int idx, rc, i;
5135         struct sge_rxq *rxq;
5136 #ifdef TCP_OFFLOAD
5137         struct sge_ofld_rxq *ofld_rxq;
5138 #endif
5139         uint8_t v;
5140
5141         idx = pi->tmr_idx;
5142
5143         rc = sysctl_handle_int(oidp, &idx, 0, req);
5144         if (rc != 0 || req->newptr == NULL)
5145                 return (rc);
5146
5147         if (idx < 0 || idx >= SGE_NTIMERS)
5148                 return (EINVAL);
5149
5150         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5151             "t4tmr");
5152         if (rc)
5153                 return (rc);
5154
5155         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
5156         for_each_rxq(pi, i, rxq) {
5157 #ifdef atomic_store_rel_8
5158                 atomic_store_rel_8(&rxq->iq.intr_params, v);
5159 #else
5160                 rxq->iq.intr_params = v;
5161 #endif
5162         }
5163 #ifdef TCP_OFFLOAD
5164         for_each_ofld_rxq(pi, i, ofld_rxq) {
5165 #ifdef atomic_store_rel_8
5166                 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
5167 #else
5168                 ofld_rxq->iq.intr_params = v;
5169 #endif
5170         }
5171 #endif
5172         pi->tmr_idx = idx;
5173
5174         end_synchronized_op(sc, LOCK_HELD);
5175         return (0);
5176 }
5177
5178 static int
5179 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
5180 {
5181         struct port_info *pi = arg1;
5182         struct adapter *sc = pi->adapter;
5183         int idx, rc;
5184
5185         idx = pi->pktc_idx;
5186
5187         rc = sysctl_handle_int(oidp, &idx, 0, req);
5188         if (rc != 0 || req->newptr == NULL)
5189                 return (rc);
5190
5191         if (idx < -1 || idx >= SGE_NCOUNTERS)
5192                 return (EINVAL);
5193
5194         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5195             "t4pktc");
5196         if (rc)
5197                 return (rc);
5198
5199         if (pi->flags & PORT_INIT_DONE)
5200                 rc = EBUSY; /* cannot be changed once the queues are created */
5201         else
5202                 pi->pktc_idx = idx;
5203
5204         end_synchronized_op(sc, LOCK_HELD);
5205         return (rc);
5206 }
5207
5208 static int
5209 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
5210 {
5211         struct port_info *pi = arg1;
5212         struct adapter *sc = pi->adapter;
5213         int qsize, rc;
5214
5215         qsize = pi->qsize_rxq;
5216
5217         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5218         if (rc != 0 || req->newptr == NULL)
5219                 return (rc);
5220
5221         if (qsize < 128 || (qsize & 7))
5222                 return (EINVAL);
5223
5224         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5225             "t4rxqs");
5226         if (rc)
5227                 return (rc);
5228
5229         if (pi->flags & PORT_INIT_DONE)
5230                 rc = EBUSY; /* cannot be changed once the queues are created */
5231         else
5232                 pi->qsize_rxq = qsize;
5233
5234         end_synchronized_op(sc, LOCK_HELD);
5235         return (rc);
5236 }
5237
5238 static int
5239 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
5240 {
5241         struct port_info *pi = arg1;
5242         struct adapter *sc = pi->adapter;
5243         int qsize, rc;
5244
5245         qsize = pi->qsize_txq;
5246
5247         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5248         if (rc != 0 || req->newptr == NULL)
5249                 return (rc);
5250
5251         /* bufring size must be powerof2 */
5252         if (qsize < 128 || !powerof2(qsize))
5253                 return (EINVAL);
5254
5255         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5256             "t4txqs");
5257         if (rc)
5258                 return (rc);
5259
5260         if (pi->flags & PORT_INIT_DONE)
5261                 rc = EBUSY; /* cannot be changed once the queues are created */
5262         else
5263                 pi->qsize_txq = qsize;
5264
5265         end_synchronized_op(sc, LOCK_HELD);
5266         return (rc);
5267 }
5268
5269 static int
5270 sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
5271 {
5272         struct port_info *pi = arg1;
5273         struct adapter *sc = pi->adapter;
5274         struct link_config *lc = &pi->link_cfg;
5275         int rc;
5276
5277         if (req->newptr == NULL) {
5278                 struct sbuf *sb;
5279                 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX";
5280
5281                 rc = sysctl_wire_old_buffer(req, 0);
5282                 if (rc != 0)
5283                         return(rc);
5284
5285                 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5286                 if (sb == NULL)
5287                         return (ENOMEM);
5288
5289                 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits);
5290                 rc = sbuf_finish(sb);
5291                 sbuf_delete(sb);
5292         } else {
5293                 char s[2];
5294                 int n;
5295
5296                 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX));
5297                 s[1] = 0;
5298
5299                 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
5300                 if (rc != 0)
5301                         return(rc);
5302
5303                 if (s[1] != 0)
5304                         return (EINVAL);
5305                 if (s[0] < '0' || s[0] > '9')
5306                         return (EINVAL);        /* not a number */
5307                 n = s[0] - '0';
5308                 if (n & ~(PAUSE_TX | PAUSE_RX))
5309                         return (EINVAL);        /* some other bit is set too */
5310
5311                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE");
5312                 if (rc)
5313                         return (rc);
5314                 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) {
5315                         int link_ok = lc->link_ok;
5316
5317                         lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX);
5318                         lc->requested_fc |= n;
5319                         rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc);
5320                         lc->link_ok = link_ok;  /* restore */
5321                 }
5322                 end_synchronized_op(sc, 0);
5323         }
5324
5325         return (rc);
5326 }
5327
5328 static int
5329 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
5330 {
5331         struct adapter *sc = arg1;
5332         int reg = arg2;
5333         uint64_t val;
5334
5335         val = t4_read_reg64(sc, reg);
5336
5337         return (sysctl_handle_64(oidp, &val, 0, req));
5338 }
5339
5340 static int
5341 sysctl_temperature(SYSCTL_HANDLER_ARGS)
5342 {
5343         struct adapter *sc = arg1;
5344         int rc, t;
5345         uint32_t param, val;
5346
5347         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
5348         if (rc)
5349                 return (rc);
5350         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5351             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5352             V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
5353         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
5354         end_synchronized_op(sc, 0);
5355         if (rc)
5356                 return (rc);
5357
5358         /* unknown is returned as 0 but we display -1 in that case */
5359         t = val == 0 ? -1 : val;
5360
5361         rc = sysctl_handle_int(oidp, &t, 0, req);
5362         return (rc);
5363 }
5364
5365 #ifdef SBUF_DRAIN
5366 static int
5367 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
5368 {
5369         struct adapter *sc = arg1;
5370         struct sbuf *sb;
5371         int rc, i;
5372         uint16_t incr[NMTUS][NCCTRL_WIN];
5373         static const char *dec_fac[] = {
5374                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
5375                 "0.9375"
5376         };
5377
5378         rc = sysctl_wire_old_buffer(req, 0);
5379         if (rc != 0)
5380                 return (rc);
5381
5382         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5383         if (sb == NULL)
5384                 return (ENOMEM);
5385
5386         t4_read_cong_tbl(sc, incr);
5387
5388         for (i = 0; i < NCCTRL_WIN; ++i) {
5389                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
5390                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
5391                     incr[5][i], incr[6][i], incr[7][i]);
5392                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
5393                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
5394                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
5395                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
5396         }
5397
5398         rc = sbuf_finish(sb);
5399         sbuf_delete(sb);
5400
5401         return (rc);
5402 }
5403
5404 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
5405         "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",   /* ibq's */
5406         "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
5407         "SGE0-RX", "SGE1-RX"    /* additional obq's (T5 onwards) */
5408 };
5409
5410 static int
5411 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
5412 {
5413         struct adapter *sc = arg1;
5414         struct sbuf *sb;
5415         int rc, i, n, qid = arg2;
5416         uint32_t *buf, *p;
5417         char *qtype;
5418         u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
5419
5420         KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
5421             ("%s: bad qid %d\n", __func__, qid));
5422
5423         if (qid < CIM_NUM_IBQ) {
5424                 /* inbound queue */
5425                 qtype = "IBQ";
5426                 n = 4 * CIM_IBQ_SIZE;
5427                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5428                 rc = t4_read_cim_ibq(sc, qid, buf, n);
5429         } else {
5430                 /* outbound queue */
5431                 qtype = "OBQ";
5432                 qid -= CIM_NUM_IBQ;
5433                 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
5434                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5435                 rc = t4_read_cim_obq(sc, qid, buf, n);
5436         }
5437
5438         if (rc < 0) {
5439                 rc = -rc;
5440                 goto done;
5441         }
5442         n = rc * sizeof(uint32_t);      /* rc has # of words actually read */
5443
5444         rc = sysctl_wire_old_buffer(req, 0);
5445         if (rc != 0)
5446                 goto done;
5447
5448         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5449         if (sb == NULL) {
5450                 rc = ENOMEM;
5451                 goto done;
5452         }
5453
5454         sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
5455         for (i = 0, p = buf; i < n; i += 16, p += 4)
5456                 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
5457                     p[2], p[3]);
5458
5459         rc = sbuf_finish(sb);
5460         sbuf_delete(sb);
5461 done:
5462         free(buf, M_CXGBE);
5463         return (rc);
5464 }
5465
5466 static int
5467 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
5468 {
5469         struct adapter *sc = arg1;
5470         u_int cfg;
5471         struct sbuf *sb;
5472         uint32_t *buf, *p;
5473         int rc;
5474
5475         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
5476         if (rc != 0)
5477                 return (rc);
5478
5479         rc = sysctl_wire_old_buffer(req, 0);
5480         if (rc != 0)
5481                 return (rc);
5482
5483         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5484         if (sb == NULL)
5485                 return (ENOMEM);
5486
5487         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
5488             M_ZERO | M_WAITOK);
5489
5490         rc = -t4_cim_read_la(sc, buf, NULL);
5491         if (rc != 0)
5492                 goto done;
5493
5494         sbuf_printf(sb, "Status   Data      PC%s",
5495             cfg & F_UPDBGLACAPTPCONLY ? "" :
5496             "     LS0Stat  LS0Addr             LS0Data");
5497
5498         KASSERT((sc->params.cim_la_size & 7) == 0,
5499             ("%s: p will walk off the end of buf", __func__));
5500
5501         for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
5502                 if (cfg & F_UPDBGLACAPTPCONLY) {
5503                         sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
5504                             p[6], p[7]);
5505                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
5506                             (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
5507                             p[4] & 0xff, p[5] >> 8);
5508                         sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
5509                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5510                             p[1] & 0xf, p[2] >> 4);
5511                 } else {
5512                         sbuf_printf(sb,
5513                             "\n  %02x   %x%07x %x%07x %08x %08x "
5514                             "%08x%08x%08x%08x",
5515                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5516                             p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
5517                             p[6], p[7]);
5518                 }
5519         }
5520
5521         rc = sbuf_finish(sb);
5522         sbuf_delete(sb);
5523 done:
5524         free(buf, M_CXGBE);
5525         return (rc);
5526 }
5527
5528 static int
5529 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
5530 {
5531         struct adapter *sc = arg1;
5532         u_int i;
5533         struct sbuf *sb;
5534         uint32_t *buf, *p;
5535         int rc;
5536
5537         rc = sysctl_wire_old_buffer(req, 0);
5538         if (rc != 0)
5539                 return (rc);
5540
5541         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5542         if (sb == NULL)
5543                 return (ENOMEM);
5544
5545         buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
5546             M_ZERO | M_WAITOK);
5547
5548         t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
5549         p = buf;
5550
5551         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5552                 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
5553                     p[1], p[0]);
5554         }
5555
5556         sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
5557         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5558                 sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
5559                     (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
5560                     (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
5561                     (p[1] >> 2) | ((p[2] & 3) << 30),
5562                     (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
5563                     p[0] & 1);
5564         }
5565
5566         rc = sbuf_finish(sb);
5567         sbuf_delete(sb);
5568         free(buf, M_CXGBE);
5569         return (rc);
5570 }
5571
5572 static int
5573 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
5574 {
5575         struct adapter *sc = arg1;
5576         u_int i;
5577         struct sbuf *sb;
5578         uint32_t *buf, *p;
5579         int rc;
5580
5581         rc = sysctl_wire_old_buffer(req, 0);
5582         if (rc != 0)
5583                 return (rc);
5584
5585         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5586         if (sb == NULL)
5587                 return (ENOMEM);
5588
5589         buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
5590             M_ZERO | M_WAITOK);
5591
5592         t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
5593         p = buf;
5594
5595         sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
5596         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5597                 sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
5598                     (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
5599                     p[4], p[3], p[2], p[1], p[0]);
5600         }
5601
5602         sbuf_printf(sb, "\n\nCntl ID               Data");
5603         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5604                 sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
5605                     (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
5606         }
5607
5608         rc = sbuf_finish(sb);
5609         sbuf_delete(sb);
5610         free(buf, M_CXGBE);
5611         return (rc);
5612 }
5613
5614 static int
5615 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
5616 {
5617         struct adapter *sc = arg1;
5618         struct sbuf *sb;
5619         int rc, i;
5620         uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5621         uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5622         uint16_t thres[CIM_NUM_IBQ];
5623         uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
5624         uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
5625         u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
5626
5627         if (is_t4(sc)) {
5628                 cim_num_obq = CIM_NUM_OBQ;
5629                 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
5630                 obq_rdaddr = A_UP_OBQ_0_REALADDR;
5631         } else {
5632                 cim_num_obq = CIM_NUM_OBQ_T5;
5633                 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
5634                 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
5635         }
5636         nq = CIM_NUM_IBQ + cim_num_obq;
5637
5638         rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
5639         if (rc == 0)
5640                 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
5641         if (rc != 0)
5642                 return (rc);
5643
5644         t4_read_cimq_cfg(sc, base, size, thres);
5645
5646         rc = sysctl_wire_old_buffer(req, 0);
5647         if (rc != 0)
5648                 return (rc);
5649
5650         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5651         if (sb == NULL)
5652                 return (ENOMEM);
5653
5654         sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
5655
5656         for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
5657                 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
5658                     qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
5659                     G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5660                     G_QUEREMFLITS(p[2]) * 16);
5661         for ( ; i < nq; i++, p += 4, wr += 2)
5662                 sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
5663                     base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
5664                     wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5665                     G_QUEREMFLITS(p[2]) * 16);
5666
5667         rc = sbuf_finish(sb);
5668         sbuf_delete(sb);
5669
5670         return (rc);
5671 }
5672
5673 static int
5674 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
5675 {
5676         struct adapter *sc = arg1;
5677         struct sbuf *sb;
5678         int rc;
5679         struct tp_cpl_stats stats;
5680
5681         rc = sysctl_wire_old_buffer(req, 0);
5682         if (rc != 0)
5683                 return (rc);
5684
5685         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5686         if (sb == NULL)
5687                 return (ENOMEM);
5688
5689         t4_tp_get_cpl_stats(sc, &stats);
5690
5691         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
5692             "channel 3\n");
5693         sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
5694                    stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
5695         sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
5696                    stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
5697
5698         rc = sbuf_finish(sb);
5699         sbuf_delete(sb);
5700
5701         return (rc);
5702 }
5703
5704 static int
5705 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
5706 {
5707         struct adapter *sc = arg1;
5708         struct sbuf *sb;
5709         int rc;
5710         struct tp_usm_stats stats;
5711
5712         rc = sysctl_wire_old_buffer(req, 0);
5713         if (rc != 0)
5714                 return(rc);
5715
5716         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5717         if (sb == NULL)
5718                 return (ENOMEM);
5719
5720         t4_get_usm_stats(sc, &stats);
5721
5722         sbuf_printf(sb, "Frames: %u\n", stats.frames);
5723         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
5724         sbuf_printf(sb, "Drops:  %u", stats.drops);
5725
5726         rc = sbuf_finish(sb);
5727         sbuf_delete(sb);
5728
5729         return (rc);
5730 }
5731
5732 const char *devlog_level_strings[] = {
5733         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
5734         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
5735         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
5736         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
5737         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
5738         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
5739 };
5740
5741 const char *devlog_facility_strings[] = {
5742         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
5743         [FW_DEVLOG_FACILITY_CF]         = "CF",
5744         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
5745         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
5746         [FW_DEVLOG_FACILITY_RES]        = "RES",
5747         [FW_DEVLOG_FACILITY_HW]         = "HW",
5748         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
5749         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
5750         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
5751         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
5752         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
5753         [FW_DEVLOG_FACILITY_VI]         = "VI",
5754         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
5755         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
5756         [FW_DEVLOG_FACILITY_TM]         = "TM",
5757         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
5758         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
5759         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
5760         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
5761         [FW_DEVLOG_FACILITY_RI]         = "RI",
5762         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
5763         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
5764         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
5765         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE"
5766 };
5767
5768 static int
5769 sysctl_devlog(SYSCTL_HANDLER_ARGS)
5770 {
5771         struct adapter *sc = arg1;
5772         struct devlog_params *dparams = &sc->params.devlog;
5773         struct fw_devlog_e *buf, *e;
5774         int i, j, rc, nentries, first = 0, m;
5775         struct sbuf *sb;
5776         uint64_t ftstamp = UINT64_MAX;
5777
5778         if (dparams->start == 0) {
5779                 dparams->memtype = FW_MEMTYPE_EDC0;
5780                 dparams->start = 0x84000;
5781                 dparams->size = 32768;
5782         }
5783
5784         nentries = dparams->size / sizeof(struct fw_devlog_e);
5785
5786         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
5787         if (buf == NULL)
5788                 return (ENOMEM);
5789
5790         m = fwmtype_to_hwmtype(dparams->memtype);
5791         rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
5792         if (rc != 0)
5793                 goto done;
5794
5795         for (i = 0; i < nentries; i++) {
5796                 e = &buf[i];
5797
5798                 if (e->timestamp == 0)
5799                         break;  /* end */
5800
5801                 e->timestamp = be64toh(e->timestamp);
5802                 e->seqno = be32toh(e->seqno);
5803                 for (j = 0; j < 8; j++)
5804                         e->params[j] = be32toh(e->params[j]);
5805
5806                 if (e->timestamp < ftstamp) {
5807                         ftstamp = e->timestamp;
5808                         first = i;
5809                 }
5810         }
5811
5812         if (buf[first].timestamp == 0)
5813                 goto done;      /* nothing in the log */
5814
5815         rc = sysctl_wire_old_buffer(req, 0);
5816         if (rc != 0)
5817                 goto done;
5818
5819         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5820         if (sb == NULL) {
5821                 rc = ENOMEM;
5822                 goto done;
5823         }
5824         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
5825             "Seq#", "Tstamp", "Level", "Facility", "Message");
5826
5827         i = first;
5828         do {
5829                 e = &buf[i];
5830                 if (e->timestamp == 0)
5831                         break;  /* end */
5832
5833                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
5834                     e->seqno, e->timestamp,
5835                     (e->level < nitems(devlog_level_strings) ?
5836                         devlog_level_strings[e->level] : "UNKNOWN"),
5837                     (e->facility < nitems(devlog_facility_strings) ?
5838                         devlog_facility_strings[e->facility] : "UNKNOWN"));
5839                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
5840                     e->params[2], e->params[3], e->params[4],
5841                     e->params[5], e->params[6], e->params[7]);
5842
5843                 if (++i == nentries)
5844                         i = 0;
5845         } while (i != first);
5846
5847         rc = sbuf_finish(sb);
5848         sbuf_delete(sb);
5849 done:
5850         free(buf, M_CXGBE);
5851         return (rc);
5852 }
5853
5854 static int
5855 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
5856 {
5857         struct adapter *sc = arg1;
5858         struct sbuf *sb;
5859         int rc;
5860         struct tp_fcoe_stats stats[4];
5861
5862         rc = sysctl_wire_old_buffer(req, 0);
5863         if (rc != 0)
5864                 return (rc);
5865
5866         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5867         if (sb == NULL)
5868                 return (ENOMEM);
5869
5870         t4_get_fcoe_stats(sc, 0, &stats[0]);
5871         t4_get_fcoe_stats(sc, 1, &stats[1]);
5872         t4_get_fcoe_stats(sc, 2, &stats[2]);
5873         t4_get_fcoe_stats(sc, 3, &stats[3]);
5874
5875         sbuf_printf(sb, "                   channel 0        channel 1        "
5876             "channel 2        channel 3\n");
5877         sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
5878             stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
5879             stats[3].octetsDDP);
5880         sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
5881             stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
5882         sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
5883             stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
5884             stats[3].framesDrop);
5885
5886         rc = sbuf_finish(sb);
5887         sbuf_delete(sb);
5888
5889         return (rc);
5890 }
5891
5892 static int
5893 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
5894 {
5895         struct adapter *sc = arg1;
5896         struct sbuf *sb;
5897         int rc, i;
5898         unsigned int map, kbps, ipg, mode;
5899         unsigned int pace_tab[NTX_SCHED];
5900
5901         rc = sysctl_wire_old_buffer(req, 0);
5902         if (rc != 0)
5903                 return (rc);
5904
5905         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5906         if (sb == NULL)
5907                 return (ENOMEM);
5908
5909         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
5910         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
5911         t4_read_pace_tbl(sc, pace_tab);
5912
5913         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
5914             "Class IPG (0.1 ns)   Flow IPG (us)");
5915
5916         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
5917                 t4_get_tx_sched(sc, i, &kbps, &ipg);
5918                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
5919                     (mode & (1 << i)) ? "flow" : "class", map & 3);
5920                 if (kbps)
5921                         sbuf_printf(sb, "%9u     ", kbps);
5922                 else
5923                         sbuf_printf(sb, " disabled     ");
5924
5925                 if (ipg)
5926                         sbuf_printf(sb, "%13u        ", ipg);
5927                 else
5928                         sbuf_printf(sb, "     disabled        ");
5929
5930                 if (pace_tab[i])
5931                         sbuf_printf(sb, "%10u", pace_tab[i]);
5932                 else
5933                         sbuf_printf(sb, "  disabled");
5934         }
5935
5936         rc = sbuf_finish(sb);
5937         sbuf_delete(sb);
5938
5939         return (rc);
5940 }
5941
5942 static int
5943 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
5944 {
5945         struct adapter *sc = arg1;
5946         struct sbuf *sb;
5947         int rc, i, j;
5948         uint64_t *p0, *p1;
5949         struct lb_port_stats s[2];
5950         static const char *stat_name[] = {
5951                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
5952                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
5953                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
5954                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
5955                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
5956                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
5957                 "BG2FramesTrunc:", "BG3FramesTrunc:"
5958         };
5959
5960         rc = sysctl_wire_old_buffer(req, 0);
5961         if (rc != 0)
5962                 return (rc);
5963
5964         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5965         if (sb == NULL)
5966                 return (ENOMEM);
5967
5968         memset(s, 0, sizeof(s));
5969
5970         for (i = 0; i < 4; i += 2) {
5971                 t4_get_lb_stats(sc, i, &s[0]);
5972                 t4_get_lb_stats(sc, i + 1, &s[1]);
5973
5974                 p0 = &s[0].octets;
5975                 p1 = &s[1].octets;
5976                 sbuf_printf(sb, "%s                       Loopback %u"
5977                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
5978
5979                 for (j = 0; j < nitems(stat_name); j++)
5980                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
5981                                    *p0++, *p1++);
5982         }
5983
5984         rc = sbuf_finish(sb);
5985         sbuf_delete(sb);
5986
5987         return (rc);
5988 }
5989
5990 static int
5991 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
5992 {
5993         int rc = 0;
5994         struct port_info *pi = arg1;
5995         struct sbuf *sb;
5996         static const char *linkdnreasons[] = {
5997                 "non-specific", "remote fault", "autoneg failed", "reserved3",
5998                 "PHY overheated", "unknown", "rx los", "reserved7"
5999         };
6000
6001         rc = sysctl_wire_old_buffer(req, 0);
6002         if (rc != 0)
6003                 return(rc);
6004         sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
6005         if (sb == NULL)
6006                 return (ENOMEM);
6007
6008         if (pi->linkdnrc < 0)
6009                 sbuf_printf(sb, "n/a");
6010         else if (pi->linkdnrc < nitems(linkdnreasons))
6011                 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
6012         else
6013                 sbuf_printf(sb, "%d", pi->linkdnrc);
6014
6015         rc = sbuf_finish(sb);
6016         sbuf_delete(sb);
6017
6018         return (rc);
6019 }
6020
6021 struct mem_desc {
6022         unsigned int base;
6023         unsigned int limit;
6024         unsigned int idx;
6025 };
6026
6027 static int
6028 mem_desc_cmp(const void *a, const void *b)
6029 {
6030         return ((const struct mem_desc *)a)->base -
6031                ((const struct mem_desc *)b)->base;
6032 }
6033
6034 static void
6035 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
6036     unsigned int to)
6037 {
6038         unsigned int size;
6039
6040         size = to - from + 1;
6041         if (size == 0)
6042                 return;
6043
6044         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
6045         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
6046 }
6047
6048 static int
6049 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
6050 {
6051         struct adapter *sc = arg1;
6052         struct sbuf *sb;
6053         int rc, i, n;
6054         uint32_t lo, hi, used, alloc;
6055         static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
6056         static const char *region[] = {
6057                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
6058                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
6059                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
6060                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
6061                 "RQUDP region:", "PBL region:", "TXPBL region:",
6062                 "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
6063                 "On-chip queues:"
6064         };
6065         struct mem_desc avail[4];
6066         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
6067         struct mem_desc *md = mem;
6068
6069         rc = sysctl_wire_old_buffer(req, 0);
6070         if (rc != 0)
6071                 return (rc);
6072
6073         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6074         if (sb == NULL)
6075                 return (ENOMEM);
6076
6077         for (i = 0; i < nitems(mem); i++) {
6078                 mem[i].limit = 0;
6079                 mem[i].idx = i;
6080         }
6081
6082         /* Find and sort the populated memory ranges */
6083         i = 0;
6084         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
6085         if (lo & F_EDRAM0_ENABLE) {
6086                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
6087                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
6088                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
6089                 avail[i].idx = 0;
6090                 i++;
6091         }
6092         if (lo & F_EDRAM1_ENABLE) {
6093                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
6094                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
6095                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
6096                 avail[i].idx = 1;
6097                 i++;
6098         }
6099         if (lo & F_EXT_MEM_ENABLE) {
6100                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
6101                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
6102                 avail[i].limit = avail[i].base +
6103                     (G_EXT_MEM_SIZE(hi) << 20);
6104                 avail[i].idx = is_t4(sc) ? 2 : 3;       /* Call it MC for T4 */
6105                 i++;
6106         }
6107         if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
6108                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
6109                 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
6110                 avail[i].limit = avail[i].base +
6111                     (G_EXT_MEM1_SIZE(hi) << 20);
6112                 avail[i].idx = 4;
6113                 i++;
6114         }
6115         if (!i)                                    /* no memory available */
6116                 return 0;
6117         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
6118
6119         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
6120         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
6121         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
6122         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
6123         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
6124         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
6125         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
6126         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
6127         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
6128
6129         /* the next few have explicit upper bounds */
6130         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
6131         md->limit = md->base - 1 +
6132                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
6133                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
6134         md++;
6135
6136         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
6137         md->limit = md->base - 1 +
6138                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
6139                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
6140         md++;
6141
6142         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6143                 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
6144                 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
6145                 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
6146         } else {
6147                 md->base = 0;
6148                 md->idx = nitems(region);  /* hide it */
6149         }
6150         md++;
6151
6152 #define ulp_region(reg) \
6153         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
6154         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
6155
6156         ulp_region(RX_ISCSI);
6157         ulp_region(RX_TDDP);
6158         ulp_region(TX_TPT);
6159         ulp_region(RX_STAG);
6160         ulp_region(RX_RQ);
6161         ulp_region(RX_RQUDP);
6162         ulp_region(RX_PBL);
6163         ulp_region(TX_PBL);
6164 #undef ulp_region
6165
6166         md->base = 0;
6167         md->idx = nitems(region);
6168         if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
6169                 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
6170                 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
6171                     A_SGE_DBVFIFO_SIZE))) << 2) - 1;
6172         }
6173         md++;
6174
6175         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
6176         md->limit = md->base + sc->tids.ntids - 1;
6177         md++;
6178         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
6179         md->limit = md->base + sc->tids.ntids - 1;
6180         md++;
6181
6182         md->base = sc->vres.ocq.start;
6183         if (sc->vres.ocq.size)
6184                 md->limit = md->base + sc->vres.ocq.size - 1;
6185         else
6186                 md->idx = nitems(region);  /* hide it */
6187         md++;
6188
6189         /* add any address-space holes, there can be up to 3 */
6190         for (n = 0; n < i - 1; n++)
6191                 if (avail[n].limit < avail[n + 1].base)
6192                         (md++)->base = avail[n].limit;
6193         if (avail[n].limit)
6194                 (md++)->base = avail[n].limit;
6195
6196         n = md - mem;
6197         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
6198
6199         for (lo = 0; lo < i; lo++)
6200                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
6201                                 avail[lo].limit - 1);
6202
6203         sbuf_printf(sb, "\n");
6204         for (i = 0; i < n; i++) {
6205                 if (mem[i].idx >= nitems(region))
6206                         continue;                        /* skip holes */
6207                 if (!mem[i].limit)
6208                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
6209                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
6210                                 mem[i].limit);
6211         }
6212
6213         sbuf_printf(sb, "\n");
6214         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
6215         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
6216         mem_region_show(sb, "uP RAM:", lo, hi);
6217
6218         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
6219         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
6220         mem_region_show(sb, "uP Extmem2:", lo, hi);
6221
6222         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
6223         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
6224                    G_PMRXMAXPAGE(lo),
6225                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
6226                    (lo & F_PMRXNUMCHN) ? 2 : 1);
6227
6228         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
6229         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
6230         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
6231                    G_PMTXMAXPAGE(lo),
6232                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
6233                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
6234         sbuf_printf(sb, "%u p-structs\n",
6235                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
6236
6237         for (i = 0; i < 4; i++) {
6238                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
6239                 if (is_t4(sc)) {
6240                         used = G_USED(lo);
6241                         alloc = G_ALLOC(lo);
6242                 } else {
6243                         used = G_T5_USED(lo);
6244                         alloc = G_T5_ALLOC(lo);
6245                 }
6246                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
6247                            i, used, alloc);
6248         }
6249         for (i = 0; i < 4; i++) {
6250                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
6251                 if (is_t4(sc)) {
6252                         used = G_USED(lo);
6253                         alloc = G_ALLOC(lo);
6254                 } else {
6255                         used = G_T5_USED(lo);
6256                         alloc = G_T5_ALLOC(lo);
6257                 }
6258                 sbuf_printf(sb,
6259                            "\nLoopback %d using %u pages out of %u allocated",
6260                            i, used, alloc);
6261         }
6262
6263         rc = sbuf_finish(sb);
6264         sbuf_delete(sb);
6265
6266         return (rc);
6267 }
6268
6269 static inline void
6270 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
6271 {
6272         *mask = x | y;
6273         y = htobe64(y);
6274         memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
6275 }
6276
6277 static int
6278 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
6279 {
6280         struct adapter *sc = arg1;
6281         struct sbuf *sb;
6282         int rc, i, n;
6283
6284         rc = sysctl_wire_old_buffer(req, 0);
6285         if (rc != 0)
6286                 return (rc);
6287
6288         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6289         if (sb == NULL)
6290                 return (ENOMEM);
6291
6292         sbuf_printf(sb,
6293             "Idx  Ethernet address     Mask     Vld Ports PF"
6294             "  VF              Replication             P0 P1 P2 P3  ML");
6295         n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
6296             NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
6297         for (i = 0; i < n; i++) {
6298                 uint64_t tcamx, tcamy, mask;
6299                 uint32_t cls_lo, cls_hi;
6300                 uint8_t addr[ETHER_ADDR_LEN];
6301
6302                 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
6303                 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
6304                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
6305                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
6306
6307                 if (tcamx & tcamy)
6308                         continue;
6309
6310                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
6311                 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
6312                            "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
6313                            addr[3], addr[4], addr[5], (uintmax_t)mask,
6314                            (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
6315                            G_PORTMAP(cls_hi), G_PF(cls_lo),
6316                            (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
6317
6318                 if (cls_lo & F_REPLICATE) {
6319                         struct fw_ldst_cmd ldst_cmd;
6320
6321                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
6322                         ldst_cmd.op_to_addrspace =
6323                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
6324                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
6325                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
6326                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
6327                         ldst_cmd.u.mps.fid_ctl =
6328                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
6329                                 V_FW_LDST_CMD_CTL(i));
6330
6331                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
6332                             "t4mps");
6333                         if (rc)
6334                                 break;
6335                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
6336                             sizeof(ldst_cmd), &ldst_cmd);
6337                         end_synchronized_op(sc, 0);
6338
6339                         if (rc != 0) {
6340                                 sbuf_printf(sb,
6341                                     " ------------ error %3u ------------", rc);
6342                                 rc = 0;
6343                         } else {
6344                                 sbuf_printf(sb, " %08x %08x %08x %08x",
6345                                     be32toh(ldst_cmd.u.mps.rplc127_96),
6346                                     be32toh(ldst_cmd.u.mps.rplc95_64),
6347                                     be32toh(ldst_cmd.u.mps.rplc63_32),
6348                                     be32toh(ldst_cmd.u.mps.rplc31_0));
6349                         }
6350                 } else
6351                         sbuf_printf(sb, "%36s", "");
6352
6353                 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
6354                     G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
6355                     G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
6356         }
6357
6358         if (rc)
6359                 (void) sbuf_finish(sb);
6360         else
6361                 rc = sbuf_finish(sb);
6362         sbuf_delete(sb);
6363
6364         return (rc);
6365 }
6366
6367 static int
6368 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
6369 {
6370         struct adapter *sc = arg1;
6371         struct sbuf *sb;
6372         int rc;
6373         uint16_t mtus[NMTUS];
6374
6375         rc = sysctl_wire_old_buffer(req, 0);
6376         if (rc != 0)
6377                 return (rc);
6378
6379         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6380         if (sb == NULL)
6381                 return (ENOMEM);
6382
6383         t4_read_mtu_tbl(sc, mtus, NULL);
6384
6385         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
6386             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
6387             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
6388             mtus[14], mtus[15]);
6389
6390         rc = sbuf_finish(sb);
6391         sbuf_delete(sb);
6392
6393         return (rc);
6394 }
6395
6396 static int
6397 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
6398 {
6399         struct adapter *sc = arg1;
6400         struct sbuf *sb;
6401         int rc, i;
6402         uint32_t cnt[PM_NSTATS];
6403         uint64_t cyc[PM_NSTATS];
6404         static const char *rx_stats[] = {
6405                 "Read:", "Write bypass:", "Write mem:", "Flush:"
6406         };
6407         static const char *tx_stats[] = {
6408                 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
6409         };
6410
6411         rc = sysctl_wire_old_buffer(req, 0);
6412         if (rc != 0)
6413                 return (rc);
6414
6415         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6416         if (sb == NULL)
6417                 return (ENOMEM);
6418
6419         t4_pmtx_get_stats(sc, cnt, cyc);
6420         sbuf_printf(sb, "                Tx pcmds             Tx bytes");
6421         for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
6422                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
6423                     cyc[i]);
6424
6425         t4_pmrx_get_stats(sc, cnt, cyc);
6426         sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
6427         for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
6428                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
6429                     cyc[i]);
6430
6431         rc = sbuf_finish(sb);
6432         sbuf_delete(sb);
6433
6434         return (rc);
6435 }
6436
6437 static int
6438 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
6439 {
6440         struct adapter *sc = arg1;
6441         struct sbuf *sb;
6442         int rc;
6443         struct tp_rdma_stats stats;
6444
6445         rc = sysctl_wire_old_buffer(req, 0);
6446         if (rc != 0)
6447                 return (rc);
6448
6449         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6450         if (sb == NULL)
6451                 return (ENOMEM);
6452
6453         t4_tp_get_rdma_stats(sc, &stats);
6454         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
6455         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
6456
6457         rc = sbuf_finish(sb);
6458         sbuf_delete(sb);
6459
6460         return (rc);
6461 }
6462
6463 static int
6464 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
6465 {
6466         struct adapter *sc = arg1;
6467         struct sbuf *sb;
6468         int rc;
6469         struct tp_tcp_stats v4, v6;
6470
6471         rc = sysctl_wire_old_buffer(req, 0);
6472         if (rc != 0)
6473                 return (rc);
6474
6475         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6476         if (sb == NULL)
6477                 return (ENOMEM);
6478
6479         t4_tp_get_tcp_stats(sc, &v4, &v6);
6480         sbuf_printf(sb,
6481             "                                IP                 IPv6\n");
6482         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
6483             v4.tcpOutRsts, v6.tcpOutRsts);
6484         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
6485             v4.tcpInSegs, v6.tcpInSegs);
6486         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
6487             v4.tcpOutSegs, v6.tcpOutSegs);
6488         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
6489             v4.tcpRetransSegs, v6.tcpRetransSegs);
6490
6491         rc = sbuf_finish(sb);
6492         sbuf_delete(sb);
6493
6494         return (rc);
6495 }
6496
6497 static int
6498 sysctl_tids(SYSCTL_HANDLER_ARGS)
6499 {
6500         struct adapter *sc = arg1;
6501         struct sbuf *sb;
6502         int rc;
6503         struct tid_info *t = &sc->tids;
6504
6505         rc = sysctl_wire_old_buffer(req, 0);
6506         if (rc != 0)
6507                 return (rc);
6508
6509         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6510         if (sb == NULL)
6511                 return (ENOMEM);
6512
6513         if (t->natids) {
6514                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
6515                     t->atids_in_use);
6516         }
6517
6518         if (t->ntids) {
6519                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6520                         uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
6521
6522                         if (b) {
6523                                 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
6524                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6525                                     t->ntids - 1);
6526                         } else {
6527                                 sbuf_printf(sb, "TID range: %u-%u",
6528                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6529                                     t->ntids - 1);
6530                         }
6531                 } else
6532                         sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
6533                 sbuf_printf(sb, ", in use: %u\n",
6534                     atomic_load_acq_int(&t->tids_in_use));
6535         }
6536
6537         if (t->nstids) {
6538                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
6539                     t->stid_base + t->nstids - 1, t->stids_in_use);
6540         }
6541
6542         if (t->nftids) {
6543                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
6544                     t->ftid_base + t->nftids - 1);
6545         }
6546
6547         if (t->netids) {
6548                 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
6549                     t->etid_base + t->netids - 1);
6550         }
6551
6552         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
6553             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
6554             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
6555
6556         rc = sbuf_finish(sb);
6557         sbuf_delete(sb);
6558
6559         return (rc);
6560 }
6561
6562 static int
6563 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
6564 {
6565         struct adapter *sc = arg1;
6566         struct sbuf *sb;
6567         int rc;
6568         struct tp_err_stats stats;
6569
6570         rc = sysctl_wire_old_buffer(req, 0);
6571         if (rc != 0)
6572                 return (rc);
6573
6574         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6575         if (sb == NULL)
6576                 return (ENOMEM);
6577
6578         t4_tp_get_err_stats(sc, &stats);
6579
6580         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
6581                       "channel 3\n");
6582         sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
6583             stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
6584             stats.macInErrs[3]);
6585         sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
6586             stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
6587             stats.hdrInErrs[3]);
6588         sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
6589             stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
6590             stats.tcpInErrs[3]);
6591         sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
6592             stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
6593             stats.tcp6InErrs[3]);
6594         sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
6595             stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
6596             stats.tnlCongDrops[3]);
6597         sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
6598             stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
6599             stats.tnlTxDrops[3]);
6600         sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
6601             stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
6602             stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
6603         sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
6604             stats.ofldChanDrops[0], stats.ofldChanDrops[1],
6605             stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
6606         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
6607             stats.ofldNoNeigh, stats.ofldCongDefer);
6608
6609         rc = sbuf_finish(sb);
6610         sbuf_delete(sb);
6611
6612         return (rc);
6613 }
6614
6615 struct field_desc {
6616         const char *name;
6617         u_int start;
6618         u_int width;
6619 };
6620
6621 static void
6622 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
6623 {
6624         char buf[32];
6625         int line_size = 0;
6626
6627         while (f->name) {
6628                 uint64_t mask = (1ULL << f->width) - 1;
6629                 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
6630                     ((uintmax_t)v >> f->start) & mask);
6631
6632                 if (line_size + len >= 79) {
6633                         line_size = 8;
6634                         sbuf_printf(sb, "\n        ");
6635                 }
6636                 sbuf_printf(sb, "%s ", buf);
6637                 line_size += len + 1;
6638                 f++;
6639         }
6640         sbuf_printf(sb, "\n");
6641 }
6642
6643 static struct field_desc tp_la0[] = {
6644         { "RcfOpCodeOut", 60, 4 },
6645         { "State", 56, 4 },
6646         { "WcfState", 52, 4 },
6647         { "RcfOpcSrcOut", 50, 2 },
6648         { "CRxError", 49, 1 },
6649         { "ERxError", 48, 1 },
6650         { "SanityFailed", 47, 1 },
6651         { "SpuriousMsg", 46, 1 },
6652         { "FlushInputMsg", 45, 1 },
6653         { "FlushInputCpl", 44, 1 },
6654         { "RssUpBit", 43, 1 },
6655         { "RssFilterHit", 42, 1 },
6656         { "Tid", 32, 10 },
6657         { "InitTcb", 31, 1 },
6658         { "LineNumber", 24, 7 },
6659         { "Emsg", 23, 1 },
6660         { "EdataOut", 22, 1 },
6661         { "Cmsg", 21, 1 },
6662         { "CdataOut", 20, 1 },
6663         { "EreadPdu", 19, 1 },
6664         { "CreadPdu", 18, 1 },
6665         { "TunnelPkt", 17, 1 },
6666         { "RcfPeerFin", 16, 1 },
6667         { "RcfReasonOut", 12, 4 },
6668         { "TxCchannel", 10, 2 },
6669         { "RcfTxChannel", 8, 2 },
6670         { "RxEchannel", 6, 2 },
6671         { "RcfRxChannel", 5, 1 },
6672         { "RcfDataOutSrdy", 4, 1 },
6673         { "RxDvld", 3, 1 },
6674         { "RxOoDvld", 2, 1 },
6675         { "RxCongestion", 1, 1 },
6676         { "TxCongestion", 0, 1 },
6677         { NULL }
6678 };
6679
6680 static struct field_desc tp_la1[] = {
6681         { "CplCmdIn", 56, 8 },
6682         { "CplCmdOut", 48, 8 },
6683         { "ESynOut", 47, 1 },
6684         { "EAckOut", 46, 1 },
6685         { "EFinOut", 45, 1 },
6686         { "ERstOut", 44, 1 },
6687         { "SynIn", 43, 1 },
6688         { "AckIn", 42, 1 },
6689         { "FinIn", 41, 1 },
6690         { "RstIn", 40, 1 },
6691         { "DataIn", 39, 1 },
6692         { "DataInVld", 38, 1 },
6693         { "PadIn", 37, 1 },
6694         { "RxBufEmpty", 36, 1 },
6695         { "RxDdp", 35, 1 },
6696         { "RxFbCongestion", 34, 1 },
6697         { "TxFbCongestion", 33, 1 },
6698         { "TxPktSumSrdy", 32, 1 },
6699         { "RcfUlpType", 28, 4 },
6700         { "Eread", 27, 1 },
6701         { "Ebypass", 26, 1 },
6702         { "Esave", 25, 1 },
6703         { "Static0", 24, 1 },
6704         { "Cread", 23, 1 },
6705         { "Cbypass", 22, 1 },
6706         { "Csave", 21, 1 },
6707         { "CPktOut", 20, 1 },
6708         { "RxPagePoolFull", 18, 2 },
6709         { "RxLpbkPkt", 17, 1 },
6710         { "TxLpbkPkt", 16, 1 },
6711         { "RxVfValid", 15, 1 },
6712         { "SynLearned", 14, 1 },
6713         { "SetDelEntry", 13, 1 },
6714         { "SetInvEntry", 12, 1 },
6715         { "CpcmdDvld", 11, 1 },
6716         { "CpcmdSave", 10, 1 },
6717         { "RxPstructsFull", 8, 2 },
6718         { "EpcmdDvld", 7, 1 },
6719         { "EpcmdFlush", 6, 1 },
6720         { "EpcmdTrimPrefix", 5, 1 },
6721         { "EpcmdTrimPostfix", 4, 1 },
6722         { "ERssIp4Pkt", 3, 1 },
6723         { "ERssIp6Pkt", 2, 1 },
6724         { "ERssTcpUdpPkt", 1, 1 },
6725         { "ERssFceFipPkt", 0, 1 },
6726         { NULL }
6727 };
6728
6729 static struct field_desc tp_la2[] = {
6730         { "CplCmdIn", 56, 8 },
6731         { "MpsVfVld", 55, 1 },
6732         { "MpsPf", 52, 3 },
6733         { "MpsVf", 44, 8 },
6734         { "SynIn", 43, 1 },
6735         { "AckIn", 42, 1 },
6736         { "FinIn", 41, 1 },
6737         { "RstIn", 40, 1 },
6738         { "DataIn", 39, 1 },
6739         { "DataInVld", 38, 1 },
6740         { "PadIn", 37, 1 },
6741         { "RxBufEmpty", 36, 1 },
6742         { "RxDdp", 35, 1 },
6743         { "RxFbCongestion", 34, 1 },
6744         { "TxFbCongestion", 33, 1 },
6745         { "TxPktSumSrdy", 32, 1 },
6746         { "RcfUlpType", 28, 4 },
6747         { "Eread", 27, 1 },
6748         { "Ebypass", 26, 1 },
6749         { "Esave", 25, 1 },
6750         { "Static0", 24, 1 },
6751         { "Cread", 23, 1 },
6752         { "Cbypass", 22, 1 },
6753         { "Csave", 21, 1 },
6754         { "CPktOut", 20, 1 },
6755         { "RxPagePoolFull", 18, 2 },
6756         { "RxLpbkPkt", 17, 1 },
6757         { "TxLpbkPkt", 16, 1 },
6758         { "RxVfValid", 15, 1 },
6759         { "SynLearned", 14, 1 },
6760         { "SetDelEntry", 13, 1 },
6761         { "SetInvEntry", 12, 1 },
6762         { "CpcmdDvld", 11, 1 },
6763         { "CpcmdSave", 10, 1 },
6764         { "RxPstructsFull", 8, 2 },
6765         { "EpcmdDvld", 7, 1 },
6766         { "EpcmdFlush", 6, 1 },
6767         { "EpcmdTrimPrefix", 5, 1 },
6768         { "EpcmdTrimPostfix", 4, 1 },
6769         { "ERssIp4Pkt", 3, 1 },
6770         { "ERssIp6Pkt", 2, 1 },
6771         { "ERssTcpUdpPkt", 1, 1 },
6772         { "ERssFceFipPkt", 0, 1 },
6773         { NULL }
6774 };
6775
6776 static void
6777 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
6778 {
6779
6780         field_desc_show(sb, *p, tp_la0);
6781 }
6782
6783 static void
6784 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
6785 {
6786
6787         if (idx)
6788                 sbuf_printf(sb, "\n");
6789         field_desc_show(sb, p[0], tp_la0);
6790         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6791                 field_desc_show(sb, p[1], tp_la0);
6792 }
6793
6794 static void
6795 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
6796 {
6797
6798         if (idx)
6799                 sbuf_printf(sb, "\n");
6800         field_desc_show(sb, p[0], tp_la0);
6801         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6802                 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
6803 }
6804
6805 static int
6806 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
6807 {
6808         struct adapter *sc = arg1;
6809         struct sbuf *sb;
6810         uint64_t *buf, *p;
6811         int rc;
6812         u_int i, inc;
6813         void (*show_func)(struct sbuf *, uint64_t *, int);
6814
6815         rc = sysctl_wire_old_buffer(req, 0);
6816         if (rc != 0)
6817                 return (rc);
6818
6819         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6820         if (sb == NULL)
6821                 return (ENOMEM);
6822
6823         buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
6824
6825         t4_tp_read_la(sc, buf, NULL);
6826         p = buf;
6827
6828         switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
6829         case 2:
6830                 inc = 2;
6831                 show_func = tp_la_show2;
6832                 break;
6833         case 3:
6834                 inc = 2;
6835                 show_func = tp_la_show3;
6836                 break;
6837         default:
6838                 inc = 1;
6839                 show_func = tp_la_show;
6840         }
6841
6842         for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
6843                 (*show_func)(sb, p, i);
6844
6845         rc = sbuf_finish(sb);
6846         sbuf_delete(sb);
6847         free(buf, M_CXGBE);
6848         return (rc);
6849 }
6850
6851 static int
6852 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
6853 {
6854         struct adapter *sc = arg1;
6855         struct sbuf *sb;
6856         int rc;
6857         u64 nrate[NCHAN], orate[NCHAN];
6858
6859         rc = sysctl_wire_old_buffer(req, 0);
6860         if (rc != 0)
6861                 return (rc);
6862
6863         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6864         if (sb == NULL)
6865                 return (ENOMEM);
6866
6867         t4_get_chan_txrate(sc, nrate, orate);
6868         sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
6869                  "channel 3\n");
6870         sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
6871             nrate[0], nrate[1], nrate[2], nrate[3]);
6872         sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
6873             orate[0], orate[1], orate[2], orate[3]);
6874
6875         rc = sbuf_finish(sb);
6876         sbuf_delete(sb);
6877
6878         return (rc);
6879 }
6880
6881 static int
6882 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
6883 {
6884         struct adapter *sc = arg1;
6885         struct sbuf *sb;
6886         uint32_t *buf, *p;
6887         int rc, i;
6888
6889         rc = sysctl_wire_old_buffer(req, 0);
6890         if (rc != 0)
6891                 return (rc);
6892
6893         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6894         if (sb == NULL)
6895                 return (ENOMEM);
6896
6897         buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
6898             M_ZERO | M_WAITOK);
6899
6900         t4_ulprx_read_la(sc, buf);
6901         p = buf;
6902
6903         sbuf_printf(sb, "      Pcmd        Type   Message"
6904             "                Data");
6905         for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
6906                 sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
6907                     p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
6908         }
6909
6910         rc = sbuf_finish(sb);
6911         sbuf_delete(sb);
6912         free(buf, M_CXGBE);
6913         return (rc);
6914 }
6915
6916 static int
6917 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
6918 {
6919         struct adapter *sc = arg1;
6920         struct sbuf *sb;
6921         int rc, v;
6922
6923         rc = sysctl_wire_old_buffer(req, 0);
6924         if (rc != 0)
6925                 return (rc);
6926
6927         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6928         if (sb == NULL)
6929                 return (ENOMEM);
6930
6931         v = t4_read_reg(sc, A_SGE_STAT_CFG);
6932         if (G_STATSOURCE_T5(v) == 7) {
6933                 if (G_STATMODE(v) == 0) {
6934                         sbuf_printf(sb, "total %d, incomplete %d",
6935                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6936                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6937                 } else if (G_STATMODE(v) == 1) {
6938                         sbuf_printf(sb, "total %d, data overflow %d",
6939                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6940                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6941                 }
6942         }
6943         rc = sbuf_finish(sb);
6944         sbuf_delete(sb);
6945
6946         return (rc);
6947 }
6948 #endif
6949
6950 static inline void
6951 txq_start(struct ifnet *ifp, struct sge_txq *txq)
6952 {
6953         struct buf_ring *br;
6954         struct mbuf *m;
6955
6956         TXQ_LOCK_ASSERT_OWNED(txq);
6957
6958         br = txq->br;
6959         m = txq->m ? txq->m : drbr_dequeue(ifp, br);
6960         if (m)
6961                 t4_eth_tx(ifp, txq, m);
6962 }
6963
6964 void
6965 t4_tx_callout(void *arg)
6966 {
6967         struct sge_eq *eq = arg;
6968         struct adapter *sc;
6969
6970         if (EQ_TRYLOCK(eq) == 0)
6971                 goto reschedule;
6972
6973         if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) {
6974                 EQ_UNLOCK(eq);
6975 reschedule:
6976                 if (__predict_true(!(eq->flags && EQ_DOOMED)))
6977                         callout_schedule(&eq->tx_callout, 1);
6978                 return;
6979         }
6980
6981         EQ_LOCK_ASSERT_OWNED(eq);
6982
6983         if (__predict_true((eq->flags & EQ_DOOMED) == 0)) {
6984
6985                 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
6986                         struct sge_txq *txq = arg;
6987                         struct port_info *pi = txq->ifp->if_softc;
6988
6989                         sc = pi->adapter;
6990                 } else {
6991                         struct sge_wrq *wrq = arg;
6992
6993                         sc = wrq->adapter;
6994                 }
6995
6996                 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task);
6997         }
6998
6999         EQ_UNLOCK(eq);
7000 }
7001
7002 void
7003 t4_tx_task(void *arg, int count)
7004 {
7005         struct sge_eq *eq = arg;
7006
7007         EQ_LOCK(eq);
7008         if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
7009                 struct sge_txq *txq = arg;
7010                 txq_start(txq->ifp, txq);
7011         } else {
7012                 struct sge_wrq *wrq = arg;
7013                 t4_wrq_tx_locked(wrq->adapter, wrq, NULL);
7014         }
7015         EQ_UNLOCK(eq);
7016 }
7017
7018 static uint32_t
7019 fconf_to_mode(uint32_t fconf)
7020 {
7021         uint32_t mode;
7022
7023         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
7024             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
7025
7026         if (fconf & F_FRAGMENTATION)
7027                 mode |= T4_FILTER_IP_FRAGMENT;
7028
7029         if (fconf & F_MPSHITTYPE)
7030                 mode |= T4_FILTER_MPS_HIT_TYPE;
7031
7032         if (fconf & F_MACMATCH)
7033                 mode |= T4_FILTER_MAC_IDX;
7034
7035         if (fconf & F_ETHERTYPE)
7036                 mode |= T4_FILTER_ETH_TYPE;
7037
7038         if (fconf & F_PROTOCOL)
7039                 mode |= T4_FILTER_IP_PROTO;
7040
7041         if (fconf & F_TOS)
7042                 mode |= T4_FILTER_IP_TOS;
7043
7044         if (fconf & F_VLAN)
7045                 mode |= T4_FILTER_VLAN;
7046
7047         if (fconf & F_VNIC_ID)
7048                 mode |= T4_FILTER_VNIC;
7049
7050         if (fconf & F_PORT)
7051                 mode |= T4_FILTER_PORT;
7052
7053         if (fconf & F_FCOE)
7054                 mode |= T4_FILTER_FCoE;
7055
7056         return (mode);
7057 }
7058
7059 static uint32_t
7060 mode_to_fconf(uint32_t mode)
7061 {
7062         uint32_t fconf = 0;
7063
7064         if (mode & T4_FILTER_IP_FRAGMENT)
7065                 fconf |= F_FRAGMENTATION;
7066
7067         if (mode & T4_FILTER_MPS_HIT_TYPE)
7068                 fconf |= F_MPSHITTYPE;
7069
7070         if (mode & T4_FILTER_MAC_IDX)
7071                 fconf |= F_MACMATCH;
7072
7073         if (mode & T4_FILTER_ETH_TYPE)
7074                 fconf |= F_ETHERTYPE;
7075
7076         if (mode & T4_FILTER_IP_PROTO)
7077                 fconf |= F_PROTOCOL;
7078
7079         if (mode & T4_FILTER_IP_TOS)
7080                 fconf |= F_TOS;
7081
7082         if (mode & T4_FILTER_VLAN)
7083                 fconf |= F_VLAN;
7084
7085         if (mode & T4_FILTER_VNIC)
7086                 fconf |= F_VNIC_ID;
7087
7088         if (mode & T4_FILTER_PORT)
7089                 fconf |= F_PORT;
7090
7091         if (mode & T4_FILTER_FCoE)
7092                 fconf |= F_FCOE;
7093
7094         return (fconf);
7095 }
7096
7097 static uint32_t
7098 fspec_to_fconf(struct t4_filter_specification *fs)
7099 {
7100         uint32_t fconf = 0;
7101
7102         if (fs->val.frag || fs->mask.frag)
7103                 fconf |= F_FRAGMENTATION;
7104
7105         if (fs->val.matchtype || fs->mask.matchtype)
7106                 fconf |= F_MPSHITTYPE;
7107
7108         if (fs->val.macidx || fs->mask.macidx)
7109                 fconf |= F_MACMATCH;
7110
7111         if (fs->val.ethtype || fs->mask.ethtype)
7112                 fconf |= F_ETHERTYPE;
7113
7114         if (fs->val.proto || fs->mask.proto)
7115                 fconf |= F_PROTOCOL;
7116
7117         if (fs->val.tos || fs->mask.tos)
7118                 fconf |= F_TOS;
7119
7120         if (fs->val.vlan_vld || fs->mask.vlan_vld)
7121                 fconf |= F_VLAN;
7122
7123         if (fs->val.vnic_vld || fs->mask.vnic_vld)
7124                 fconf |= F_VNIC_ID;
7125
7126         if (fs->val.iport || fs->mask.iport)
7127                 fconf |= F_PORT;
7128
7129         if (fs->val.fcoe || fs->mask.fcoe)
7130                 fconf |= F_FCOE;
7131
7132         return (fconf);
7133 }
7134
7135 static int
7136 get_filter_mode(struct adapter *sc, uint32_t *mode)
7137 {
7138         int rc;
7139         uint32_t fconf;
7140
7141         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7142             "t4getfm");
7143         if (rc)
7144                 return (rc);
7145
7146         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
7147             A_TP_VLAN_PRI_MAP);
7148
7149         if (sc->params.tp.vlan_pri_map != fconf) {
7150                 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
7151                     device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
7152                     fconf);
7153                 sc->params.tp.vlan_pri_map = fconf;
7154         }
7155
7156         *mode = fconf_to_mode(sc->params.tp.vlan_pri_map);
7157
7158         end_synchronized_op(sc, LOCK_HELD);
7159         return (0);
7160 }
7161
7162 static int
7163 set_filter_mode(struct adapter *sc, uint32_t mode)
7164 {
7165         uint32_t fconf;
7166         int rc;
7167
7168         fconf = mode_to_fconf(mode);
7169
7170         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7171             "t4setfm");
7172         if (rc)
7173                 return (rc);
7174
7175         if (sc->tids.ftids_in_use > 0) {
7176                 rc = EBUSY;
7177                 goto done;
7178         }
7179
7180 #ifdef TCP_OFFLOAD
7181         if (sc->offload_map) {
7182                 rc = EBUSY;
7183                 goto done;
7184         }
7185 #endif
7186
7187 #ifdef notyet
7188         rc = -t4_set_filter_mode(sc, fconf);
7189         if (rc == 0)
7190                 sc->filter_mode = fconf;
7191 #else
7192         rc = ENOTSUP;
7193 #endif
7194
7195 done:
7196         end_synchronized_op(sc, LOCK_HELD);
7197         return (rc);
7198 }
7199
7200 static inline uint64_t
7201 get_filter_hits(struct adapter *sc, uint32_t fid)
7202 {
7203         uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
7204         uint64_t hits;
7205
7206         memwin_info(sc, 0, &mw_base, NULL);
7207         off = position_memwin(sc, 0,
7208             tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
7209         if (is_t4(sc)) {
7210                 hits = t4_read_reg64(sc, mw_base + off + 16);
7211                 hits = be64toh(hits);
7212         } else {
7213                 hits = t4_read_reg(sc, mw_base + off + 24);
7214                 hits = be32toh(hits);
7215         }
7216
7217         return (hits);
7218 }
7219
7220 static int
7221 get_filter(struct adapter *sc, struct t4_filter *t)
7222 {
7223         int i, rc, nfilters = sc->tids.nftids;
7224         struct filter_entry *f;
7225
7226         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7227             "t4getf");
7228         if (rc)
7229                 return (rc);
7230
7231         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
7232             t->idx >= nfilters) {
7233                 t->idx = 0xffffffff;
7234                 goto done;
7235         }
7236
7237         f = &sc->tids.ftid_tab[t->idx];
7238         for (i = t->idx; i < nfilters; i++, f++) {
7239                 if (f->valid) {
7240                         t->idx = i;
7241                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
7242                         t->smtidx = f->smtidx;
7243                         if (f->fs.hitcnts)
7244                                 t->hits = get_filter_hits(sc, t->idx);
7245                         else
7246                                 t->hits = UINT64_MAX;
7247                         t->fs = f->fs;
7248
7249                         goto done;
7250                 }
7251         }
7252
7253         t->idx = 0xffffffff;
7254 done:
7255         end_synchronized_op(sc, LOCK_HELD);
7256         return (0);
7257 }
7258
7259 static int
7260 set_filter(struct adapter *sc, struct t4_filter *t)
7261 {
7262         unsigned int nfilters, nports;
7263         struct filter_entry *f;
7264         int i, rc;
7265
7266         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
7267         if (rc)
7268                 return (rc);
7269
7270         nfilters = sc->tids.nftids;
7271         nports = sc->params.nports;
7272
7273         if (nfilters == 0) {
7274                 rc = ENOTSUP;
7275                 goto done;
7276         }
7277
7278         if (!(sc->flags & FULL_INIT_DONE)) {
7279                 rc = EAGAIN;
7280                 goto done;
7281         }
7282
7283         if (t->idx >= nfilters) {
7284                 rc = EINVAL;
7285                 goto done;
7286         }
7287
7288         /* Validate against the global filter mode */
7289         if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
7290             sc->params.tp.vlan_pri_map) {
7291                 rc = E2BIG;
7292                 goto done;
7293         }
7294
7295         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
7296                 rc = EINVAL;
7297                 goto done;
7298         }
7299
7300         if (t->fs.val.iport >= nports) {
7301                 rc = EINVAL;
7302                 goto done;
7303         }
7304
7305         /* Can't specify an iq if not steering to it */
7306         if (!t->fs.dirsteer && t->fs.iq) {
7307                 rc = EINVAL;
7308                 goto done;
7309         }
7310
7311         /* IPv6 filter idx must be 4 aligned */
7312         if (t->fs.type == 1 &&
7313             ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
7314                 rc = EINVAL;
7315                 goto done;
7316         }
7317
7318         if (sc->tids.ftid_tab == NULL) {
7319                 KASSERT(sc->tids.ftids_in_use == 0,
7320                     ("%s: no memory allocated but filters_in_use > 0",
7321                     __func__));
7322
7323                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
7324                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
7325                 if (sc->tids.ftid_tab == NULL) {
7326                         rc = ENOMEM;
7327                         goto done;
7328                 }
7329                 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
7330         }
7331
7332         for (i = 0; i < 4; i++) {
7333                 f = &sc->tids.ftid_tab[t->idx + i];
7334
7335                 if (f->pending || f->valid) {
7336                         rc = EBUSY;
7337                         goto done;
7338                 }
7339                 if (f->locked) {
7340                         rc = EPERM;
7341                         goto done;
7342                 }
7343
7344                 if (t->fs.type == 0)
7345                         break;
7346         }
7347
7348         f = &sc->tids.ftid_tab[t->idx];
7349         f->fs = t->fs;
7350
7351         rc = set_filter_wr(sc, t->idx);
7352 done:
7353         end_synchronized_op(sc, 0);
7354
7355         if (rc == 0) {
7356                 mtx_lock(&sc->tids.ftid_lock);
7357                 for (;;) {
7358                         if (f->pending == 0) {
7359                                 rc = f->valid ? 0 : EIO;
7360                                 break;
7361                         }
7362
7363                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7364                             PCATCH, "t4setfw", 0)) {
7365                                 rc = EINPROGRESS;
7366                                 break;
7367                         }
7368                 }
7369                 mtx_unlock(&sc->tids.ftid_lock);
7370         }
7371         return (rc);
7372 }
7373
7374 static int
7375 del_filter(struct adapter *sc, struct t4_filter *t)
7376 {
7377         unsigned int nfilters;
7378         struct filter_entry *f;
7379         int rc;
7380
7381         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
7382         if (rc)
7383                 return (rc);
7384
7385         nfilters = sc->tids.nftids;
7386
7387         if (nfilters == 0) {
7388                 rc = ENOTSUP;
7389                 goto done;
7390         }
7391
7392         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
7393             t->idx >= nfilters) {
7394                 rc = EINVAL;
7395                 goto done;
7396         }
7397
7398         if (!(sc->flags & FULL_INIT_DONE)) {
7399                 rc = EAGAIN;
7400                 goto done;
7401         }
7402
7403         f = &sc->tids.ftid_tab[t->idx];
7404
7405         if (f->pending) {
7406                 rc = EBUSY;
7407                 goto done;
7408         }
7409         if (f->locked) {
7410                 rc = EPERM;
7411                 goto done;
7412         }
7413
7414         if (f->valid) {
7415                 t->fs = f->fs;  /* extra info for the caller */
7416                 rc = del_filter_wr(sc, t->idx);
7417         }
7418
7419 done:
7420         end_synchronized_op(sc, 0);
7421
7422         if (rc == 0) {
7423                 mtx_lock(&sc->tids.ftid_lock);
7424                 for (;;) {
7425                         if (f->pending == 0) {
7426                                 rc = f->valid ? EIO : 0;
7427                                 break;
7428                         }
7429
7430                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7431                             PCATCH, "t4delfw", 0)) {
7432                                 rc = EINPROGRESS;
7433                                 break;
7434                         }
7435                 }
7436                 mtx_unlock(&sc->tids.ftid_lock);
7437         }
7438
7439         return (rc);
7440 }
7441
7442 static void
7443 clear_filter(struct filter_entry *f)
7444 {
7445         if (f->l2t)
7446                 t4_l2t_release(f->l2t);
7447
7448         bzero(f, sizeof (*f));
7449 }
7450
7451 static int
7452 set_filter_wr(struct adapter *sc, int fidx)
7453 {
7454         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7455         struct wrqe *wr;
7456         struct fw_filter_wr *fwr;
7457         unsigned int ftid;
7458
7459         ASSERT_SYNCHRONIZED_OP(sc);
7460
7461         if (f->fs.newdmac || f->fs.newvlan) {
7462                 /* This filter needs an L2T entry; allocate one. */
7463                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
7464                 if (f->l2t == NULL)
7465                         return (EAGAIN);
7466                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
7467                     f->fs.dmac)) {
7468                         t4_l2t_release(f->l2t);
7469                         f->l2t = NULL;
7470                         return (ENOMEM);
7471                 }
7472         }
7473
7474         ftid = sc->tids.ftid_base + fidx;
7475
7476         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7477         if (wr == NULL)
7478                 return (ENOMEM);
7479
7480         fwr = wrtod(wr);
7481         bzero(fwr, sizeof (*fwr));
7482
7483         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
7484         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
7485         fwr->tid_to_iq =
7486             htobe32(V_FW_FILTER_WR_TID(ftid) |
7487                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
7488                 V_FW_FILTER_WR_NOREPLY(0) |
7489                 V_FW_FILTER_WR_IQ(f->fs.iq));
7490         fwr->del_filter_to_l2tix =
7491             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
7492                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
7493                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
7494                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
7495                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
7496                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
7497                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
7498                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
7499                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
7500                     f->fs.newvlan == VLAN_REWRITE) |
7501                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
7502                     f->fs.newvlan == VLAN_REWRITE) |
7503                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
7504                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
7505                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
7506                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
7507         fwr->ethtype = htobe16(f->fs.val.ethtype);
7508         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
7509         fwr->frag_to_ovlan_vldm =
7510             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
7511                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
7512                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
7513                 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
7514                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
7515                 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
7516         fwr->smac_sel = 0;
7517         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
7518             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
7519         fwr->maci_to_matchtypem =
7520             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
7521                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
7522                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
7523                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
7524                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
7525                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
7526                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
7527                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
7528         fwr->ptcl = f->fs.val.proto;
7529         fwr->ptclm = f->fs.mask.proto;
7530         fwr->ttyp = f->fs.val.tos;
7531         fwr->ttypm = f->fs.mask.tos;
7532         fwr->ivlan = htobe16(f->fs.val.vlan);
7533         fwr->ivlanm = htobe16(f->fs.mask.vlan);
7534         fwr->ovlan = htobe16(f->fs.val.vnic);
7535         fwr->ovlanm = htobe16(f->fs.mask.vnic);
7536         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
7537         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
7538         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
7539         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
7540         fwr->lp = htobe16(f->fs.val.dport);
7541         fwr->lpm = htobe16(f->fs.mask.dport);
7542         fwr->fp = htobe16(f->fs.val.sport);
7543         fwr->fpm = htobe16(f->fs.mask.sport);
7544         if (f->fs.newsmac)
7545                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
7546
7547         f->pending = 1;
7548         sc->tids.ftids_in_use++;
7549
7550         t4_wrq_tx(sc, wr);
7551         return (0);
7552 }
7553
7554 static int
7555 del_filter_wr(struct adapter *sc, int fidx)
7556 {
7557         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7558         struct wrqe *wr;
7559         struct fw_filter_wr *fwr;
7560         unsigned int ftid;
7561
7562         ftid = sc->tids.ftid_base + fidx;
7563
7564         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7565         if (wr == NULL)
7566                 return (ENOMEM);
7567         fwr = wrtod(wr);
7568         bzero(fwr, sizeof (*fwr));
7569
7570         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
7571
7572         f->pending = 1;
7573         t4_wrq_tx(sc, wr);
7574         return (0);
7575 }
7576
7577 int
7578 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
7579 {
7580         struct adapter *sc = iq->adapter;
7581         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
7582         unsigned int idx = GET_TID(rpl);
7583         unsigned int rc;
7584         struct filter_entry *f;
7585
7586         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
7587             rss->opcode));
7588
7589         if (is_ftid(sc, idx)) {
7590
7591                 idx -= sc->tids.ftid_base;
7592                 f = &sc->tids.ftid_tab[idx];
7593                 rc = G_COOKIE(rpl->cookie);
7594
7595                 mtx_lock(&sc->tids.ftid_lock);
7596                 if (rc == FW_FILTER_WR_FLT_ADDED) {
7597                         KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
7598                             __func__, idx));
7599                         f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
7600                         f->pending = 0;  /* asynchronous setup completed */
7601                         f->valid = 1;
7602                 } else {
7603                         if (rc != FW_FILTER_WR_FLT_DELETED) {
7604                                 /* Add or delete failed, display an error */
7605                                 log(LOG_ERR,
7606                                     "filter %u setup failed with error %u\n",
7607                                     idx, rc);
7608                         }
7609
7610                         clear_filter(f);
7611                         sc->tids.ftids_in_use--;
7612                 }
7613                 wakeup(&sc->tids.ftid_tab);
7614                 mtx_unlock(&sc->tids.ftid_lock);
7615         }
7616
7617         return (0);
7618 }
7619
7620 static int
7621 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
7622 {
7623         int rc;
7624
7625         if (cntxt->cid > M_CTXTQID)
7626                 return (EINVAL);
7627
7628         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
7629             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
7630                 return (EINVAL);
7631
7632         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
7633         if (rc)
7634                 return (rc);
7635
7636         if (sc->flags & FW_OK) {
7637                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
7638                     &cntxt->data[0]);
7639                 if (rc == 0)
7640                         goto done;
7641         }
7642
7643         /*
7644          * Read via firmware failed or wasn't even attempted.  Read directly via
7645          * the backdoor.
7646          */
7647         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
7648 done:
7649         end_synchronized_op(sc, 0);
7650         return (rc);
7651 }
7652
7653 static int
7654 load_fw(struct adapter *sc, struct t4_data *fw)
7655 {
7656         int rc;
7657         uint8_t *fw_data;
7658
7659         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
7660         if (rc)
7661                 return (rc);
7662
7663         if (sc->flags & FULL_INIT_DONE) {
7664                 rc = EBUSY;
7665                 goto done;
7666         }
7667
7668         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
7669         if (fw_data == NULL) {
7670                 rc = ENOMEM;
7671                 goto done;
7672         }
7673
7674         rc = copyin(fw->data, fw_data, fw->len);
7675         if (rc == 0)
7676                 rc = -t4_load_fw(sc, fw_data, fw->len);
7677
7678         free(fw_data, M_CXGBE);
7679 done:
7680         end_synchronized_op(sc, 0);
7681         return (rc);
7682 }
7683
7684 static int
7685 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
7686 {
7687         uint32_t addr, off, remaining, i, n;
7688         uint32_t *buf, *b;
7689         uint32_t mw_base, mw_aperture;
7690         int rc;
7691         uint8_t *dst;
7692
7693         rc = validate_mem_range(sc, mr->addr, mr->len);
7694         if (rc != 0)
7695                 return (rc);
7696
7697         memwin_info(sc, win, &mw_base, &mw_aperture);
7698         buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
7699         addr = mr->addr;
7700         remaining = mr->len;
7701         dst = (void *)mr->data;
7702
7703         while (remaining) {
7704                 off = position_memwin(sc, win, addr);
7705
7706                 /* number of bytes that we'll copy in the inner loop */
7707                 n = min(remaining, mw_aperture - off);
7708                 for (i = 0; i < n; i += 4)
7709                         *b++ = t4_read_reg(sc, mw_base + off + i);
7710
7711                 rc = copyout(buf, dst, n);
7712                 if (rc != 0)
7713                         break;
7714
7715                 b = buf;
7716                 dst += n;
7717                 remaining -= n;
7718                 addr += n;
7719         }
7720
7721         free(buf, M_CXGBE);
7722         return (rc);
7723 }
7724
7725 static int
7726 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
7727 {
7728         int rc;
7729
7730         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
7731                 return (EINVAL);
7732
7733         if (i2cd->len > sizeof(i2cd->data))
7734                 return (EFBIG);
7735
7736         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
7737         if (rc)
7738                 return (rc);
7739         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
7740             i2cd->offset, i2cd->len, &i2cd->data[0]);
7741         end_synchronized_op(sc, 0);
7742
7743         return (rc);
7744 }
7745
7746 static int
7747 in_range(int val, int lo, int hi)
7748 {
7749
7750         return (val < 0 || (val <= hi && val >= lo));
7751 }
7752
7753 static int
7754 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
7755 {
7756         int fw_subcmd, fw_type, rc;
7757
7758         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
7759         if (rc)
7760                 return (rc);
7761
7762         if (!(sc->flags & FULL_INIT_DONE)) {
7763                 rc = EAGAIN;
7764                 goto done;
7765         }
7766
7767         /*
7768          * Translate the cxgbetool parameters into T4 firmware parameters.  (The
7769          * sub-command and type are in common locations.)
7770          */
7771         if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
7772                 fw_subcmd = FW_SCHED_SC_CONFIG;
7773         else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
7774                 fw_subcmd = FW_SCHED_SC_PARAMS;
7775         else {
7776                 rc = EINVAL;
7777                 goto done;
7778         }
7779         if (p->type == SCHED_CLASS_TYPE_PACKET)
7780                 fw_type = FW_SCHED_TYPE_PKTSCHED;
7781         else {
7782                 rc = EINVAL;
7783                 goto done;
7784         }
7785
7786         if (fw_subcmd == FW_SCHED_SC_CONFIG) {
7787                 /* Vet our parameters ..*/
7788                 if (p->u.config.minmax < 0) {
7789                         rc = EINVAL;
7790                         goto done;
7791                 }
7792
7793                 /* And pass the request to the firmware ...*/
7794                 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1);
7795                 goto done;
7796         }
7797
7798         if (fw_subcmd == FW_SCHED_SC_PARAMS) {
7799                 int fw_level;
7800                 int fw_mode;
7801                 int fw_rateunit;
7802                 int fw_ratemode;
7803
7804                 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
7805                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
7806                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
7807                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
7808                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
7809                         fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
7810                 else {
7811                         rc = EINVAL;
7812                         goto done;
7813                 }
7814
7815                 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
7816                         fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
7817                 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
7818                         fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
7819                 else {
7820                         rc = EINVAL;
7821                         goto done;
7822                 }
7823
7824                 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
7825                         fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
7826                 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
7827                         fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
7828                 else {
7829                         rc = EINVAL;
7830                         goto done;
7831                 }
7832
7833                 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
7834                         fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
7835                 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
7836                         fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
7837                 else {
7838                         rc = EINVAL;
7839                         goto done;
7840                 }
7841
7842                 /* Vet our parameters ... */
7843                 if (!in_range(p->u.params.channel, 0, 3) ||
7844                     !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
7845                     !in_range(p->u.params.minrate, 0, 10000000) ||
7846                     !in_range(p->u.params.maxrate, 0, 10000000) ||
7847                     !in_range(p->u.params.weight, 0, 100)) {
7848                         rc = ERANGE;
7849                         goto done;
7850                 }
7851
7852                 /*
7853                  * Translate any unset parameters into the firmware's
7854                  * nomenclature and/or fail the call if the parameters
7855                  * are required ...
7856                  */
7857                 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
7858                     p->u.params.channel < 0 || p->u.params.cl < 0) {
7859                         rc = EINVAL;
7860                         goto done;
7861                 }
7862                 if (p->u.params.minrate < 0)
7863                         p->u.params.minrate = 0;
7864                 if (p->u.params.maxrate < 0) {
7865                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7866                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7867                                 rc = EINVAL;
7868                                 goto done;
7869                         } else
7870                                 p->u.params.maxrate = 0;
7871                 }
7872                 if (p->u.params.weight < 0) {
7873                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
7874                                 rc = EINVAL;
7875                                 goto done;
7876                         } else
7877                                 p->u.params.weight = 0;
7878                 }
7879                 if (p->u.params.pktsize < 0) {
7880                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7881                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7882                                 rc = EINVAL;
7883                                 goto done;
7884                         } else
7885                                 p->u.params.pktsize = 0;
7886                 }
7887
7888                 /* See what the firmware thinks of the request ... */
7889                 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
7890                     fw_rateunit, fw_ratemode, p->u.params.channel,
7891                     p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
7892                     p->u.params.weight, p->u.params.pktsize, 1);
7893                 goto done;
7894         }
7895
7896         rc = EINVAL;
7897 done:
7898         end_synchronized_op(sc, 0);
7899         return (rc);
7900 }
7901
7902 static int
7903 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
7904 {
7905         struct port_info *pi = NULL;
7906         struct sge_txq *txq;
7907         uint32_t fw_mnem, fw_queue, fw_class;
7908         int i, rc;
7909
7910         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
7911         if (rc)
7912                 return (rc);
7913
7914         if (!(sc->flags & FULL_INIT_DONE)) {
7915                 rc = EAGAIN;
7916                 goto done;
7917         }
7918
7919         if (p->port >= sc->params.nports) {
7920                 rc = EINVAL;
7921                 goto done;
7922         }
7923
7924         pi = sc->port[p->port];
7925         if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
7926                 rc = EINVAL;
7927                 goto done;
7928         }
7929
7930         /*
7931          * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
7932          * Scheduling Class in this case).
7933          */
7934         fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
7935             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
7936         fw_class = p->cl < 0 ? 0xffffffff : p->cl;
7937
7938         /*
7939          * If op.queue is non-negative, then we're only changing the scheduling
7940          * on a single specified TX queue.
7941          */
7942         if (p->queue >= 0) {
7943                 txq = &sc->sge.txq[pi->first_txq + p->queue];
7944                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7945                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7946                     &fw_class);
7947                 goto done;
7948         }
7949
7950         /*
7951          * Change the scheduling on all the TX queues for the
7952          * interface.
7953          */
7954         for_each_txq(pi, i, txq) {
7955                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7956                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7957                     &fw_class);
7958                 if (rc)
7959                         goto done;
7960         }
7961
7962         rc = 0;
7963 done:
7964         end_synchronized_op(sc, 0);
7965         return (rc);
7966 }
7967
7968 int
7969 t4_os_find_pci_capability(struct adapter *sc, int cap)
7970 {
7971         int i;
7972
7973         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
7974 }
7975
7976 int
7977 t4_os_pci_save_state(struct adapter *sc)
7978 {
7979         device_t dev;
7980         struct pci_devinfo *dinfo;
7981
7982         dev = sc->dev;
7983         dinfo = device_get_ivars(dev);
7984
7985         pci_cfg_save(dev, dinfo, 0);
7986         return (0);
7987 }
7988
7989 int
7990 t4_os_pci_restore_state(struct adapter *sc)
7991 {
7992         device_t dev;
7993         struct pci_devinfo *dinfo;
7994
7995         dev = sc->dev;
7996         dinfo = device_get_ivars(dev);
7997
7998         pci_cfg_restore(dev, dinfo);
7999         return (0);
8000 }
8001
8002 void
8003 t4_os_portmod_changed(const struct adapter *sc, int idx)
8004 {
8005         struct port_info *pi = sc->port[idx];
8006         static const char *mod_str[] = {
8007                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
8008         };
8009
8010         build_medialist(pi, &pi->media);
8011 #ifdef DEV_NETMAP
8012         build_medialist(pi, &pi->nm_media);
8013 #endif
8014
8015         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
8016                 if_printf(pi->ifp, "transceiver unplugged.\n");
8017         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
8018                 if_printf(pi->ifp, "unknown transceiver inserted.\n");
8019         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
8020                 if_printf(pi->ifp, "unsupported transceiver inserted.\n");
8021         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
8022                 if_printf(pi->ifp, "%s transceiver inserted.\n",
8023                     mod_str[pi->mod_type]);
8024         } else {
8025                 if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
8026                     pi->mod_type);
8027         }
8028 }
8029
8030 void
8031 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
8032 {
8033         struct port_info *pi = sc->port[idx];
8034         struct ifnet *ifp = pi->ifp;
8035
8036         if (link_stat) {
8037                 pi->linkdnrc = -1;
8038                 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
8039                 if_link_state_change(ifp, LINK_STATE_UP);
8040         } else {
8041                 if (reason >= 0)
8042                         pi->linkdnrc = reason;
8043                 if_link_state_change(ifp, LINK_STATE_DOWN);
8044         }
8045 }
8046
8047 void
8048 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
8049 {
8050         struct adapter *sc;
8051
8052         sx_slock(&t4_list_lock);
8053         SLIST_FOREACH(sc, &t4_list, link) {
8054                 /*
8055                  * func should not make any assumptions about what state sc is
8056                  * in - the only guarantee is that sc->sc_lock is a valid lock.
8057                  */
8058                 func(sc, arg);
8059         }
8060         sx_sunlock(&t4_list_lock);
8061 }
8062
8063 static int
8064 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
8065 {
8066        return (0);
8067 }
8068
8069 static int
8070 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
8071 {
8072        return (0);
8073 }
8074
8075 static int
8076 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
8077     struct thread *td)
8078 {
8079         int rc;
8080         struct adapter *sc = dev->si_drv1;
8081
8082         rc = priv_check(td, PRIV_DRIVER);
8083         if (rc != 0)
8084                 return (rc);
8085
8086         switch (cmd) {
8087         case CHELSIO_T4_GETREG: {
8088                 struct t4_reg *edata = (struct t4_reg *)data;
8089
8090                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8091                         return (EFAULT);
8092
8093                 if (edata->size == 4)
8094                         edata->val = t4_read_reg(sc, edata->addr);
8095                 else if (edata->size == 8)
8096                         edata->val = t4_read_reg64(sc, edata->addr);
8097                 else
8098                         return (EINVAL);
8099
8100                 break;
8101         }
8102         case CHELSIO_T4_SETREG: {
8103                 struct t4_reg *edata = (struct t4_reg *)data;
8104
8105                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8106                         return (EFAULT);
8107
8108                 if (edata->size == 4) {
8109                         if (edata->val & 0xffffffff00000000)
8110                                 return (EINVAL);
8111                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
8112                 } else if (edata->size == 8)
8113                         t4_write_reg64(sc, edata->addr, edata->val);
8114                 else
8115                         return (EINVAL);
8116                 break;
8117         }
8118         case CHELSIO_T4_REGDUMP: {
8119                 struct t4_regdump *regs = (struct t4_regdump *)data;
8120                 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
8121                 uint8_t *buf;
8122
8123                 if (regs->len < reglen) {
8124                         regs->len = reglen; /* hint to the caller */
8125                         return (ENOBUFS);
8126                 }
8127
8128                 regs->len = reglen;
8129                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
8130                 t4_get_regs(sc, regs, buf);
8131                 rc = copyout(buf, regs->data, reglen);
8132                 free(buf, M_CXGBE);
8133                 break;
8134         }
8135         case CHELSIO_T4_GET_FILTER_MODE:
8136                 rc = get_filter_mode(sc, (uint32_t *)data);
8137                 break;
8138         case CHELSIO_T4_SET_FILTER_MODE:
8139                 rc = set_filter_mode(sc, *(uint32_t *)data);
8140                 break;
8141         case CHELSIO_T4_GET_FILTER:
8142                 rc = get_filter(sc, (struct t4_filter *)data);
8143                 break;
8144         case CHELSIO_T4_SET_FILTER:
8145                 rc = set_filter(sc, (struct t4_filter *)data);
8146                 break;
8147         case CHELSIO_T4_DEL_FILTER:
8148                 rc = del_filter(sc, (struct t4_filter *)data);
8149                 break;
8150         case CHELSIO_T4_GET_SGE_CONTEXT:
8151                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
8152                 break;
8153         case CHELSIO_T4_LOAD_FW:
8154                 rc = load_fw(sc, (struct t4_data *)data);
8155                 break;
8156         case CHELSIO_T4_GET_MEM:
8157                 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
8158                 break;
8159         case CHELSIO_T4_GET_I2C:
8160                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
8161                 break;
8162         case CHELSIO_T4_CLEAR_STATS: {
8163                 int i;
8164                 u_int port_id = *(uint32_t *)data;
8165                 struct port_info *pi;
8166
8167                 if (port_id >= sc->params.nports)
8168                         return (EINVAL);
8169                 pi = sc->port[port_id];
8170
8171                 /* MAC stats */
8172                 t4_clr_port_stats(sc, pi->tx_chan);
8173
8174                 if (pi->flags & PORT_INIT_DONE) {
8175                         struct sge_rxq *rxq;
8176                         struct sge_txq *txq;
8177                         struct sge_wrq *wrq;
8178
8179                         for_each_rxq(pi, i, rxq) {
8180 #if defined(INET) || defined(INET6)
8181                                 rxq->lro.lro_queued = 0;
8182                                 rxq->lro.lro_flushed = 0;
8183 #endif
8184                                 rxq->rxcsum = 0;
8185                                 rxq->vlan_extraction = 0;
8186                         }
8187
8188                         for_each_txq(pi, i, txq) {
8189                                 txq->txcsum = 0;
8190                                 txq->tso_wrs = 0;
8191                                 txq->vlan_insertion = 0;
8192                                 txq->imm_wrs = 0;
8193                                 txq->sgl_wrs = 0;
8194                                 txq->txpkt_wrs = 0;
8195                                 txq->txpkts_wrs = 0;
8196                                 txq->txpkts_pkts = 0;
8197                                 txq->br->br_drops = 0;
8198                                 txq->no_dmamap = 0;
8199                                 txq->no_desc = 0;
8200                         }
8201
8202 #ifdef TCP_OFFLOAD
8203                         /* nothing to clear for each ofld_rxq */
8204
8205                         for_each_ofld_txq(pi, i, wrq) {
8206                                 wrq->tx_wrs = 0;
8207                                 wrq->no_desc = 0;
8208                         }
8209 #endif
8210                         wrq = &sc->sge.ctrlq[pi->port_id];
8211                         wrq->tx_wrs = 0;
8212                         wrq->no_desc = 0;
8213                 }
8214                 break;
8215         }
8216         case CHELSIO_T4_SCHED_CLASS:
8217                 rc = set_sched_class(sc, (struct t4_sched_params *)data);
8218                 break;
8219         case CHELSIO_T4_SCHED_QUEUE:
8220                 rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
8221                 break;
8222         case CHELSIO_T4_GET_TRACER:
8223                 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
8224                 break;
8225         case CHELSIO_T4_SET_TRACER:
8226                 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
8227                 break;
8228         default:
8229                 rc = EINVAL;
8230         }
8231
8232         return (rc);
8233 }
8234
8235 #ifdef TCP_OFFLOAD
8236 void
8237 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask,
8238     const unsigned int *pgsz_order)
8239 {
8240         struct port_info *pi = ifp->if_softc;
8241         struct adapter *sc = pi->adapter;
8242
8243         t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
8244         t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |
8245                 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) |
8246                 V_HPZ3(pgsz_order[3]));
8247 }
8248
8249 static int
8250 toe_capability(struct port_info *pi, int enable)
8251 {
8252         int rc;
8253         struct adapter *sc = pi->adapter;
8254
8255         ASSERT_SYNCHRONIZED_OP(sc);
8256
8257         if (!is_offload(sc))
8258                 return (ENODEV);
8259
8260         if (enable) {
8261                 if (!(sc->flags & FULL_INIT_DONE)) {
8262                         rc = cxgbe_init_synchronized(pi);
8263                         if (rc)
8264                                 return (rc);
8265                 }
8266
8267                 if (isset(&sc->offload_map, pi->port_id))
8268                         return (0);
8269
8270                 if (!(sc->flags & TOM_INIT_DONE)) {
8271                         rc = t4_activate_uld(sc, ULD_TOM);
8272                         if (rc == EAGAIN) {
8273                                 log(LOG_WARNING,
8274                                     "You must kldload t4_tom.ko before trying "
8275                                     "to enable TOE on a cxgbe interface.\n");
8276                         }
8277                         if (rc != 0)
8278                                 return (rc);
8279                         KASSERT(sc->tom_softc != NULL,
8280                             ("%s: TOM activated but softc NULL", __func__));
8281                         KASSERT(sc->flags & TOM_INIT_DONE,
8282                             ("%s: TOM activated but flag not set", __func__));
8283                 }
8284
8285                 setbit(&sc->offload_map, pi->port_id);
8286         } else {
8287                 if (!isset(&sc->offload_map, pi->port_id))
8288                         return (0);
8289
8290                 KASSERT(sc->flags & TOM_INIT_DONE,
8291                     ("%s: TOM never initialized?", __func__));
8292                 clrbit(&sc->offload_map, pi->port_id);
8293         }
8294
8295         return (0);
8296 }
8297
8298 /*
8299  * Add an upper layer driver to the global list.
8300  */
8301 int
8302 t4_register_uld(struct uld_info *ui)
8303 {
8304         int rc = 0;
8305         struct uld_info *u;
8306
8307         sx_xlock(&t4_uld_list_lock);
8308         SLIST_FOREACH(u, &t4_uld_list, link) {
8309             if (u->uld_id == ui->uld_id) {
8310                     rc = EEXIST;
8311                     goto done;
8312             }
8313         }
8314
8315         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
8316         ui->refcount = 0;
8317 done:
8318         sx_xunlock(&t4_uld_list_lock);
8319         return (rc);
8320 }
8321
8322 int
8323 t4_unregister_uld(struct uld_info *ui)
8324 {
8325         int rc = EINVAL;
8326         struct uld_info *u;
8327
8328         sx_xlock(&t4_uld_list_lock);
8329
8330         SLIST_FOREACH(u, &t4_uld_list, link) {
8331             if (u == ui) {
8332                     if (ui->refcount > 0) {
8333                             rc = EBUSY;
8334                             goto done;
8335                     }
8336
8337                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
8338                     rc = 0;
8339                     goto done;
8340             }
8341         }
8342 done:
8343         sx_xunlock(&t4_uld_list_lock);
8344         return (rc);
8345 }
8346
8347 int
8348 t4_activate_uld(struct adapter *sc, int id)
8349 {
8350         int rc = EAGAIN;
8351         struct uld_info *ui;
8352
8353         ASSERT_SYNCHRONIZED_OP(sc);
8354
8355         sx_slock(&t4_uld_list_lock);
8356
8357         SLIST_FOREACH(ui, &t4_uld_list, link) {
8358                 if (ui->uld_id == id) {
8359                         if (!(sc->flags & FULL_INIT_DONE)) {
8360                                 rc = adapter_full_init(sc);
8361                                 if (rc != 0)
8362                                         goto done;
8363                         }
8364
8365                         rc = ui->activate(sc);
8366                         if (rc == 0)
8367                                 ui->refcount++;
8368                         goto done;
8369                 }
8370         }
8371 done:
8372         sx_sunlock(&t4_uld_list_lock);
8373
8374         return (rc);
8375 }
8376
8377 int
8378 t4_deactivate_uld(struct adapter *sc, int id)
8379 {
8380         int rc = EINVAL;
8381         struct uld_info *ui;
8382
8383         ASSERT_SYNCHRONIZED_OP(sc);
8384
8385         sx_slock(&t4_uld_list_lock);
8386
8387         SLIST_FOREACH(ui, &t4_uld_list, link) {
8388                 if (ui->uld_id == id) {
8389                         rc = ui->deactivate(sc);
8390                         if (rc == 0)
8391                                 ui->refcount--;
8392                         goto done;
8393                 }
8394         }
8395 done:
8396         sx_sunlock(&t4_uld_list_lock);
8397
8398         return (rc);
8399 }
8400 #endif
8401
8402 /*
8403  * Come up with reasonable defaults for some of the tunables, provided they're
8404  * not set by the user (in which case we'll use the values as is).
8405  */
8406 static void
8407 tweak_tunables(void)
8408 {
8409         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
8410
8411         if (t4_ntxq10g < 1)
8412                 t4_ntxq10g = min(nc, NTXQ_10G);
8413
8414         if (t4_ntxq1g < 1)
8415                 t4_ntxq1g = min(nc, NTXQ_1G);
8416
8417         if (t4_nrxq10g < 1)
8418                 t4_nrxq10g = min(nc, NRXQ_10G);
8419
8420         if (t4_nrxq1g < 1)
8421                 t4_nrxq1g = min(nc, NRXQ_1G);
8422
8423 #ifdef TCP_OFFLOAD
8424         if (t4_nofldtxq10g < 1)
8425                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
8426
8427         if (t4_nofldtxq1g < 1)
8428                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
8429
8430         if (t4_nofldrxq10g < 1)
8431                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
8432
8433         if (t4_nofldrxq1g < 1)
8434                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
8435
8436         if (t4_toecaps_allowed == -1)
8437                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
8438 #else
8439         if (t4_toecaps_allowed == -1)
8440                 t4_toecaps_allowed = 0;
8441 #endif
8442
8443 #ifdef DEV_NETMAP
8444         if (t4_nnmtxq10g < 1)
8445                 t4_nnmtxq10g = min(nc, NNMTXQ_10G);
8446
8447         if (t4_nnmtxq1g < 1)
8448                 t4_nnmtxq1g = min(nc, NNMTXQ_1G);
8449
8450         if (t4_nnmrxq10g < 1)
8451                 t4_nnmrxq10g = min(nc, NNMRXQ_10G);
8452
8453         if (t4_nnmrxq1g < 1)
8454                 t4_nnmrxq1g = min(nc, NNMRXQ_1G);
8455 #endif
8456
8457         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
8458                 t4_tmr_idx_10g = TMR_IDX_10G;
8459
8460         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
8461                 t4_pktc_idx_10g = PKTC_IDX_10G;
8462
8463         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
8464                 t4_tmr_idx_1g = TMR_IDX_1G;
8465
8466         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
8467                 t4_pktc_idx_1g = PKTC_IDX_1G;
8468
8469         if (t4_qsize_txq < 128)
8470                 t4_qsize_txq = 128;
8471
8472         if (t4_qsize_rxq < 128)
8473                 t4_qsize_rxq = 128;
8474         while (t4_qsize_rxq & 7)
8475                 t4_qsize_rxq++;
8476
8477         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
8478 }
8479
8480 static struct sx mlu;   /* mod load unload */
8481 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
8482
8483 static int
8484 mod_event(module_t mod, int cmd, void *arg)
8485 {
8486         int rc = 0;
8487         static int loaded = 0;
8488
8489         switch (cmd) {
8490         case MOD_LOAD:
8491                 sx_xlock(&mlu);
8492                 if (loaded++ == 0) {
8493                         t4_sge_modload();
8494                         sx_init(&t4_list_lock, "T4/T5 adapters");
8495                         SLIST_INIT(&t4_list);
8496 #ifdef TCP_OFFLOAD
8497                         sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
8498                         SLIST_INIT(&t4_uld_list);
8499 #endif
8500                         t4_tracer_modload();
8501                         tweak_tunables();
8502                 }
8503                 sx_xunlock(&mlu);
8504                 break;
8505
8506         case MOD_UNLOAD:
8507                 sx_xlock(&mlu);
8508                 if (--loaded == 0) {
8509                         int tries;
8510
8511                         sx_slock(&t4_list_lock);
8512                         if (!SLIST_EMPTY(&t4_list)) {
8513                                 rc = EBUSY;
8514                                 sx_sunlock(&t4_list_lock);
8515                                 goto done_unload;
8516                         }
8517 #ifdef TCP_OFFLOAD
8518                         sx_slock(&t4_uld_list_lock);
8519                         if (!SLIST_EMPTY(&t4_uld_list)) {
8520                                 rc = EBUSY;
8521                                 sx_sunlock(&t4_uld_list_lock);
8522                                 sx_sunlock(&t4_list_lock);
8523                                 goto done_unload;
8524                         }
8525 #endif
8526                         tries = 0;
8527                         while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
8528                                 uprintf("%ju clusters with custom free routine "
8529                                     "still is use.\n", t4_sge_extfree_refs());
8530                                 pause("t4unload", 2 * hz);
8531                         }
8532 #ifdef TCP_OFFLOAD
8533                         sx_sunlock(&t4_uld_list_lock);
8534 #endif
8535                         sx_sunlock(&t4_list_lock);
8536
8537                         if (t4_sge_extfree_refs() == 0) {
8538                                 t4_tracer_modunload();
8539 #ifdef TCP_OFFLOAD
8540                                 sx_destroy(&t4_uld_list_lock);
8541 #endif
8542                                 sx_destroy(&t4_list_lock);
8543                                 t4_sge_modunload();
8544                                 loaded = 0;
8545                         } else {
8546                                 rc = EBUSY;
8547                                 loaded++;       /* undo earlier decrement */
8548                         }
8549                 }
8550 done_unload:
8551                 sx_xunlock(&mlu);
8552                 break;
8553         }
8554
8555         return (rc);
8556 }
8557
8558 static devclass_t t4_devclass, t5_devclass;
8559 static devclass_t cxgbe_devclass, cxl_devclass;
8560
8561 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
8562 MODULE_VERSION(t4nex, 1);
8563 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
8564
8565 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
8566 MODULE_VERSION(t5nex, 1);
8567 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
8568
8569 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
8570 MODULE_VERSION(cxgbe, 1);
8571
8572 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
8573 MODULE_VERSION(cxl, 1);