sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/conf.h>
  36 #include <sys/priv.h>
  37 #include <sys/kernel.h>
  38 #include <sys/bus.h>
  39 #include <sys/module.h>
  40 #include <sys/malloc.h>
  41 #include <sys/queue.h>
  42 #include <sys/taskqueue.h>
  43 #include <sys/pciio.h>
  44 #include <dev/pci/pcireg.h>
  45 #include <dev/pci/pcivar.h>
  46 #include <dev/pci/pci_private.h>
  47 #include <sys/firmware.h>
  48 #include <sys/sbuf.h>
  49 #include <sys/smp.h>
  50 #include <sys/socket.h>
  51 #include <sys/sockio.h>
  52 #include <sys/sysctl.h>
  53 #include <net/ethernet.h>
  54 #include <net/if.h>
  55 #include <net/if_types.h>
  56 #include <net/if_dl.h>
  57 #include <net/if_vlan_var.h>
  58 #if defined(__i386__) || defined(__amd64__)
  59 #include <vm/vm.h>
  60 #include <vm/pmap.h>
  61 #endif
  62
  63 #include "common/common.h"
  64 #include "common/t4_msg.h"
  65 #include "common/t4_regs.h"
  66 #include "common/t4_regs_values.h"
  67 #include "t4_ioctl.h"
  68 #include "t4_l2t.h"
  69
  70 /* T4 bus driver interface */
  71 static int t4_probe(device_t);
  72 static int t4_attach(device_t);
  73 static int t4_detach(device_t);
  74 static device_method_t t4_methods[] = {
  75         DEVMETHOD(device_probe,         t4_probe),
  76         DEVMETHOD(device_attach,        t4_attach),
  77         DEVMETHOD(device_detach,        t4_detach),
  78
  79         DEVMETHOD_END
  80 };
  81 static driver_t t4_driver = {
  82         "t4nex",
  83         t4_methods,
  84         sizeof(struct adapter)
  85 };
  86
  87
  88 /* T4 port (cxgbe) interface */
  89 static int cxgbe_probe(device_t);
  90 static int cxgbe_attach(device_t);
  91 static int cxgbe_detach(device_t);
  92 static device_method_t cxgbe_methods[] = {
  93         DEVMETHOD(device_probe,         cxgbe_probe),
  94         DEVMETHOD(device_attach,        cxgbe_attach),
  95         DEVMETHOD(device_detach,        cxgbe_detach),
  96         { 0, 0 }
  97 };
  98 static driver_t cxgbe_driver = {
  99         "cxgbe",
 100         cxgbe_methods,
 101         sizeof(struct port_info)
 102 };
 103
 104 static d_ioctl_t t4_ioctl;
 105 static d_open_t t4_open;
 106 static d_close_t t4_close;
 107
 108 static struct cdevsw t4_cdevsw = {
 109        .d_version = D_VERSION,
 110        .d_flags = 0,
 111        .d_open = t4_open,
 112        .d_close = t4_close,
 113        .d_ioctl = t4_ioctl,
 114        .d_name = "t4nex",
 115 };
 116
 117 /* T5 bus driver interface */
 118 static int t5_probe(device_t);
 119 static device_method_t t5_methods[] = {
 120         DEVMETHOD(device_probe,         t5_probe),
 121         DEVMETHOD(device_attach,        t4_attach),
 122         DEVMETHOD(device_detach,        t4_detach),
 123
 124         DEVMETHOD_END
 125 };
 126 static driver_t t5_driver = {
 127         "t5nex",
 128         t5_methods,
 129         sizeof(struct adapter)
 130 };
 131
 132
 133 /* T5 port (cxl) interface */
 134 static driver_t cxl_driver = {
 135         "cxl",
 136         cxgbe_methods,
 137         sizeof(struct port_info)
 138 };
 139
 140 static struct cdevsw t5_cdevsw = {
 141        .d_version = D_VERSION,
 142        .d_flags = 0,
 143        .d_open = t4_open,
 144        .d_close = t4_close,
 145        .d_ioctl = t4_ioctl,
 146        .d_name = "t5nex",
 147 };
 148
 149 /* ifnet + media interface */
 150 static void cxgbe_init(void *);
 151 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 152 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 153 static void cxgbe_qflush(struct ifnet *);
 154 static int cxgbe_media_change(struct ifnet *);
 155 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 156
 157 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
 158
 159 /*
 160  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 161  * then ADAPTER_LOCK, then t4_uld_list_lock.
 162  */
 163 static struct sx t4_list_lock;
 164 SLIST_HEAD(, adapter) t4_list;
 165 #ifdef TCP_OFFLOAD
 166 static struct sx t4_uld_list_lock;
 167 SLIST_HEAD(, uld_info) t4_uld_list;
 168 #endif
 169
 170 /*
 171  * Tunables.  See tweak_tunables() too.
 172  *
 173  * Each tunable is set to a default value here if it's known at compile-time.
 174  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
 175  * provide a reasonable default when the driver is loaded.
 176  *
 177  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
 178  * T5 are under hw.cxl.
 179  */
 180
 181 /*
 182  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 183  */
 184 #define NTXQ_10G 16
 185 static int t4_ntxq10g = -1;
 186 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 187
 188 #define NRXQ_10G 8
 189 static int t4_nrxq10g = -1;
 190 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 191
 192 #define NTXQ_1G 4
 193 static int t4_ntxq1g = -1;
 194 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 195
 196 #define NRXQ_1G 2
 197 static int t4_nrxq1g = -1;
 198 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 199
 200 static int t4_rsrv_noflowq = 0;
 201 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
 202
 203 #ifdef TCP_OFFLOAD
 204 #define NOFLDTXQ_10G 8
 205 static int t4_nofldtxq10g = -1;
 206 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 207
 208 #define NOFLDRXQ_10G 2
 209 static int t4_nofldrxq10g = -1;
 210 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 211
 212 #define NOFLDTXQ_1G 2
 213 static int t4_nofldtxq1g = -1;
 214 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 215
 216 #define NOFLDRXQ_1G 1
 217 static int t4_nofldrxq1g = -1;
 218 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 219 #endif
 220
 221 #ifdef DEV_NETMAP
 222 #define NNMTXQ_10G 2
 223 static int t4_nnmtxq10g = -1;
 224 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g);
 225
 226 #define NNMRXQ_10G 2
 227 static int t4_nnmrxq10g = -1;
 228 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g);
 229
 230 #define NNMTXQ_1G 1
 231 static int t4_nnmtxq1g = -1;
 232 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g);
 233
 234 #define NNMRXQ_1G 1
 235 static int t4_nnmrxq1g = -1;
 236 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g);
 237 #endif
 238
 239 /*
 240  * Holdoff parameters for 10G and 1G ports.
 241  */
 242 #define TMR_IDX_10G 1
 243 static int t4_tmr_idx_10g = TMR_IDX_10G;
 244 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 245
 246 #define PKTC_IDX_10G (-1)
 247 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 248 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 249
 250 #define TMR_IDX_1G 1
 251 static int t4_tmr_idx_1g = TMR_IDX_1G;
 252 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 253
 254 #define PKTC_IDX_1G (-1)
 255 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 256 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 257
 258 /*
 259  * Size (# of entries) of each tx and rx queue.
 260  */
 261 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 262 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 263
 264 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 265 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 266
 267 /*
 268  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 269  */
 270 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 271 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 272
 273 /*
 274  * Configuration file.
 275  */
 276 #define DEFAULT_CF      "default"
 277 #define FLASH_CF        "flash"
 278 #define UWIRE_CF        "uwire"
 279 #define FPGA_CF         "fpga"
 280 static char t4_cfg_file[32] = DEFAULT_CF;
 281 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 282
 283 /*
 284  * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively).
 285  * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
 286  * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
 287  *            mark or when signalled to do so, 0 to never emit PAUSE.
 288  */
 289 static int t4_pause_settings = PAUSE_TX | PAUSE_RX;
 290 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings);
 291
 292 /*
 293  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
 294  * encouraged respectively).
 295  */
 296 static unsigned int t4_fw_install = 1;
 297 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 298
 299 /*
 300  * ASIC features that will be used.  Disable the ones you don't want so that the
 301  * chip resources aren't wasted on features that will not be used.
 302  */
 303 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 304 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 305
 306 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 307 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 308
 309 static int t4_toecaps_allowed = -1;
 310 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 311
 312 static int t4_rdmacaps_allowed = 0;
 313 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 314
 315 static int t4_iscsicaps_allowed = 0;
 316 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 317
 318 static int t4_fcoecaps_allowed = 0;
 319 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 320
 321 static int t5_write_combine = 0;
 322 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 323
 324 struct intrs_and_queues {
 325         uint16_t intr_type;     /* INTx, MSI, or MSI-X */
 326         uint16_t nirq;          /* Total # of vectors */
 327         uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
 328         uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */
 329         uint16_t ntxq10g;       /* # of NIC txq's for each 10G port */
 330         uint16_t nrxq10g;       /* # of NIC rxq's for each 10G port */
 331         uint16_t ntxq1g;        /* # of NIC txq's for each 1G port */
 332         uint16_t nrxq1g;        /* # of NIC rxq's for each 1G port */
 333         uint16_t rsrv_noflowq;  /* Flag whether to reserve queue 0 */
 334 #ifdef TCP_OFFLOAD
 335         uint16_t nofldtxq10g;   /* # of TOE txq's for each 10G port */
 336         uint16_t nofldrxq10g;   /* # of TOE rxq's for each 10G port */
 337         uint16_t nofldtxq1g;    /* # of TOE txq's for each 1G port */
 338         uint16_t nofldrxq1g;    /* # of TOE rxq's for each 1G port */
 339 #endif
 340 #ifdef DEV_NETMAP
 341         uint16_t nnmtxq10g;     /* # of netmap txq's for each 10G port */
 342         uint16_t nnmrxq10g;     /* # of netmap rxq's for each 10G port */
 343         uint16_t nnmtxq1g;      /* # of netmap txq's for each 1G port */
 344         uint16_t nnmrxq1g;      /* # of netmap rxq's for each 1G port */
 345 #endif
 346 };
 347
 348 struct filter_entry {
 349         uint32_t valid:1;       /* filter allocated and valid */
 350         uint32_t locked:1;      /* filter is administratively locked */
 351         uint32_t pending:1;     /* filter action is pending firmware reply */
 352         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 353         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 354
 355         struct t4_filter_specification fs;
 356 };
 357
 358 static int map_bars_0_and_4(struct adapter *);
 359 static int map_bar_2(struct adapter *);
 360 static void setup_memwin(struct adapter *);
 361 static int validate_mem_range(struct adapter *, uint32_t, int);
 362 static int fwmtype_to_hwmtype(int);
 363 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
 364     uint32_t *);
 365 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
 366 static uint32_t position_memwin(struct adapter *, int, uint32_t);
 367 static int cfg_itype_and_nqueues(struct adapter *, int, int,
 368     struct intrs_and_queues *);
 369 static int prep_firmware(struct adapter *);
 370 static int partition_resources(struct adapter *, const struct firmware *,
 371     const char *);
 372 static int get_params__pre_init(struct adapter *);
 373 static int get_params__post_init(struct adapter *);
 374 static int set_params__post_init(struct adapter *);
 375 static void t4_set_desc(struct adapter *);
 376 static void build_medialist(struct port_info *, struct ifmedia *);
 377 static int cxgbe_init_synchronized(struct port_info *);
 378 static int cxgbe_uninit_synchronized(struct port_info *);
 379 static int setup_intr_handlers(struct adapter *);
 380 static void quiesce_eq(struct adapter *, struct sge_eq *);
 381 static void quiesce_iq(struct adapter *, struct sge_iq *);
 382 static void quiesce_fl(struct adapter *, struct sge_fl *);
 383 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 384     driver_intr_t *, void *, char *);
 385 static int t4_free_irq(struct adapter *, struct irq *);
 386 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
 387     unsigned int);
 388 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 389 static void cxgbe_tick(void *);
 390 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 391 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
 392     struct mbuf *);
 393 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 394 static int fw_msg_not_handled(struct adapter *, const __be64 *);
 395 static int t4_sysctls(struct adapter *);
 396 static int cxgbe_sysctls(struct port_info *);
 397 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 398 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 399 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
 400 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 401 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 402 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 403 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 404 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 405 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
 406 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 407 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 408 #ifdef SBUF_DRAIN
 409 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 410 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 411 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 412 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
 413 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
 414 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 415 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 416 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 417 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 418 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 419 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 420 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 421 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
 422 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 423 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
 424 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 425 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 426 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 427 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 428 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 429 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 430 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
 431 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 432 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 433 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 434 #endif
 435 static inline void txq_start(struct ifnet *, struct sge_txq *);
 436 static uint32_t fconf_to_mode(uint32_t);
 437 static uint32_t mode_to_fconf(uint32_t);
 438 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 439 static int get_filter_mode(struct adapter *, uint32_t *);
 440 static int set_filter_mode(struct adapter *, uint32_t);
 441 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 442 static int get_filter(struct adapter *, struct t4_filter *);
 443 static int set_filter(struct adapter *, struct t4_filter *);
 444 static int del_filter(struct adapter *, struct t4_filter *);
 445 static void clear_filter(struct filter_entry *);
 446 static int set_filter_wr(struct adapter *, int);
 447 static int del_filter_wr(struct adapter *, int);
 448 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 449 static int load_fw(struct adapter *, struct t4_data *);
 450 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 451 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 452 static int set_sched_class(struct adapter *, struct t4_sched_params *);
 453 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
 454 #ifdef TCP_OFFLOAD
 455 static int toe_capability(struct port_info *, int);
 456 #endif
 457 static int mod_event(module_t, int, void *);
 458
 459 struct {
 460         uint16_t device;
 461         char *desc;
 462 } t4_pciids[] = {
 463         {0xa000, "Chelsio Terminator 4 FPGA"},
 464         {0x4400, "Chelsio T440-dbg"},
 465         {0x4401, "Chelsio T420-CR"},
 466         {0x4402, "Chelsio T422-CR"},
 467         {0x4403, "Chelsio T440-CR"},
 468         {0x4404, "Chelsio T420-BCH"},
 469         {0x4405, "Chelsio T440-BCH"},
 470         {0x4406, "Chelsio T440-CH"},
 471         {0x4407, "Chelsio T420-SO"},
 472         {0x4408, "Chelsio T420-CX"},
 473         {0x4409, "Chelsio T420-BT"},
 474         {0x440a, "Chelsio T404-BT"},
 475         {0x440e, "Chelsio T440-LP-CR"},
 476 }, t5_pciids[] = {
 477         {0xb000, "Chelsio Terminator 5 FPGA"},
 478         {0x5400, "Chelsio T580-dbg"},
 479         {0x5401,  "Chelsio T520-CR"},           /* 2 x 10G */
 480         {0x5402,  "Chelsio T522-CR"},           /* 2 x 10G, 2 X 1G */
 481         {0x5403,  "Chelsio T540-CR"},           /* 4 x 10G */
 482         {0x5407,  "Chelsio T520-SO"},           /* 2 x 10G, nomem */
 483         {0x5409,  "Chelsio T520-BT"},           /* 2 x 10GBaseT */
 484         {0x540a,  "Chelsio T504-BT"},           /* 4 x 1G */
 485         {0x540d,  "Chelsio T580-CR"},           /* 2 x 40G */
 486         {0x540e,  "Chelsio T540-LP-CR"},        /* 4 x 10G */
 487         {0x5410,  "Chelsio T580-LP-CR"},        /* 2 x 40G */
 488         {0x5411,  "Chelsio T520-LL-CR"},        /* 2 x 10G */
 489         {0x5412,  "Chelsio T560-CR"},           /* 1 x 40G, 2 x 10G */
 490         {0x5414,  "Chelsio T580-LP-SO-CR"},     /* 2 x 40G, nomem */
 491 #ifdef notyet
 492         {0x5404,  "Chelsio T520-BCH"},
 493         {0x5405,  "Chelsio T540-BCH"},
 494         {0x5406,  "Chelsio T540-CH"},
 495         {0x5408,  "Chelsio T520-CX"},
 496         {0x540b,  "Chelsio B520-SR"},
 497         {0x540c,  "Chelsio B504-BT"},
 498         {0x540f,  "Chelsio Amsterdam"},
 499         {0x5413,  "Chelsio T580-CHR"},
 500 #endif
 501 };
 502
 503 #ifdef TCP_OFFLOAD
 504 /*
 505  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 506  * exactly the same for both rxq and ofld_rxq.
 507  */
 508 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 509 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 510 #endif
 511
 512 /* No easy way to include t4_msg.h before adapter.h so we check this way */
 513 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
 514 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
 515
 516 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
 517
 518 static int
 519 t4_probe(device_t dev)
 520 {
 521         int i;
 522         uint16_t v = pci_get_vendor(dev);
 523         uint16_t d = pci_get_device(dev);
 524         uint8_t f = pci_get_function(dev);
 525
 526         if (v != PCI_VENDOR_ID_CHELSIO)
 527                 return (ENXIO);
 528
 529         /* Attach only to PF0 of the FPGA */
 530         if (d == 0xa000 && f != 0)
 531                 return (ENXIO);
 532
 533         for (i = 0; i < nitems(t4_pciids); i++) {
 534                 if (d == t4_pciids[i].device) {
 535                         device_set_desc(dev, t4_pciids[i].desc);
 536                         return (BUS_PROBE_DEFAULT);
 537                 }
 538         }
 539
 540         return (ENXIO);
 541 }
 542
 543 static int
 544 t5_probe(device_t dev)
 545 {
 546         int i;
 547         uint16_t v = pci_get_vendor(dev);
 548         uint16_t d = pci_get_device(dev);
 549         uint8_t f = pci_get_function(dev);
 550
 551         if (v != PCI_VENDOR_ID_CHELSIO)
 552                 return (ENXIO);
 553
 554         /* Attach only to PF0 of the FPGA */
 555         if (d == 0xb000 && f != 0)
 556                 return (ENXIO);
 557
 558         for (i = 0; i < nitems(t5_pciids); i++) {
 559                 if (d == t5_pciids[i].device) {
 560                         device_set_desc(dev, t5_pciids[i].desc);
 561                         return (BUS_PROBE_DEFAULT);
 562                 }
 563         }
 564
 565         return (ENXIO);
 566 }
 567
 568 static int
 569 t4_attach(device_t dev)
 570 {
 571         struct adapter *sc;
 572         int rc = 0, i, n10g, n1g, rqidx, tqidx;
 573         struct intrs_and_queues iaq;
 574         struct sge *s;
 575 #ifdef TCP_OFFLOAD
 576         int ofld_rqidx, ofld_tqidx;
 577 #endif
 578 #ifdef DEV_NETMAP
 579         int nm_rqidx, nm_tqidx;
 580 #endif
 581
 582         sc = device_get_softc(dev);
 583         sc->dev = dev;
 584
 585         pci_enable_busmaster(dev);
 586         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 587                 uint32_t v;
 588
 589                 pci_set_max_read_req(dev, 4096);
 590                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 591                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 592                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 593
 594                 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
 595         }
 596
 597         sc->traceq = -1;
 598         mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
 599         snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
 600             device_get_nameunit(dev));
 601
 602         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 603             device_get_nameunit(dev));
 604         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 605         sx_xlock(&t4_list_lock);
 606         SLIST_INSERT_HEAD(&t4_list, sc, link);
 607         sx_xunlock(&t4_list_lock);
 608
 609         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 610         TAILQ_INIT(&sc->sfl);
 611         callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
 612
 613         rc = map_bars_0_and_4(sc);
 614         if (rc != 0)
 615                 goto done; /* error message displayed already */
 616
 617         /*
 618          * This is the real PF# to which we're attaching.  Works from within PCI
 619          * passthrough environments too, where pci_get_function() could return a
 620          * different PF# depending on the passthrough configuration.  We need to
 621          * use the real PF# in all our communication with the firmware.
 622          */
 623         sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 624         sc->mbox = sc->pf;
 625
 626         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 627         sc->an_handler = an_not_handled;
 628         for (i = 0; i < nitems(sc->cpl_handler); i++)
 629                 sc->cpl_handler[i] = cpl_not_handled;
 630         for (i = 0; i < nitems(sc->fw_msg_handler); i++)
 631                 sc->fw_msg_handler[i] = fw_msg_not_handled;
 632         t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 633         t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
 634         t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt);
 635         t4_init_sge_cpl_handlers(sc);
 636
 637         /* Prepare the adapter for operation */
 638         rc = -t4_prep_adapter(sc);
 639         if (rc != 0) {
 640                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 641                 goto done;
 642         }
 643
 644         /*
 645          * Do this really early, with the memory windows set up even before the
 646          * character device.  The userland tool's register i/o and mem read
 647          * will work even in "recovery mode".
 648          */
 649         setup_memwin(sc);
 650         sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
 651             device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
 652             device_get_nameunit(dev));
 653         if (sc->cdev == NULL)
 654                 device_printf(dev, "failed to create nexus char device.\n");
 655         else
 656                 sc->cdev->si_drv1 = sc;
 657
 658         /* Go no further if recovery mode has been requested. */
 659         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 660                 device_printf(dev, "recovery mode.\n");
 661                 goto done;
 662         }
 663
 664         /* Prepare the firmware for operation */
 665         rc = prep_firmware(sc);
 666         if (rc != 0)
 667                 goto done; /* error message displayed already */
 668
 669         rc = get_params__post_init(sc);
 670         if (rc != 0)
 671                 goto done; /* error message displayed already */
 672
 673         rc = set_params__post_init(sc);
 674         if (rc != 0)
 675                 goto done; /* error message displayed already */
 676
 677         rc = map_bar_2(sc);
 678         if (rc != 0)
 679                 goto done; /* error message displayed already */
 680
 681         rc = t4_create_dma_tag(sc);
 682         if (rc != 0)
 683                 goto done; /* error message displayed already */
 684
 685         /*
 686          * First pass over all the ports - allocate VIs and initialize some
 687          * basic parameters like mac address, port type, etc.  We also figure
 688          * out whether a port is 10G or 1G and use that information when
 689          * calculating how many interrupts to attempt to allocate.
 690          */
 691         n10g = n1g = 0;
 692         for_each_port(sc, i) {
 693                 struct port_info *pi;
 694
 695                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 696                 sc->port[i] = pi;
 697
 698                 /* These must be set before t4_port_init */
 699                 pi->adapter = sc;
 700                 pi->port_id = i;
 701
 702                 /* Allocate the vi and initialize parameters like mac addr */
 703                 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 704                 if (rc != 0) {
 705                         device_printf(dev, "unable to initialize port %d: %d\n",
 706                             i, rc);
 707                         free(pi, M_CXGBE);
 708                         sc->port[i] = NULL;
 709                         goto done;
 710                 }
 711
 712                 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX);
 713                 pi->link_cfg.requested_fc |= t4_pause_settings;
 714                 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX);
 715                 pi->link_cfg.fc |= t4_pause_settings;
 716
 717                 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
 718                 if (rc != 0) {
 719                         device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
 720                         free(pi, M_CXGBE);
 721                         sc->port[i] = NULL;
 722                         goto done;
 723                 }
 724
 725                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 726                     device_get_nameunit(dev), i);
 727                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 728                 sc->chan_map[pi->tx_chan] = i;
 729
 730                 if (is_10G_port(pi) || is_40G_port(pi)) {
 731                         n10g++;
 732                         pi->tmr_idx = t4_tmr_idx_10g;
 733                         pi->pktc_idx = t4_pktc_idx_10g;
 734                 } else {
 735                         n1g++;
 736                         pi->tmr_idx = t4_tmr_idx_1g;
 737                         pi->pktc_idx = t4_pktc_idx_1g;
 738                 }
 739
 740                 pi->xact_addr_filt = -1;
 741                 pi->linkdnrc = -1;
 742
 743                 pi->qsize_rxq = t4_qsize_rxq;
 744                 pi->qsize_txq = t4_qsize_txq;
 745
 746                 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
 747                 if (pi->dev == NULL) {
 748                         device_printf(dev,
 749                             "failed to add device for port %d.\n", i);
 750                         rc = ENXIO;
 751                         goto done;
 752                 }
 753                 device_set_softc(pi->dev, pi);
 754         }
 755
 756         /*
 757          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 758          */
 759         rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
 760         if (rc != 0)
 761                 goto done; /* error message displayed already */
 762
 763         sc->intr_type = iaq.intr_type;
 764         sc->intr_count = iaq.nirq;
 765
 766         s = &sc->sge;
 767         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 768         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 769         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
 770         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 771         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
 772 #ifdef TCP_OFFLOAD
 773         if (is_offload(sc)) {
 774                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 775                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 776                 s->neq += s->nofldtxq + s->nofldrxq;
 777                 s->niq += s->nofldrxq;
 778
 779                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 780                     M_CXGBE, M_ZERO | M_WAITOK);
 781                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 782                     M_CXGBE, M_ZERO | M_WAITOK);
 783         }
 784 #endif
 785 #ifdef DEV_NETMAP
 786         s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g;
 787         s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g;
 788         s->neq += s->nnmtxq + s->nnmrxq;
 789         s->niq += s->nnmrxq;
 790
 791         s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
 792             M_CXGBE, M_ZERO | M_WAITOK);
 793         s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
 794             M_CXGBE, M_ZERO | M_WAITOK);
 795 #endif
 796
 797         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 798             M_ZERO | M_WAITOK);
 799         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 800             M_ZERO | M_WAITOK);
 801         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 802             M_ZERO | M_WAITOK);
 803         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 804             M_ZERO | M_WAITOK);
 805         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 806             M_ZERO | M_WAITOK);
 807
 808         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 809             M_ZERO | M_WAITOK);
 810
 811         t4_init_l2t(sc, M_WAITOK);
 812
 813         /*
 814          * Second pass over the ports.  This time we know the number of rx and
 815          * tx queues that each port should get.
 816          */
 817         rqidx = tqidx = 0;
 818 #ifdef TCP_OFFLOAD
 819         ofld_rqidx = ofld_tqidx = 0;
 820 #endif
 821 #ifdef DEV_NETMAP
 822         nm_rqidx = nm_tqidx = 0;
 823 #endif
 824         for_each_port(sc, i) {
 825                 struct port_info *pi = sc->port[i];
 826
 827                 if (pi == NULL)
 828                         continue;
 829
 830                 pi->first_rxq = rqidx;
 831                 pi->first_txq = tqidx;
 832                 if (is_10G_port(pi) || is_40G_port(pi)) {
 833                         pi->flags |= iaq.intr_flags_10g;
 834                         pi->nrxq = iaq.nrxq10g;
 835                         pi->ntxq = iaq.ntxq10g;
 836                 } else {
 837                         pi->flags |= iaq.intr_flags_1g;
 838                         pi->nrxq = iaq.nrxq1g;
 839                         pi->ntxq = iaq.ntxq1g;
 840                 }
 841
 842                 if (pi->ntxq > 1)
 843                         pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
 844                 else
 845                         pi->rsrv_noflowq = 0;
 846
 847                 rqidx += pi->nrxq;
 848                 tqidx += pi->ntxq;
 849 #ifdef TCP_OFFLOAD
 850                 if (is_offload(sc)) {
 851                         pi->first_ofld_rxq = ofld_rqidx;
 852                         pi->first_ofld_txq = ofld_tqidx;
 853                         if (is_10G_port(pi) || is_40G_port(pi)) {
 854                                 pi->nofldrxq = iaq.nofldrxq10g;
 855                                 pi->nofldtxq = iaq.nofldtxq10g;
 856                         } else {
 857                                 pi->nofldrxq = iaq.nofldrxq1g;
 858                                 pi->nofldtxq = iaq.nofldtxq1g;
 859                         }
 860                         ofld_rqidx += pi->nofldrxq;
 861                         ofld_tqidx += pi->nofldtxq;
 862                 }
 863 #endif
 864 #ifdef DEV_NETMAP
 865                 pi->first_nm_rxq = nm_rqidx;
 866                 pi->first_nm_txq = nm_tqidx;
 867                 if (is_10G_port(pi) || is_40G_port(pi)) {
 868                         pi->nnmrxq = iaq.nnmrxq10g;
 869                         pi->nnmtxq = iaq.nnmtxq10g;
 870                 } else {
 871                         pi->nnmrxq = iaq.nnmrxq1g;
 872                         pi->nnmtxq = iaq.nnmtxq1g;
 873                 }
 874                 nm_rqidx += pi->nnmrxq;
 875                 nm_tqidx += pi->nnmtxq;
 876 #endif
 877         }
 878
 879         rc = setup_intr_handlers(sc);
 880         if (rc != 0) {
 881                 device_printf(dev,
 882                     "failed to setup interrupt handlers: %d\n", rc);
 883                 goto done;
 884         }
 885
 886         rc = bus_generic_attach(dev);
 887         if (rc != 0) {
 888                 device_printf(dev,
 889                     "failed to attach all child ports: %d\n", rc);
 890                 goto done;
 891         }
 892
 893         device_printf(dev,
 894             "PCIe x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
 895             sc->params.pci.width, sc->params.nports, sc->intr_count,
 896             sc->intr_type == INTR_MSIX ? "MSI-X" :
 897             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
 898             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
 899
 900         t4_set_desc(sc);
 901
 902 done:
 903         if (rc != 0 && sc->cdev) {
 904                 /* cdev was created and so cxgbetool works; recover that way. */
 905                 device_printf(dev,
 906                     "error during attach, adapter is now in recovery mode.\n");
 907                 rc = 0;
 908         }
 909
 910         if (rc != 0)
 911                 t4_detach(dev);
 912         else
 913                 t4_sysctls(sc);
 914
 915         return (rc);
 916 }
 917
 918 /*
 919  * Idempotent
 920  */
 921 static int
 922 t4_detach(device_t dev)
 923 {
 924         struct adapter *sc;
 925         struct port_info *pi;
 926         int i, rc;
 927
 928         sc = device_get_softc(dev);
 929
 930         if (sc->flags & FULL_INIT_DONE)
 931                 t4_intr_disable(sc);
 932
 933         if (sc->cdev) {
 934                 destroy_dev(sc->cdev);
 935                 sc->cdev = NULL;
 936         }
 937
 938         rc = bus_generic_detach(dev);
 939         if (rc) {
 940                 device_printf(dev,
 941                     "failed to detach child devices: %d\n", rc);
 942                 return (rc);
 943         }
 944
 945         for (i = 0; i < sc->intr_count; i++)
 946                 t4_free_irq(sc, &sc->irq[i]);
 947
 948         for (i = 0; i < MAX_NPORTS; i++) {
 949                 pi = sc->port[i];
 950                 if (pi) {
 951                         t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid);
 952                         if (pi->dev)
 953                                 device_delete_child(dev, pi->dev);
 954
 955                         mtx_destroy(&pi->pi_lock);
 956                         free(pi, M_CXGBE);
 957                 }
 958         }
 959
 960         if (sc->flags & FULL_INIT_DONE)
 961                 adapter_full_uninit(sc);
 962
 963         if (sc->flags & FW_OK)
 964                 t4_fw_bye(sc, sc->mbox);
 965
 966         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
 967                 pci_release_msi(dev);
 968
 969         if (sc->regs_res)
 970                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
 971                     sc->regs_res);
 972
 973         if (sc->udbs_res)
 974                 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
 975                     sc->udbs_res);
 976
 977         if (sc->msix_res)
 978                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
 979                     sc->msix_res);
 980
 981         if (sc->l2t)
 982                 t4_free_l2t(sc->l2t);
 983
 984 #ifdef TCP_OFFLOAD
 985         free(sc->sge.ofld_rxq, M_CXGBE);
 986         free(sc->sge.ofld_txq, M_CXGBE);
 987 #endif
 988 #ifdef DEV_NETMAP
 989         free(sc->sge.nm_rxq, M_CXGBE);
 990         free(sc->sge.nm_txq, M_CXGBE);
 991 #endif
 992         free(sc->irq, M_CXGBE);
 993         free(sc->sge.rxq, M_CXGBE);
 994         free(sc->sge.txq, M_CXGBE);
 995         free(sc->sge.ctrlq, M_CXGBE);
 996         free(sc->sge.iqmap, M_CXGBE);
 997         free(sc->sge.eqmap, M_CXGBE);
 998         free(sc->tids.ftid_tab, M_CXGBE);
 999         t4_destroy_dma_tag(sc);
1000         if (mtx_initialized(&sc->sc_lock)) {
1001                 sx_xlock(&t4_list_lock);
1002                 SLIST_REMOVE(&t4_list, sc, adapter, link);
1003                 sx_xunlock(&t4_list_lock);
1004                 mtx_destroy(&sc->sc_lock);
1005         }
1006
1007         if (mtx_initialized(&sc->tids.ftid_lock))
1008                 mtx_destroy(&sc->tids.ftid_lock);
1009         if (mtx_initialized(&sc->sfl_lock))
1010                 mtx_destroy(&sc->sfl_lock);
1011         if (mtx_initialized(&sc->ifp_lock))
1012                 mtx_destroy(&sc->ifp_lock);
1013
1014         bzero(sc, sizeof(*sc));
1015
1016         return (0);
1017 }
1018
1019 static int
1020 cxgbe_probe(device_t dev)
1021 {
1022         char buf[128];
1023         struct port_info *pi = device_get_softc(dev);
1024
1025         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
1026         device_set_desc_copy(dev, buf);
1027
1028         return (BUS_PROBE_DEFAULT);
1029 }
1030
1031 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
1032     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1033     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
1034 #define T4_CAP_ENABLE (T4_CAP)
1035
1036 static int
1037 cxgbe_attach(device_t dev)
1038 {
1039         struct port_info *pi = device_get_softc(dev);
1040         struct ifnet *ifp;
1041         char *s;
1042         int n, o;
1043
1044         /* Allocate an ifnet and set it up */
1045         ifp = if_alloc(IFT_ETHER);
1046         if (ifp == NULL) {
1047                 device_printf(dev, "Cannot allocate ifnet\n");
1048                 return (ENOMEM);
1049         }
1050         pi->ifp = ifp;
1051         ifp->if_softc = pi;
1052
1053         callout_init(&pi->tick, CALLOUT_MPSAFE);
1054
1055         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1056         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1057
1058         ifp->if_init = cxgbe_init;
1059         ifp->if_ioctl = cxgbe_ioctl;
1060         ifp->if_transmit = cxgbe_transmit;
1061         ifp->if_qflush = cxgbe_qflush;
1062
1063         ifp->if_capabilities = T4_CAP;
1064 #ifdef TCP_OFFLOAD
1065         if (is_offload(pi->adapter))
1066                 ifp->if_capabilities |= IFCAP_TOE;
1067 #endif
1068         ifp->if_capenable = T4_CAP_ENABLE;
1069         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1070             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1071
1072         ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1073         ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS;
1074         ifp->if_hw_tsomaxsegsize = 65536;
1075
1076         /* Initialize ifmedia for this port */
1077         ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1078             cxgbe_media_status);
1079         build_medialist(pi, &pi->media);
1080
1081         pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1082             EVENTHANDLER_PRI_ANY);
1083
1084         ether_ifattach(ifp, pi->hw_addr);
1085
1086         n = 128;
1087         s = malloc(n, M_CXGBE, M_WAITOK);
1088         o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq);
1089         MPASS(n > o);
1090 #ifdef TCP_OFFLOAD
1091         if (is_offload(pi->adapter)) {
1092                 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)",
1093                     pi->nofldtxq, pi->nofldrxq);
1094                 MPASS(n > o);
1095         }
1096 #endif
1097 #ifdef DEV_NETMAP
1098         o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq,
1099             pi->nnmrxq);
1100         MPASS(n > o);
1101 #endif
1102         device_printf(dev, "%s\n", s);
1103         free(s, M_CXGBE);
1104
1105 #ifdef DEV_NETMAP
1106         /* nm_media handled here to keep implementation private to this file */
1107         ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change,
1108             cxgbe_media_status);
1109         build_medialist(pi, &pi->nm_media);
1110         create_netmap_ifnet(pi);        /* logs errors it something fails */
1111 #endif
1112         cxgbe_sysctls(pi);
1113
1114         return (0);
1115 }
1116
1117 static int
1118 cxgbe_detach(device_t dev)
1119 {
1120         struct port_info *pi = device_get_softc(dev);
1121         struct adapter *sc = pi->adapter;
1122         struct ifnet *ifp = pi->ifp;
1123
1124         /* Tell if_ioctl and if_init that the port is going away */
1125         ADAPTER_LOCK(sc);
1126         SET_DOOMED(pi);
1127         wakeup(&sc->flags);
1128         while (IS_BUSY(sc))
1129                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
1130         SET_BUSY(sc);
1131 #ifdef INVARIANTS
1132         sc->last_op = "t4detach";
1133         sc->last_op_thr = curthread;
1134 #endif
1135         ADAPTER_UNLOCK(sc);
1136
1137         if (pi->flags & HAS_TRACEQ) {
1138                 sc->traceq = -1;        /* cloner should not create ifnet */
1139                 t4_tracer_port_detach(sc);
1140         }
1141
1142         if (pi->vlan_c)
1143                 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
1144
1145         PORT_LOCK(pi);
1146         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1147         callout_stop(&pi->tick);
1148         PORT_UNLOCK(pi);
1149         callout_drain(&pi->tick);
1150
1151         /* Let detach proceed even if these fail. */
1152         cxgbe_uninit_synchronized(pi);
1153         port_full_uninit(pi);
1154
1155         ifmedia_removeall(&pi->media);
1156         ether_ifdetach(pi->ifp);
1157         if_free(pi->ifp);
1158
1159 #ifdef DEV_NETMAP
1160         /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */
1161         destroy_netmap_ifnet(pi);
1162 #endif
1163
1164         ADAPTER_LOCK(sc);
1165         CLR_BUSY(sc);
1166         wakeup(&sc->flags);
1167         ADAPTER_UNLOCK(sc);
1168
1169         return (0);
1170 }
1171
1172 static void
1173 cxgbe_init(void *arg)
1174 {
1175         struct port_info *pi = arg;
1176         struct adapter *sc = pi->adapter;
1177
1178         if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
1179                 return;
1180         cxgbe_init_synchronized(pi);
1181         end_synchronized_op(sc, 0);
1182 }
1183
1184 static int
1185 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1186 {
1187         int rc = 0, mtu, flags, can_sleep;
1188         struct port_info *pi = ifp->if_softc;
1189         struct adapter *sc = pi->adapter;
1190         struct ifreq *ifr = (struct ifreq *)data;
1191         uint32_t mask;
1192
1193         switch (cmd) {
1194         case SIOCSIFMTU:
1195                 mtu = ifr->ifr_mtu;
1196                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1197                         return (EINVAL);
1198
1199                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
1200                 if (rc)
1201                         return (rc);
1202                 ifp->if_mtu = mtu;
1203                 if (pi->flags & PORT_INIT_DONE) {
1204                         t4_update_fl_bufsize(ifp);
1205                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1206                                 rc = update_mac_settings(ifp, XGMAC_MTU);
1207                 }
1208                 end_synchronized_op(sc, 0);
1209                 break;
1210
1211         case SIOCSIFFLAGS:
1212                 can_sleep = 0;
1213 redo_sifflags:
1214                 rc = begin_synchronized_op(sc, pi,
1215                     can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg");
1216                 if (rc)
1217                         return (rc);
1218
1219                 if (ifp->if_flags & IFF_UP) {
1220                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1221                                 flags = pi->if_flags;
1222                                 if ((ifp->if_flags ^ flags) &
1223                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1224                                         if (can_sleep == 1) {
1225                                                 end_synchronized_op(sc, 0);
1226                                                 can_sleep = 0;
1227                                                 goto redo_sifflags;
1228                                         }
1229                                         rc = update_mac_settings(ifp,
1230                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1231                                 }
1232                         } else {
1233                                 if (can_sleep == 0) {
1234                                         end_synchronized_op(sc, LOCK_HELD);
1235                                         can_sleep = 1;
1236                                         goto redo_sifflags;
1237                                 }
1238                                 rc = cxgbe_init_synchronized(pi);
1239                         }
1240                         pi->if_flags = ifp->if_flags;
1241                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1242                         if (can_sleep == 0) {
1243                                 end_synchronized_op(sc, LOCK_HELD);
1244                                 can_sleep = 1;
1245                                 goto redo_sifflags;
1246                         }
1247                         rc = cxgbe_uninit_synchronized(pi);
1248                 }
1249                 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD);
1250                 break;
1251
1252         case SIOCADDMULTI:
1253         case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1254                 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
1255                 if (rc)
1256                         return (rc);
1257                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1258                         rc = update_mac_settings(ifp, XGMAC_MCADDRS);
1259                 end_synchronized_op(sc, LOCK_HELD);
1260                 break;
1261
1262         case SIOCSIFCAP:
1263                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
1264                 if (rc)
1265                         return (rc);
1266
1267                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1268                 if (mask & IFCAP_TXCSUM) {
1269                         ifp->if_capenable ^= IFCAP_TXCSUM;
1270                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1271
1272                         if (IFCAP_TSO4 & ifp->if_capenable &&
1273                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1274                                 ifp->if_capenable &= ~IFCAP_TSO4;
1275                                 if_printf(ifp,
1276                                     "tso4 disabled due to -txcsum.\n");
1277                         }
1278                 }
1279                 if (mask & IFCAP_TXCSUM_IPV6) {
1280                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1281                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1282
1283                         if (IFCAP_TSO6 & ifp->if_capenable &&
1284                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1285                                 ifp->if_capenable &= ~IFCAP_TSO6;
1286                                 if_printf(ifp,
1287                                     "tso6 disabled due to -txcsum6.\n");
1288                         }
1289                 }
1290                 if (mask & IFCAP_RXCSUM)
1291                         ifp->if_capenable ^= IFCAP_RXCSUM;
1292                 if (mask & IFCAP_RXCSUM_IPV6)
1293                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1294
1295                 /*
1296                  * Note that we leave CSUM_TSO alone (it is always set).  The
1297                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1298                  * sending a TSO request our way, so it's sufficient to toggle
1299                  * IFCAP_TSOx only.
1300                  */
1301                 if (mask & IFCAP_TSO4) {
1302                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1303                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1304                                 if_printf(ifp, "enable txcsum first.\n");
1305                                 rc = EAGAIN;
1306                                 goto fail;
1307                         }
1308                         ifp->if_capenable ^= IFCAP_TSO4;
1309                 }
1310                 if (mask & IFCAP_TSO6) {
1311                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1312                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1313                                 if_printf(ifp, "enable txcsum6 first.\n");
1314                                 rc = EAGAIN;
1315                                 goto fail;
1316                         }
1317                         ifp->if_capenable ^= IFCAP_TSO6;
1318                 }
1319                 if (mask & IFCAP_LRO) {
1320 #if defined(INET) || defined(INET6)
1321                         int i;
1322                         struct sge_rxq *rxq;
1323
1324                         ifp->if_capenable ^= IFCAP_LRO;
1325                         for_each_rxq(pi, i, rxq) {
1326                                 if (ifp->if_capenable & IFCAP_LRO)
1327                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1328                                 else
1329                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1330                         }
1331 #endif
1332                 }
1333 #ifdef TCP_OFFLOAD
1334                 if (mask & IFCAP_TOE) {
1335                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1336
1337                         rc = toe_capability(pi, enable);
1338                         if (rc != 0)
1339                                 goto fail;
1340
1341                         ifp->if_capenable ^= mask;
1342                 }
1343 #endif
1344                 if (mask & IFCAP_VLAN_HWTAGGING) {
1345                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1346                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1347                                 rc = update_mac_settings(ifp, XGMAC_VLANEX);
1348                 }
1349                 if (mask & IFCAP_VLAN_MTU) {
1350                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1351
1352                         /* Need to find out how to disable auto-mtu-inflation */
1353                 }
1354                 if (mask & IFCAP_VLAN_HWTSO)
1355                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1356                 if (mask & IFCAP_VLAN_HWCSUM)
1357                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1358
1359 #ifdef VLAN_CAPABILITIES
1360                 VLAN_CAPABILITIES(ifp);
1361 #endif
1362 fail:
1363                 end_synchronized_op(sc, 0);
1364                 break;
1365
1366         case SIOCSIFMEDIA:
1367         case SIOCGIFMEDIA:
1368                 ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1369                 break;
1370
1371         default:
1372                 rc = ether_ioctl(ifp, cmd, data);
1373         }
1374
1375         return (rc);
1376 }
1377
1378 static int
1379 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1380 {
1381         struct port_info *pi = ifp->if_softc;
1382         struct adapter *sc = pi->adapter;
1383         struct sge_txq *txq = &sc->sge.txq[pi->first_txq];
1384         struct buf_ring *br;
1385         int rc;
1386
1387         M_ASSERTPKTHDR(m);
1388
1389         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1390                 m_freem(m);
1391                 return (ENETDOWN);
1392         }
1393
1394         if (m->m_flags & M_FLOWID)
1395                 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq))
1396                     + pi->rsrv_noflowq);
1397         br = txq->br;
1398
1399         if (TXQ_TRYLOCK(txq) == 0) {
1400                 struct sge_eq *eq = &txq->eq;
1401
1402                 /*
1403                  * It is possible that t4_eth_tx finishes up and releases the
1404                  * lock between the TRYLOCK above and the drbr_enqueue here.  We
1405                  * need to make sure that this mbuf doesn't just sit there in
1406                  * the drbr.
1407                  */
1408
1409                 rc = drbr_enqueue(ifp, br, m);
1410                 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 &&
1411                     !(eq->flags & EQ_DOOMED))
1412                         callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq);
1413                 return (rc);
1414         }
1415
1416         /*
1417          * txq->m is the mbuf that is held up due to a temporary shortage of
1418          * resources and it should be put on the wire first.  Then what's in
1419          * drbr and finally the mbuf that was just passed in to us.
1420          *
1421          * Return code should indicate the fate of the mbuf that was passed in
1422          * this time.
1423          */
1424
1425         TXQ_LOCK_ASSERT_OWNED(txq);
1426         if (drbr_needs_enqueue(ifp, br) || txq->m) {
1427
1428                 /* Queued for transmission. */
1429
1430                 rc = drbr_enqueue(ifp, br, m);
1431                 m = txq->m ? txq->m : drbr_dequeue(ifp, br);
1432                 (void) t4_eth_tx(ifp, txq, m);
1433                 TXQ_UNLOCK(txq);
1434                 return (rc);
1435         }
1436
1437         /* Direct transmission. */
1438         rc = t4_eth_tx(ifp, txq, m);
1439         if (rc != 0 && txq->m)
1440                 rc = 0; /* held, will be transmitted soon (hopefully) */
1441
1442         TXQ_UNLOCK(txq);
1443         return (rc);
1444 }
1445
1446 static void
1447 cxgbe_qflush(struct ifnet *ifp)
1448 {
1449         struct port_info *pi = ifp->if_softc;
1450         struct sge_txq *txq;
1451         int i;
1452         struct mbuf *m;
1453
1454         /* queues do not exist if !PORT_INIT_DONE. */
1455         if (pi->flags & PORT_INIT_DONE) {
1456                 for_each_txq(pi, i, txq) {
1457                         TXQ_LOCK(txq);
1458                         m_freem(txq->m);
1459                         txq->m = NULL;
1460                         while ((m = buf_ring_dequeue_sc(txq->br)) != NULL)
1461                                 m_freem(m);
1462                         TXQ_UNLOCK(txq);
1463                 }
1464         }
1465         if_qflush(ifp);
1466 }
1467
1468 static int
1469 cxgbe_media_change(struct ifnet *ifp)
1470 {
1471         struct port_info *pi = ifp->if_softc;
1472
1473         device_printf(pi->dev, "%s unimplemented.\n", __func__);
1474
1475         return (EOPNOTSUPP);
1476 }
1477
1478 static void
1479 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1480 {
1481         struct port_info *pi = ifp->if_softc;
1482         struct ifmedia *media = NULL;
1483         struct ifmedia_entry *cur;
1484         int speed = pi->link_cfg.speed;
1485         int data = (pi->port_type << 8) | pi->mod_type;
1486
1487         if (ifp == pi->ifp)
1488                 media = &pi->media;
1489 #ifdef DEV_NETMAP
1490         else if (ifp == pi->nm_ifp)
1491                 media = &pi->nm_media;
1492 #endif
1493         MPASS(media != NULL);
1494
1495         cur = media->ifm_cur;
1496         if (cur->ifm_data != data) {
1497                 build_medialist(pi, media);
1498                 cur = media->ifm_cur;
1499         }
1500
1501         ifmr->ifm_status = IFM_AVALID;
1502         if (!pi->link_cfg.link_ok)
1503                 return;
1504
1505         ifmr->ifm_status |= IFM_ACTIVE;
1506
1507         /* active and current will differ iff current media is autoselect. */
1508         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1509                 return;
1510
1511         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1512         if (speed == SPEED_10000)
1513                 ifmr->ifm_active |= IFM_10G_T;
1514         else if (speed == SPEED_1000)
1515                 ifmr->ifm_active |= IFM_1000_T;
1516         else if (speed == SPEED_100)
1517                 ifmr->ifm_active |= IFM_100_TX;
1518         else if (speed == SPEED_10)
1519                 ifmr->ifm_active |= IFM_10_T;
1520         else
1521                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1522                             speed));
1523 }
1524
1525 void
1526 t4_fatal_err(struct adapter *sc)
1527 {
1528         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1529         t4_intr_disable(sc);
1530         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1531             device_get_nameunit(sc->dev));
1532 }
1533
1534 static int
1535 map_bars_0_and_4(struct adapter *sc)
1536 {
1537         sc->regs_rid = PCIR_BAR(0);
1538         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1539             &sc->regs_rid, RF_ACTIVE);
1540         if (sc->regs_res == NULL) {
1541                 device_printf(sc->dev, "cannot map registers.\n");
1542                 return (ENXIO);
1543         }
1544         sc->bt = rman_get_bustag(sc->regs_res);
1545         sc->bh = rman_get_bushandle(sc->regs_res);
1546         sc->mmio_len = rman_get_size(sc->regs_res);
1547         setbit(&sc->doorbells, DOORBELL_KDB);
1548
1549         sc->msix_rid = PCIR_BAR(4);
1550         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1551             &sc->msix_rid, RF_ACTIVE);
1552         if (sc->msix_res == NULL) {
1553                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1554                 return (ENXIO);
1555         }
1556
1557         return (0);
1558 }
1559
1560 static int
1561 map_bar_2(struct adapter *sc)
1562 {
1563
1564         /*
1565          * T4: only iWARP driver uses the userspace doorbells.  There is no need
1566          * to map it if RDMA is disabled.
1567          */
1568         if (is_t4(sc) && sc->rdmacaps == 0)
1569                 return (0);
1570
1571         sc->udbs_rid = PCIR_BAR(2);
1572         sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1573             &sc->udbs_rid, RF_ACTIVE);
1574         if (sc->udbs_res == NULL) {
1575                 device_printf(sc->dev, "cannot map doorbell BAR.\n");
1576                 return (ENXIO);
1577         }
1578         sc->udbs_base = rman_get_virtual(sc->udbs_res);
1579
1580         if (is_t5(sc)) {
1581                 setbit(&sc->doorbells, DOORBELL_UDB);
1582 #if defined(__i386__) || defined(__amd64__)
1583                 if (t5_write_combine) {
1584                         int rc;
1585
1586                         /*
1587                          * Enable write combining on BAR2.  This is the
1588                          * userspace doorbell BAR and is split into 128B
1589                          * (UDBS_SEG_SIZE) doorbell regions, each associated
1590                          * with an egress queue.  The first 64B has the doorbell
1591                          * and the second 64B can be used to submit a tx work
1592                          * request with an implicit doorbell.
1593                          */
1594
1595                         rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
1596                             rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
1597                         if (rc == 0) {
1598                                 clrbit(&sc->doorbells, DOORBELL_UDB);
1599                                 setbit(&sc->doorbells, DOORBELL_WCWR);
1600                                 setbit(&sc->doorbells, DOORBELL_UDBWC);
1601                         } else {
1602                                 device_printf(sc->dev,
1603                                     "couldn't enable write combining: %d\n",
1604                                     rc);
1605                         }
1606
1607                         t4_write_reg(sc, A_SGE_STAT_CFG,
1608                             V_STATSOURCE_T5(7) | V_STATMODE(0));
1609                 }
1610 #endif
1611         }
1612
1613         return (0);
1614 }
1615
1616 static const struct memwin t4_memwin[] = {
1617         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1618         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1619         { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
1620 };
1621
1622 static const struct memwin t5_memwin[] = {
1623         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1624         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1625         { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1626 };
1627
1628 static void
1629 setup_memwin(struct adapter *sc)
1630 {
1631         const struct memwin *mw;
1632         int i, n;
1633         uint32_t bar0;
1634
1635         if (is_t4(sc)) {
1636                 /*
1637                  * Read low 32b of bar0 indirectly via the hardware backdoor
1638                  * mechanism.  Works from within PCI passthrough environments
1639                  * too, where rman_get_start() can return a different value.  We
1640                  * need to program the T4 memory window decoders with the actual
1641                  * addresses that will be coming across the PCIe link.
1642                  */
1643                 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1644                 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1645
1646                 mw = &t4_memwin[0];
1647                 n = nitems(t4_memwin);
1648         } else {
1649                 /* T5 uses the relative offset inside the PCIe BAR */
1650                 bar0 = 0;
1651
1652                 mw = &t5_memwin[0];
1653                 n = nitems(t5_memwin);
1654         }
1655
1656         for (i = 0; i < n; i++, mw++) {
1657                 t4_write_reg(sc,
1658                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
1659                     (mw->base + bar0) | V_BIR(0) |
1660                     V_WINDOW(ilog2(mw->aperture) - 10));
1661         }
1662
1663         /* flush */
1664         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1665 }
1666
1667 /*
1668  * Verify that the memory range specified by the addr/len pair is valid and lies
1669  * entirely within a single region (EDCx or MCx).
1670  */
1671 static int
1672 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
1673 {
1674         uint32_t em, addr_len, maddr, mlen;
1675
1676         /* Memory can only be accessed in naturally aligned 4 byte units */
1677         if (addr & 3 || len & 3 || len == 0)
1678                 return (EINVAL);
1679
1680         /* Enabled memories */
1681         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1682         if (em & F_EDRAM0_ENABLE) {
1683                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1684                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1685                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1686                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1687                     addr + len <= maddr + mlen)
1688                         return (0);
1689         }
1690         if (em & F_EDRAM1_ENABLE) {
1691                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1692                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1693                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1694                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1695                     addr + len <= maddr + mlen)
1696                         return (0);
1697         }
1698         if (em & F_EXT_MEM_ENABLE) {
1699                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1700                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1701                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1702                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1703                     addr + len <= maddr + mlen)
1704                         return (0);
1705         }
1706         if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
1707                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1708                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1709                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1710                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1711                     addr + len <= maddr + mlen)
1712                         return (0);
1713         }
1714
1715         return (EFAULT);
1716 }
1717
1718 static int
1719 fwmtype_to_hwmtype(int mtype)
1720 {
1721
1722         switch (mtype) {
1723         case FW_MEMTYPE_EDC0:
1724                 return (MEM_EDC0);
1725         case FW_MEMTYPE_EDC1:
1726                 return (MEM_EDC1);
1727         case FW_MEMTYPE_EXTMEM:
1728                 return (MEM_MC0);
1729         case FW_MEMTYPE_EXTMEM1:
1730                 return (MEM_MC1);
1731         default:
1732                 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
1733         }
1734 }
1735
1736 /*
1737  * Verify that the memory range specified by the memtype/offset/len pair is
1738  * valid and lies entirely within the memtype specified.  The global address of
1739  * the start of the range is returned in addr.
1740  */
1741 static int
1742 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1743     uint32_t *addr)
1744 {
1745         uint32_t em, addr_len, maddr, mlen;
1746
1747         /* Memory can only be accessed in naturally aligned 4 byte units */
1748         if (off & 3 || len & 3 || len == 0)
1749                 return (EINVAL);
1750
1751         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1752         switch (fwmtype_to_hwmtype(mtype)) {
1753         case MEM_EDC0:
1754                 if (!(em & F_EDRAM0_ENABLE))
1755                         return (EINVAL);
1756                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1757                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1758                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1759                 break;
1760         case MEM_EDC1:
1761                 if (!(em & F_EDRAM1_ENABLE))
1762                         return (EINVAL);
1763                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1764                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1765                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1766                 break;
1767         case MEM_MC:
1768                 if (!(em & F_EXT_MEM_ENABLE))
1769                         return (EINVAL);
1770                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1771                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1772                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1773                 break;
1774         case MEM_MC1:
1775                 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
1776                         return (EINVAL);
1777                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1778                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1779                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1780                 break;
1781         default:
1782                 return (EINVAL);
1783         }
1784
1785         if (mlen > 0 && off < mlen && off + len <= mlen) {
1786                 *addr = maddr + off;    /* global address */
1787                 return (0);
1788         }
1789
1790         return (EFAULT);
1791 }
1792
1793 static void
1794 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1795 {
1796         const struct memwin *mw;
1797
1798         if (is_t4(sc)) {
1799                 KASSERT(win >= 0 && win < nitems(t4_memwin),
1800                     ("%s: incorrect memwin# (%d)", __func__, win));
1801                 mw = &t4_memwin[win];
1802         } else {
1803                 KASSERT(win >= 0 && win < nitems(t5_memwin),
1804                     ("%s: incorrect memwin# (%d)", __func__, win));
1805                 mw = &t5_memwin[win];
1806         }
1807
1808         if (base != NULL)
1809                 *base = mw->base;
1810         if (aperture != NULL)
1811                 *aperture = mw->aperture;
1812 }
1813
1814 /*
1815  * Positions the memory window such that it can be used to access the specified
1816  * address in the chip's address space.  The return value is the offset of addr
1817  * from the start of the window.
1818  */
1819 static uint32_t
1820 position_memwin(struct adapter *sc, int n, uint32_t addr)
1821 {
1822         uint32_t start, pf;
1823         uint32_t reg;
1824
1825         KASSERT(n >= 0 && n <= 3,
1826             ("%s: invalid window %d.", __func__, n));
1827         KASSERT((addr & 3) == 0,
1828             ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
1829
1830         if (is_t4(sc)) {
1831                 pf = 0;
1832                 start = addr & ~0xf;    /* start must be 16B aligned */
1833         } else {
1834                 pf = V_PFNUM(sc->pf);
1835                 start = addr & ~0x7f;   /* start must be 128B aligned */
1836         }
1837         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1838
1839         t4_write_reg(sc, reg, start | pf);
1840         t4_read_reg(sc, reg);
1841
1842         return (addr - start);
1843 }
1844
1845 static int
1846 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1847     struct intrs_and_queues *iaq)
1848 {
1849         int rc, itype, navail, nrxq10g, nrxq1g, n;
1850         int nofldrxq10g = 0, nofldrxq1g = 0;
1851         int nnmrxq10g = 0, nnmrxq1g = 0;
1852
1853         bzero(iaq, sizeof(*iaq));
1854
1855         iaq->ntxq10g = t4_ntxq10g;
1856         iaq->ntxq1g = t4_ntxq1g;
1857         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1858         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1859         iaq->rsrv_noflowq = t4_rsrv_noflowq;
1860 #ifdef TCP_OFFLOAD
1861         if (is_offload(sc)) {
1862                 iaq->nofldtxq10g = t4_nofldtxq10g;
1863                 iaq->nofldtxq1g = t4_nofldtxq1g;
1864                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1865                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1866         }
1867 #endif
1868 #ifdef DEV_NETMAP
1869         iaq->nnmtxq10g = t4_nnmtxq10g;
1870         iaq->nnmtxq1g = t4_nnmtxq1g;
1871         iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g;
1872         iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g;
1873 #endif
1874
1875         for (itype = INTR_MSIX; itype; itype >>= 1) {
1876
1877                 if ((itype & t4_intr_types) == 0)
1878                         continue;       /* not allowed */
1879
1880                 if (itype == INTR_MSIX)
1881                         navail = pci_msix_count(sc->dev);
1882                 else if (itype == INTR_MSI)
1883                         navail = pci_msi_count(sc->dev);
1884                 else
1885                         navail = 1;
1886 restart:
1887                 if (navail == 0)
1888                         continue;
1889
1890                 iaq->intr_type = itype;
1891                 iaq->intr_flags_10g = 0;
1892                 iaq->intr_flags_1g = 0;
1893
1894                 /*
1895                  * Best option: an interrupt vector for errors, one for the
1896                  * firmware event queue, and one for every rxq (NIC, TOE, and
1897                  * netmap).
1898                  */
1899                 iaq->nirq = T4_EXTRA_INTR;
1900                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g);
1901                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g);
1902                 if (iaq->nirq <= navail &&
1903                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
1904                         iaq->intr_flags_10g = INTR_ALL;
1905                         iaq->intr_flags_1g = INTR_ALL;
1906                         goto allocate;
1907                 }
1908
1909                 /*
1910                  * Second best option: a vector for errors, one for the firmware
1911                  * event queue, and vectors for either all the NIC rx queues or
1912                  * all the TOE rx queues.  The queues that don't get vectors
1913                  * will forward their interrupts to those that do.
1914                  *
1915                  * Note: netmap rx queues cannot be created early and so they
1916                  * can't be setup to receive forwarded interrupts for others.
1917                  */
1918                 iaq->nirq = T4_EXTRA_INTR;
1919                 if (nrxq10g >= nofldrxq10g) {
1920                         iaq->intr_flags_10g = INTR_RXQ;
1921                         iaq->nirq += n10g * nrxq10g;
1922 #ifdef DEV_NETMAP
1923                         iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g);
1924 #endif
1925                 } else {
1926                         iaq->intr_flags_10g = INTR_OFLD_RXQ;
1927                         iaq->nirq += n10g * nofldrxq10g;
1928 #ifdef DEV_NETMAP
1929                         iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g);
1930 #endif
1931                 }
1932                 if (nrxq1g >= nofldrxq1g) {
1933                         iaq->intr_flags_1g = INTR_RXQ;
1934                         iaq->nirq += n1g * nrxq1g;
1935 #ifdef DEV_NETMAP
1936                         iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g);
1937 #endif
1938                 } else {
1939                         iaq->intr_flags_1g = INTR_OFLD_RXQ;
1940                         iaq->nirq += n1g * nofldrxq1g;
1941 #ifdef DEV_NETMAP
1942                         iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g);
1943 #endif
1944                 }
1945                 if (iaq->nirq <= navail &&
1946                     (itype != INTR_MSI || powerof2(iaq->nirq)))
1947                         goto allocate;
1948
1949                 /*
1950                  * Next best option: an interrupt vector for errors, one for the
1951                  * firmware event queue, and at least one per port.  At this
1952                  * point we know we'll have to downsize nrxq and/or nofldrxq
1953                  * and/or nnmrxq to fit what's available to us.
1954                  */
1955                 iaq->nirq = T4_EXTRA_INTR;
1956                 iaq->nirq += n10g + n1g;
1957                 if (iaq->nirq <= navail) {
1958                         int leftover = navail - iaq->nirq;
1959
1960                         if (n10g > 0) {
1961                                 int target = max(nrxq10g, nofldrxq10g);
1962
1963                                 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
1964                                     INTR_RXQ : INTR_OFLD_RXQ;
1965
1966                                 n = 1;
1967                                 while (n < target && leftover >= n10g) {
1968                                         leftover -= n10g;
1969                                         iaq->nirq += n10g;
1970                                         n++;
1971                                 }
1972                                 iaq->nrxq10g = min(n, nrxq10g);
1973 #ifdef TCP_OFFLOAD
1974                                 iaq->nofldrxq10g = min(n, nofldrxq10g);
1975 #endif
1976 #ifdef DEV_NETMAP
1977                                 iaq->nnmrxq10g = min(n, nnmrxq10g);
1978 #endif
1979                         }
1980
1981                         if (n1g > 0) {
1982                                 int target = max(nrxq1g, nofldrxq1g);
1983
1984                                 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
1985                                     INTR_RXQ : INTR_OFLD_RXQ;
1986
1987                                 n = 1;
1988                                 while (n < target && leftover >= n1g) {
1989                                         leftover -= n1g;
1990                                         iaq->nirq += n1g;
1991                                         n++;
1992                                 }
1993                                 iaq->nrxq1g = min(n, nrxq1g);
1994 #ifdef TCP_OFFLOAD
1995                                 iaq->nofldrxq1g = min(n, nofldrxq1g);
1996 #endif
1997 #ifdef DEV_NETMAP
1998                                 iaq->nnmrxq1g = min(n, nnmrxq1g);
1999 #endif
2000                         }
2001
2002                         if (itype != INTR_MSI || powerof2(iaq->nirq))
2003                                 goto allocate;
2004                 }
2005
2006                 /*
2007                  * Least desirable option: one interrupt vector for everything.
2008                  */
2009                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
2010                 iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
2011 #ifdef TCP_OFFLOAD
2012                 if (is_offload(sc))
2013                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2014 #endif
2015 #ifdef DEV_NETMAP
2016                 iaq->nnmrxq10g = iaq->nnmrxq1g = 1;
2017 #endif
2018
2019 allocate:
2020                 navail = iaq->nirq;
2021                 rc = 0;
2022                 if (itype == INTR_MSIX)
2023                         rc = pci_alloc_msix(sc->dev, &navail);
2024                 else if (itype == INTR_MSI)
2025                         rc = pci_alloc_msi(sc->dev, &navail);
2026
2027                 if (rc == 0) {
2028                         if (navail == iaq->nirq)
2029                                 return (0);
2030
2031                         /*
2032                          * Didn't get the number requested.  Use whatever number
2033                          * the kernel is willing to allocate (it's in navail).
2034                          */
2035                         device_printf(sc->dev, "fewer vectors than requested, "
2036                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
2037                             itype, iaq->nirq, navail);
2038                         pci_release_msi(sc->dev);
2039                         goto restart;
2040                 }
2041
2042                 device_printf(sc->dev,
2043                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
2044                     itype, rc, iaq->nirq, navail);
2045         }
2046
2047         device_printf(sc->dev,
2048             "failed to find a usable interrupt type.  "
2049             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
2050             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
2051
2052         return (ENXIO);
2053 }
2054
2055 #define FW_VERSION(chip) ( \
2056     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
2057     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
2058     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
2059     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
2060 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
2061
2062 struct fw_info {
2063         uint8_t chip;
2064         char *kld_name;
2065         char *fw_mod_name;
2066         struct fw_hdr fw_hdr;   /* XXX: waste of space, need a sparse struct */
2067 } fw_info[] = {
2068         {
2069                 .chip = CHELSIO_T4,
2070                 .kld_name = "t4fw_cfg",
2071                 .fw_mod_name = "t4fw",
2072                 .fw_hdr = {
2073                         .chip = FW_HDR_CHIP_T4,
2074                         .fw_ver = htobe32_const(FW_VERSION(T4)),
2075                         .intfver_nic = FW_INTFVER(T4, NIC),
2076                         .intfver_vnic = FW_INTFVER(T4, VNIC),
2077                         .intfver_ofld = FW_INTFVER(T4, OFLD),
2078                         .intfver_ri = FW_INTFVER(T4, RI),
2079                         .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
2080                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
2081                         .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
2082                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
2083                 },
2084         }, {
2085                 .chip = CHELSIO_T5,
2086                 .kld_name = "t5fw_cfg",
2087                 .fw_mod_name = "t5fw",
2088                 .fw_hdr = {
2089                         .chip = FW_HDR_CHIP_T5,
2090                         .fw_ver = htobe32_const(FW_VERSION(T5)),
2091                         .intfver_nic = FW_INTFVER(T5, NIC),
2092                         .intfver_vnic = FW_INTFVER(T5, VNIC),
2093                         .intfver_ofld = FW_INTFVER(T5, OFLD),
2094                         .intfver_ri = FW_INTFVER(T5, RI),
2095                         .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
2096                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
2097                         .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
2098                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
2099                 },
2100         }
2101 };
2102
2103 static struct fw_info *
2104 find_fw_info(int chip)
2105 {
2106         int i;
2107
2108         for (i = 0; i < nitems(fw_info); i++) {
2109                 if (fw_info[i].chip == chip)
2110                         return (&fw_info[i]);
2111         }
2112         return (NULL);
2113 }
2114
2115 /*
2116  * Is the given firmware API compatible with the one the driver was compiled
2117  * with?
2118  */
2119 static int
2120 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
2121 {
2122
2123         /* short circuit if it's the exact same firmware version */
2124         if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
2125                 return (1);
2126
2127         /*
2128          * XXX: Is this too conservative?  Perhaps I should limit this to the
2129          * features that are supported in the driver.
2130          */
2131 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
2132         if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
2133             SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
2134             SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
2135                 return (1);
2136 #undef SAME_INTF
2137
2138         return (0);
2139 }
2140
2141 /*
2142  * The firmware in the KLD is usable, but should it be installed?  This routine
2143  * explains itself in detail if it indicates the KLD firmware should be
2144  * installed.
2145  */
2146 static int
2147 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
2148 {
2149         const char *reason;
2150
2151         if (!card_fw_usable) {
2152                 reason = "incompatible or unusable";
2153                 goto install;
2154         }
2155
2156         if (k > c) {
2157                 reason = "older than the version bundled with this driver";
2158                 goto install;
2159         }
2160
2161         if (t4_fw_install == 2 && k != c) {
2162                 reason = "different than the version bundled with this driver";
2163                 goto install;
2164         }
2165
2166         return (0);
2167
2168 install:
2169         if (t4_fw_install == 0) {
2170                 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2171                     "but the driver is prohibited from installing a different "
2172                     "firmware on the card.\n",
2173                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2174                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2175
2176                 return (0);
2177         }
2178
2179         device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2180             "installing firmware %u.%u.%u.%u on card.\n",
2181             G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2182             G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2183             G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2184             G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2185
2186         return (1);
2187 }
2188 /*
2189  * Establish contact with the firmware and determine if we are the master driver
2190  * or not, and whether we are responsible for chip initialization.
2191  */
2192 static int
2193 prep_firmware(struct adapter *sc)
2194 {
2195         const struct firmware *fw = NULL, *default_cfg;
2196         int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2197         enum dev_state state;
2198         struct fw_info *fw_info;
2199         struct fw_hdr *card_fw;         /* fw on the card */
2200         const struct fw_hdr *kld_fw;    /* fw in the KLD */
2201         const struct fw_hdr *drv_fw;    /* fw header the driver was compiled
2202                                            against */
2203
2204         /* Contact firmware. */
2205         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2206         if (rc < 0 || state == DEV_STATE_ERR) {
2207                 rc = -rc;
2208                 device_printf(sc->dev,
2209                     "failed to connect to the firmware: %d, %d.\n", rc, state);
2210                 return (rc);
2211         }
2212         pf = rc;
2213         if (pf == sc->mbox)
2214                 sc->flags |= MASTER_PF;
2215         else if (state == DEV_STATE_UNINIT) {
2216                 /*
2217                  * We didn't get to be the master so we definitely won't be
2218                  * configuring the chip.  It's a bug if someone else hasn't
2219                  * configured it already.
2220                  */
2221                 device_printf(sc->dev, "couldn't be master(%d), "
2222                     "device not already initialized either(%d).\n", rc, state);
2223                 return (EDOOFUS);
2224         }
2225
2226         /* This is the firmware whose headers the driver was compiled against */
2227         fw_info = find_fw_info(chip_id(sc));
2228         if (fw_info == NULL) {
2229                 device_printf(sc->dev,
2230                     "unable to look up firmware information for chip %d.\n",
2231                     chip_id(sc));
2232                 return (EINVAL);
2233         }
2234         drv_fw = &fw_info->fw_hdr;
2235
2236         /*
2237          * The firmware KLD contains many modules.  The KLD name is also the
2238          * name of the module that contains the default config file.
2239          */
2240         default_cfg = firmware_get(fw_info->kld_name);
2241
2242         /* Read the header of the firmware on the card */
2243         card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2244         rc = -t4_read_flash(sc, FLASH_FW_START,
2245             sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2246         if (rc == 0)
2247                 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2248         else {
2249                 device_printf(sc->dev,
2250                     "Unable to read card's firmware header: %d\n", rc);
2251                 card_fw_usable = 0;
2252         }
2253
2254         /* This is the firmware in the KLD */
2255         fw = firmware_get(fw_info->fw_mod_name);
2256         if (fw != NULL) {
2257                 kld_fw = (const void *)fw->data;
2258                 kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2259         } else {
2260                 kld_fw = NULL;
2261                 kld_fw_usable = 0;
2262         }
2263
2264         if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2265             (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2266                 /*
2267                  * Common case: the firmware on the card is an exact match and
2268                  * the KLD is an exact match too, or the KLD is
2269                  * absent/incompatible.  Note that t4_fw_install = 2 is ignored
2270                  * here -- use cxgbetool loadfw if you want to reinstall the
2271                  * same firmware as the one on the card.
2272                  */
2273         } else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2274             should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2275             be32toh(card_fw->fw_ver))) {
2276
2277                 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
2278                 if (rc != 0) {
2279                         device_printf(sc->dev,
2280                             "failed to install firmware: %d\n", rc);
2281                         goto done;
2282                 }
2283
2284                 /* Installed successfully, update the cached header too. */
2285                 memcpy(card_fw, kld_fw, sizeof(*card_fw));
2286                 card_fw_usable = 1;
2287                 need_fw_reset = 0;      /* already reset as part of load_fw */
2288         }
2289
2290         if (!card_fw_usable) {
2291                 uint32_t d, c, k;
2292
2293                 d = ntohl(drv_fw->fw_ver);
2294                 c = ntohl(card_fw->fw_ver);
2295                 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
2296
2297                 device_printf(sc->dev, "Cannot find a usable firmware: "
2298                     "fw_install %d, chip state %d, "
2299                     "driver compiled with %d.%d.%d.%d, "
2300                     "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
2301                     t4_fw_install, state,
2302                     G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
2303                     G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
2304                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2305                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
2306                     G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2307                     G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2308                 rc = EINVAL;
2309                 goto done;
2310         }
2311
2312         /* We're using whatever's on the card and it's known to be good. */
2313         sc->params.fw_vers = ntohl(card_fw->fw_ver);
2314         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
2315             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2316             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2317             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2318             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2319         t4_get_tp_version(sc, &sc->params.tp_vers);
2320
2321         /* Reset device */
2322         if (need_fw_reset &&
2323             (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
2324                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
2325                 if (rc != ETIMEDOUT && rc != EIO)
2326                         t4_fw_bye(sc, sc->mbox);
2327                 goto done;
2328         }
2329         sc->flags |= FW_OK;
2330
2331         rc = get_params__pre_init(sc);
2332         if (rc != 0)
2333                 goto done; /* error message displayed already */
2334
2335         /* Partition adapter resources as specified in the config file. */
2336         if (state == DEV_STATE_UNINIT) {
2337
2338                 KASSERT(sc->flags & MASTER_PF,
2339                     ("%s: trying to change chip settings when not master.",
2340                     __func__));
2341
2342                 rc = partition_resources(sc, default_cfg, fw_info->kld_name);
2343                 if (rc != 0)
2344                         goto done;      /* error message displayed already */
2345
2346                 t4_tweak_chip_settings(sc);
2347
2348                 /* get basic stuff going */
2349                 rc = -t4_fw_initialize(sc, sc->mbox);
2350                 if (rc != 0) {
2351                         device_printf(sc->dev, "fw init failed: %d.\n", rc);
2352                         goto done;
2353                 }
2354         } else {
2355                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
2356                 sc->cfcsum = 0;
2357         }
2358
2359 done:
2360         free(card_fw, M_CXGBE);
2361         if (fw != NULL)
2362                 firmware_put(fw, FIRMWARE_UNLOAD);
2363         if (default_cfg != NULL)
2364                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
2365
2366         return (rc);
2367 }
2368
2369 #define FW_PARAM_DEV(param) \
2370         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2371          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2372 #define FW_PARAM_PFVF(param) \
2373         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2374          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2375
2376 /*
2377  * Partition chip resources for use between various PFs, VFs, etc.
2378  */
2379 static int
2380 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
2381     const char *name_prefix)
2382 {
2383         const struct firmware *cfg = NULL;
2384         int rc = 0;
2385         struct fw_caps_config_cmd caps;
2386         uint32_t mtype, moff, finicsum, cfcsum;
2387
2388         /*
2389          * Figure out what configuration file to use.  Pick the default config
2390          * file for the card if the user hasn't specified one explicitly.
2391          */
2392         snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
2393         if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
2394                 /* Card specific overrides go here. */
2395                 if (pci_get_device(sc->dev) == 0x440a)
2396                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
2397                 if (is_fpga(sc))
2398                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
2399         }
2400
2401         /*
2402          * We need to load another module if the profile is anything except
2403          * "default" or "flash".
2404          */
2405         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
2406             strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2407                 char s[32];
2408
2409                 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
2410                 cfg = firmware_get(s);
2411                 if (cfg == NULL) {
2412                         if (default_cfg != NULL) {
2413                                 device_printf(sc->dev,
2414                                     "unable to load module \"%s\" for "
2415                                     "configuration profile \"%s\", will use "
2416                                     "the default config file instead.\n",
2417                                     s, sc->cfg_file);
2418                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2419                                     "%s", DEFAULT_CF);
2420                         } else {
2421                                 device_printf(sc->dev,
2422                                     "unable to load module \"%s\" for "
2423                                     "configuration profile \"%s\", will use "
2424                                     "the config file on the card's flash "
2425                                     "instead.\n", s, sc->cfg_file);
2426                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2427                                     "%s", FLASH_CF);
2428                         }
2429                 }
2430         }
2431
2432         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
2433             default_cfg == NULL) {
2434                 device_printf(sc->dev,
2435                     "default config file not available, will use the config "
2436                     "file on the card's flash instead.\n");
2437                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
2438         }
2439
2440         if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2441                 u_int cflen, i, n;
2442                 const uint32_t *cfdata;
2443                 uint32_t param, val, addr, off, mw_base, mw_aperture;
2444
2445                 KASSERT(cfg != NULL || default_cfg != NULL,
2446                     ("%s: no config to upload", __func__));
2447
2448                 /*
2449                  * Ask the firmware where it wants us to upload the config file.
2450                  */
2451                 param = FW_PARAM_DEV(CF);
2452                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2453                 if (rc != 0) {
2454                         /* No support for config file?  Shouldn't happen. */
2455                         device_printf(sc->dev,
2456                             "failed to query config file location: %d.\n", rc);
2457                         goto done;
2458                 }
2459                 mtype = G_FW_PARAMS_PARAM_Y(val);
2460                 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
2461
2462                 /*
2463                  * XXX: sheer laziness.  We deliberately added 4 bytes of
2464                  * useless stuffing/comments at the end of the config file so
2465                  * it's ok to simply throw away the last remaining bytes when
2466                  * the config file is not an exact multiple of 4.  This also
2467                  * helps with the validate_mt_off_len check.
2468                  */
2469                 if (cfg != NULL) {
2470                         cflen = cfg->datasize & ~3;
2471                         cfdata = cfg->data;
2472                 } else {
2473                         cflen = default_cfg->datasize & ~3;
2474                         cfdata = default_cfg->data;
2475                 }
2476
2477                 if (cflen > FLASH_CFG_MAX_SIZE) {
2478                         device_printf(sc->dev,
2479                             "config file too long (%d, max allowed is %d).  "
2480                             "Will try to use the config on the card, if any.\n",
2481                             cflen, FLASH_CFG_MAX_SIZE);
2482                         goto use_config_on_flash;
2483                 }
2484
2485                 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
2486                 if (rc != 0) {
2487                         device_printf(sc->dev,
2488                             "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
2489                             "Will try to use the config on the card, if any.\n",
2490                             __func__, mtype, moff, cflen, rc);
2491                         goto use_config_on_flash;
2492                 }
2493
2494                 memwin_info(sc, 2, &mw_base, &mw_aperture);
2495                 while (cflen) {
2496                         off = position_memwin(sc, 2, addr);
2497                         n = min(cflen, mw_aperture - off);
2498                         for (i = 0; i < n; i += 4)
2499                                 t4_write_reg(sc, mw_base + off + i, *cfdata++);
2500                         cflen -= n;
2501                         addr += n;
2502                 }
2503         } else {
2504 use_config_on_flash:
2505                 mtype = FW_MEMTYPE_FLASH;
2506                 moff = t4_flash_cfg_addr(sc);
2507         }
2508
2509         bzero(&caps, sizeof(caps));
2510         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2511             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2512         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
2513             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
2514             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
2515         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2516         if (rc != 0) {
2517                 device_printf(sc->dev,
2518                     "failed to pre-process config file: %d "
2519                     "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
2520                 goto done;
2521         }
2522
2523         finicsum = be32toh(caps.finicsum);
2524         cfcsum = be32toh(caps.cfcsum);
2525         if (finicsum != cfcsum) {
2526                 device_printf(sc->dev,
2527                     "WARNING: config file checksum mismatch: %08x %08x\n",
2528                     finicsum, cfcsum);
2529         }
2530         sc->cfcsum = cfcsum;
2531
2532 #define LIMIT_CAPS(x) do { \
2533         caps.x &= htobe16(t4_##x##_allowed); \
2534 } while (0)
2535
2536         /*
2537          * Let the firmware know what features will (not) be used so it can tune
2538          * things accordingly.
2539          */
2540         LIMIT_CAPS(linkcaps);
2541         LIMIT_CAPS(niccaps);
2542         LIMIT_CAPS(toecaps);
2543         LIMIT_CAPS(rdmacaps);
2544         LIMIT_CAPS(iscsicaps);
2545         LIMIT_CAPS(fcoecaps);
2546 #undef LIMIT_CAPS
2547
2548         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2549             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
2550         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2551         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
2552         if (rc != 0) {
2553                 device_printf(sc->dev,
2554                     "failed to process config file: %d.\n", rc);
2555         }
2556 done:
2557         if (cfg != NULL)
2558                 firmware_put(cfg, FIRMWARE_UNLOAD);
2559         return (rc);
2560 }
2561
2562 /*
2563  * Retrieve parameters that are needed (or nice to have) very early.
2564  */
2565 static int
2566 get_params__pre_init(struct adapter *sc)
2567 {
2568         int rc;
2569         uint32_t param[2], val[2];
2570         struct fw_devlog_cmd cmd;
2571         struct devlog_params *dlog = &sc->params.devlog;
2572
2573         param[0] = FW_PARAM_DEV(PORTVEC);
2574         param[1] = FW_PARAM_DEV(CCLK);
2575         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2576         if (rc != 0) {
2577                 device_printf(sc->dev,
2578                     "failed to query parameters (pre_init): %d.\n", rc);
2579                 return (rc);
2580         }
2581
2582         sc->params.portvec = val[0];
2583         sc->params.nports = bitcount32(val[0]);
2584         sc->params.vpd.cclk = val[1];
2585
2586         /* Read device log parameters. */
2587         bzero(&cmd, sizeof(cmd));
2588         cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
2589             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2590         cmd.retval_len16 = htobe32(FW_LEN16(cmd));
2591         rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
2592         if (rc != 0) {
2593                 device_printf(sc->dev,
2594                     "failed to get devlog parameters: %d.\n", rc);
2595                 bzero(dlog, sizeof (*dlog));
2596                 rc = 0; /* devlog isn't critical for device operation */
2597         } else {
2598                 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
2599                 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
2600                 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
2601                 dlog->size = be32toh(cmd.memsize_devlog);
2602         }
2603
2604         return (rc);
2605 }
2606
2607 /*
2608  * Retrieve various parameters that are of interest to the driver.  The device
2609  * has been initialized by the firmware at this point.
2610  */
2611 static int
2612 get_params__post_init(struct adapter *sc)
2613 {
2614         int rc;
2615         uint32_t param[7], val[7];
2616         struct fw_caps_config_cmd caps;
2617
2618         param[0] = FW_PARAM_PFVF(IQFLINT_START);
2619         param[1] = FW_PARAM_PFVF(EQ_START);
2620         param[2] = FW_PARAM_PFVF(FILTER_START);
2621         param[3] = FW_PARAM_PFVF(FILTER_END);
2622         param[4] = FW_PARAM_PFVF(L2T_START);
2623         param[5] = FW_PARAM_PFVF(L2T_END);
2624         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2625         if (rc != 0) {
2626                 device_printf(sc->dev,
2627                     "failed to query parameters (post_init): %d.\n", rc);
2628                 return (rc);
2629         }
2630
2631         sc->sge.iq_start = val[0];
2632         sc->sge.eq_start = val[1];
2633         sc->tids.ftid_base = val[2];
2634         sc->tids.nftids = val[3] - val[2] + 1;
2635         sc->params.ftid_min = val[2];
2636         sc->params.ftid_max = val[3];
2637         sc->vres.l2t.start = val[4];
2638         sc->vres.l2t.size = val[5] - val[4] + 1;
2639         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
2640             ("%s: L2 table size (%u) larger than expected (%u)",
2641             __func__, sc->vres.l2t.size, L2T_SIZE));
2642
2643         /* get capabilites */
2644         bzero(&caps, sizeof(caps));
2645         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2646             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2647         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2648         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2649         if (rc != 0) {
2650                 device_printf(sc->dev,
2651                     "failed to get card capabilities: %d.\n", rc);
2652                 return (rc);
2653         }
2654
2655 #define READ_CAPS(x) do { \
2656         sc->x = htobe16(caps.x); \
2657 } while (0)
2658         READ_CAPS(linkcaps);
2659         READ_CAPS(niccaps);
2660         READ_CAPS(toecaps);
2661         READ_CAPS(rdmacaps);
2662         READ_CAPS(iscsicaps);
2663         READ_CAPS(fcoecaps);
2664
2665         if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
2666                 param[0] = FW_PARAM_PFVF(ETHOFLD_START);
2667                 param[1] = FW_PARAM_PFVF(ETHOFLD_END);
2668                 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2669                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
2670                 if (rc != 0) {
2671                         device_printf(sc->dev,
2672                             "failed to query NIC parameters: %d.\n", rc);
2673                         return (rc);
2674                 }
2675                 sc->tids.etid_base = val[0];
2676                 sc->params.etid_min = val[0];
2677                 sc->tids.netids = val[1] - val[0] + 1;
2678                 sc->params.netids = sc->tids.netids;
2679                 sc->params.eo_wr_cred = val[2];
2680                 sc->params.ethoffload = 1;
2681         }
2682
2683         if (sc->toecaps) {
2684                 /* query offload-related parameters */
2685                 param[0] = FW_PARAM_DEV(NTID);
2686                 param[1] = FW_PARAM_PFVF(SERVER_START);
2687                 param[2] = FW_PARAM_PFVF(SERVER_END);
2688                 param[3] = FW_PARAM_PFVF(TDDP_START);
2689                 param[4] = FW_PARAM_PFVF(TDDP_END);
2690                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2691                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2692                 if (rc != 0) {
2693                         device_printf(sc->dev,
2694                             "failed to query TOE parameters: %d.\n", rc);
2695                         return (rc);
2696                 }
2697                 sc->tids.ntids = val[0];
2698                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
2699                 sc->tids.stid_base = val[1];
2700                 sc->tids.nstids = val[2] - val[1] + 1;
2701                 sc->vres.ddp.start = val[3];
2702                 sc->vres.ddp.size = val[4] - val[3] + 1;
2703                 sc->params.ofldq_wr_cred = val[5];
2704                 sc->params.offload = 1;
2705         }
2706         if (sc->rdmacaps) {
2707                 param[0] = FW_PARAM_PFVF(STAG_START);
2708                 param[1] = FW_PARAM_PFVF(STAG_END);
2709                 param[2] = FW_PARAM_PFVF(RQ_START);
2710                 param[3] = FW_PARAM_PFVF(RQ_END);
2711                 param[4] = FW_PARAM_PFVF(PBL_START);
2712                 param[5] = FW_PARAM_PFVF(PBL_END);
2713                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2714                 if (rc != 0) {
2715                         device_printf(sc->dev,
2716                             "failed to query RDMA parameters(1): %d.\n", rc);
2717                         return (rc);
2718                 }
2719                 sc->vres.stag.start = val[0];
2720                 sc->vres.stag.size = val[1] - val[0] + 1;
2721                 sc->vres.rq.start = val[2];
2722                 sc->vres.rq.size = val[3] - val[2] + 1;
2723                 sc->vres.pbl.start = val[4];
2724                 sc->vres.pbl.size = val[5] - val[4] + 1;
2725
2726                 param[0] = FW_PARAM_PFVF(SQRQ_START);
2727                 param[1] = FW_PARAM_PFVF(SQRQ_END);
2728                 param[2] = FW_PARAM_PFVF(CQ_START);
2729                 param[3] = FW_PARAM_PFVF(CQ_END);
2730                 param[4] = FW_PARAM_PFVF(OCQ_START);
2731                 param[5] = FW_PARAM_PFVF(OCQ_END);
2732                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2733                 if (rc != 0) {
2734                         device_printf(sc->dev,
2735                             "failed to query RDMA parameters(2): %d.\n", rc);
2736                         return (rc);
2737                 }
2738                 sc->vres.qp.start = val[0];
2739                 sc->vres.qp.size = val[1] - val[0] + 1;
2740                 sc->vres.cq.start = val[2];
2741                 sc->vres.cq.size = val[3] - val[2] + 1;
2742                 sc->vres.ocq.start = val[4];
2743                 sc->vres.ocq.size = val[5] - val[4] + 1;
2744         }
2745         if (sc->iscsicaps) {
2746                 param[0] = FW_PARAM_PFVF(ISCSI_START);
2747                 param[1] = FW_PARAM_PFVF(ISCSI_END);
2748                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2749                 if (rc != 0) {
2750                         device_printf(sc->dev,
2751                             "failed to query iSCSI parameters: %d.\n", rc);
2752                         return (rc);
2753                 }
2754                 sc->vres.iscsi.start = val[0];
2755                 sc->vres.iscsi.size = val[1] - val[0] + 1;
2756         }
2757
2758         /*
2759          * We've got the params we wanted to query via the firmware.  Now grab
2760          * some others directly from the chip.
2761          */
2762         rc = t4_read_chip_settings(sc);
2763
2764         return (rc);
2765 }
2766
2767 static int
2768 set_params__post_init(struct adapter *sc)
2769 {
2770         uint32_t param, val;
2771
2772         /* ask for encapsulated CPLs */
2773         param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
2774         val = 1;
2775         (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2776
2777         return (0);
2778 }
2779
2780 #undef FW_PARAM_PFVF
2781 #undef FW_PARAM_DEV
2782
2783 static void
2784 t4_set_desc(struct adapter *sc)
2785 {
2786         char buf[128];
2787         struct adapter_params *p = &sc->params;
2788
2789         snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
2790             "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
2791             chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
2792
2793         device_set_desc_copy(sc->dev, buf);
2794 }
2795
2796 static void
2797 build_medialist(struct port_info *pi, struct ifmedia *media)
2798 {
2799         int data, m;
2800
2801         PORT_LOCK(pi);
2802
2803         ifmedia_removeall(media);
2804
2805         m = IFM_ETHER | IFM_FDX;
2806         data = (pi->port_type << 8) | pi->mod_type;
2807
2808         switch(pi->port_type) {
2809         case FW_PORT_TYPE_BT_XFI:
2810                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2811                 break;
2812
2813         case FW_PORT_TYPE_BT_XAUI:
2814                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2815                 /* fall through */
2816
2817         case FW_PORT_TYPE_BT_SGMII:
2818                 ifmedia_add(media, m | IFM_1000_T, data, NULL);
2819                 ifmedia_add(media, m | IFM_100_TX, data, NULL);
2820                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL);
2821                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2822                 break;
2823
2824         case FW_PORT_TYPE_CX4:
2825                 ifmedia_add(media, m | IFM_10G_CX4, data, NULL);
2826                 ifmedia_set(media, m | IFM_10G_CX4);
2827                 break;
2828
2829         case FW_PORT_TYPE_QSFP_10G:
2830         case FW_PORT_TYPE_SFP:
2831         case FW_PORT_TYPE_FIBER_XFI:
2832         case FW_PORT_TYPE_FIBER_XAUI:
2833                 switch (pi->mod_type) {
2834
2835                 case FW_PORT_MOD_TYPE_LR:
2836                         ifmedia_add(media, m | IFM_10G_LR, data, NULL);
2837                         ifmedia_set(media, m | IFM_10G_LR);
2838                         break;
2839
2840                 case FW_PORT_MOD_TYPE_SR:
2841                         ifmedia_add(media, m | IFM_10G_SR, data, NULL);
2842                         ifmedia_set(media, m | IFM_10G_SR);
2843                         break;
2844
2845                 case FW_PORT_MOD_TYPE_LRM:
2846                         ifmedia_add(media, m | IFM_10G_LRM, data, NULL);
2847                         ifmedia_set(media, m | IFM_10G_LRM);
2848                         break;
2849
2850                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2851                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2852                         ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL);
2853                         ifmedia_set(media, m | IFM_10G_TWINAX);
2854                         break;
2855
2856                 case FW_PORT_MOD_TYPE_NONE:
2857                         m &= ~IFM_FDX;
2858                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2859                         ifmedia_set(media, m | IFM_NONE);
2860                         break;
2861
2862                 case FW_PORT_MOD_TYPE_NA:
2863                 case FW_PORT_MOD_TYPE_ER:
2864                 default:
2865                         device_printf(pi->dev,
2866                             "unknown port_type (%d), mod_type (%d)\n",
2867                             pi->port_type, pi->mod_type);
2868                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2869                         ifmedia_set(media, m | IFM_UNKNOWN);
2870                         break;
2871                 }
2872                 break;
2873
2874         case FW_PORT_TYPE_QSFP:
2875                 switch (pi->mod_type) {
2876
2877                 case FW_PORT_MOD_TYPE_LR:
2878                         ifmedia_add(media, m | IFM_40G_LR4, data, NULL);
2879                         ifmedia_set(media, m | IFM_40G_LR4);
2880                         break;
2881
2882                 case FW_PORT_MOD_TYPE_SR:
2883                         ifmedia_add(media, m | IFM_40G_SR4, data, NULL);
2884                         ifmedia_set(media, m | IFM_40G_SR4);
2885                         break;
2886
2887                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2888                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2889                         ifmedia_add(media, m | IFM_40G_CR4, data, NULL);
2890                         ifmedia_set(media, m | IFM_40G_CR4);
2891                         break;
2892
2893                 case FW_PORT_MOD_TYPE_NONE:
2894                         m &= ~IFM_FDX;
2895                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2896                         ifmedia_set(media, m | IFM_NONE);
2897                         break;
2898
2899                 default:
2900                         device_printf(pi->dev,
2901                             "unknown port_type (%d), mod_type (%d)\n",
2902                             pi->port_type, pi->mod_type);
2903                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2904                         ifmedia_set(media, m | IFM_UNKNOWN);
2905                         break;
2906                 }
2907                 break;
2908
2909         default:
2910                 device_printf(pi->dev,
2911                     "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
2912                     pi->mod_type);
2913                 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2914                 ifmedia_set(media, m | IFM_UNKNOWN);
2915                 break;
2916         }
2917
2918         PORT_UNLOCK(pi);
2919 }
2920
2921 #define FW_MAC_EXACT_CHUNK      7
2922
2923 /*
2924  * Program the port's XGMAC based on parameters in ifnet.  The caller also
2925  * indicates which parameters should be programmed (the rest are left alone).
2926  */
2927 int
2928 update_mac_settings(struct ifnet *ifp, int flags)
2929 {
2930         int rc = 0;
2931         struct port_info *pi = ifp->if_softc;
2932         struct adapter *sc = pi->adapter;
2933         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
2934         uint16_t viid = 0xffff;
2935         int16_t *xact_addr_filt = NULL;
2936
2937         ASSERT_SYNCHRONIZED_OP(sc);
2938         KASSERT(flags, ("%s: not told what to update.", __func__));
2939
2940         if (ifp == pi->ifp) {
2941                 viid = pi->viid;
2942                 xact_addr_filt = &pi->xact_addr_filt;
2943         }
2944 #ifdef DEV_NETMAP
2945         else if (ifp == pi->nm_ifp) {
2946                 viid = pi->nm_viid;
2947                 xact_addr_filt = &pi->nm_xact_addr_filt;
2948         }
2949 #endif
2950         if (flags & XGMAC_MTU)
2951                 mtu = ifp->if_mtu;
2952
2953         if (flags & XGMAC_PROMISC)
2954                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
2955
2956         if (flags & XGMAC_ALLMULTI)
2957                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
2958
2959         if (flags & XGMAC_VLANEX)
2960                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
2961
2962         if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
2963                 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti,
2964                     1, vlanex, false);
2965                 if (rc) {
2966                         if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
2967                             rc);
2968                         return (rc);
2969                 }
2970         }
2971
2972         if (flags & XGMAC_UCADDR) {
2973                 uint8_t ucaddr[ETHER_ADDR_LEN];
2974
2975                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
2976                 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr,
2977                     true, true);
2978                 if (rc < 0) {
2979                         rc = -rc;
2980                         if_printf(ifp, "change_mac failed: %d\n", rc);
2981                         return (rc);
2982                 } else {
2983                         *xact_addr_filt = rc;
2984                         rc = 0;
2985                 }
2986         }
2987
2988         if (flags & XGMAC_MCADDRS) {
2989                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
2990                 int del = 1;
2991                 uint64_t hash = 0;
2992                 struct ifmultiaddr *ifma;
2993                 int i = 0, j;
2994
2995                 if_maddr_rlock(ifp);
2996                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2997                         if (ifma->ifma_addr->sa_family != AF_LINK)
2998                                 continue;
2999                         mcaddr[i++] =
3000                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
3001
3002                         if (i == FW_MAC_EXACT_CHUNK) {
3003                                 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del,
3004                                     i, mcaddr, NULL, &hash, 0);
3005                                 if (rc < 0) {
3006                                         rc = -rc;
3007                                         for (j = 0; j < i; j++) {
3008                                                 if_printf(ifp,
3009                                                     "failed to add mc address"
3010                                                     " %02x:%02x:%02x:"
3011                                                     "%02x:%02x:%02x rc=%d\n",
3012                                                     mcaddr[j][0], mcaddr[j][1],
3013                                                     mcaddr[j][2], mcaddr[j][3],
3014                                                     mcaddr[j][4], mcaddr[j][5],
3015                                                     rc);
3016                                         }
3017                                         goto mcfail;
3018                                 }
3019                                 del = 0;
3020                                 i = 0;
3021                         }
3022                 }
3023                 if (i > 0) {
3024                         rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i,
3025                             mcaddr, NULL, &hash, 0);
3026                         if (rc < 0) {
3027                                 rc = -rc;
3028                                 for (j = 0; j < i; j++) {
3029                                         if_printf(ifp,
3030                                             "failed to add mc address"
3031                                             " %02x:%02x:%02x:"
3032                                             "%02x:%02x:%02x rc=%d\n",
3033                                             mcaddr[j][0], mcaddr[j][1],
3034                                             mcaddr[j][2], mcaddr[j][3],
3035                                             mcaddr[j][4], mcaddr[j][5],
3036                                             rc);
3037                                 }
3038                                 goto mcfail;
3039                         }
3040                 }
3041
3042                 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0);
3043                 if (rc != 0)
3044                         if_printf(ifp, "failed to set mc address hash: %d", rc);
3045 mcfail:
3046                 if_maddr_runlock(ifp);
3047         }
3048
3049         return (rc);
3050 }
3051
3052 int
3053 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
3054     char *wmesg)
3055 {
3056         int rc, pri;
3057
3058 #ifdef WITNESS
3059         /* the caller thinks it's ok to sleep, but is it really? */
3060         if (flags & SLEEP_OK)
3061                 pause("t4slptst", 1);
3062 #endif
3063
3064         if (INTR_OK)
3065                 pri = PCATCH;
3066         else
3067                 pri = 0;
3068
3069         ADAPTER_LOCK(sc);
3070         for (;;) {
3071
3072                 if (pi && IS_DOOMED(pi)) {
3073                         rc = ENXIO;
3074                         goto done;
3075                 }
3076
3077                 if (!IS_BUSY(sc)) {
3078                         rc = 0;
3079                         break;
3080                 }
3081
3082                 if (!(flags & SLEEP_OK)) {
3083                         rc = EBUSY;
3084                         goto done;
3085                 }
3086
3087                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
3088                         rc = EINTR;
3089                         goto done;
3090                 }
3091         }
3092
3093         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
3094         SET_BUSY(sc);
3095 #ifdef INVARIANTS
3096         sc->last_op = wmesg;
3097         sc->last_op_thr = curthread;
3098 #endif
3099
3100 done:
3101         if (!(flags & HOLD_LOCK) || rc)
3102                 ADAPTER_UNLOCK(sc);
3103
3104         return (rc);
3105 }
3106
3107 void
3108 end_synchronized_op(struct adapter *sc, int flags)
3109 {
3110
3111         if (flags & LOCK_HELD)
3112                 ADAPTER_LOCK_ASSERT_OWNED(sc);
3113         else
3114                 ADAPTER_LOCK(sc);
3115
3116         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
3117         CLR_BUSY(sc);
3118         wakeup(&sc->flags);
3119         ADAPTER_UNLOCK(sc);
3120 }
3121
3122 static int
3123 cxgbe_init_synchronized(struct port_info *pi)
3124 {
3125         struct adapter *sc = pi->adapter;
3126         struct ifnet *ifp = pi->ifp;
3127         int rc = 0;
3128
3129         ASSERT_SYNCHRONIZED_OP(sc);
3130
3131         if (isset(&sc->open_device_map, pi->port_id)) {
3132                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
3133                     ("mismatch between open_device_map and if_drv_flags"));
3134                 return (0);     /* already running */
3135         }
3136
3137         if (!(sc->flags & FULL_INIT_DONE) &&
3138             ((rc = adapter_full_init(sc)) != 0))
3139                 return (rc);    /* error message displayed already */
3140
3141         if (!(pi->flags & PORT_INIT_DONE) &&
3142             ((rc = port_full_init(pi)) != 0))
3143                 return (rc); /* error message displayed already */
3144
3145         rc = update_mac_settings(ifp, XGMAC_ALL);
3146         if (rc)
3147                 goto done;      /* error message displayed already */
3148
3149         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
3150         if (rc != 0) {
3151                 if_printf(ifp, "enable_vi failed: %d\n", rc);
3152                 goto done;
3153         }
3154
3155         /*
3156          * The first iq of the first port to come up is used for tracing.
3157          */
3158         if (sc->traceq < 0) {
3159                 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
3160                 t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
3161                     A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
3162                     V_QUEUENUMBER(sc->traceq));
3163                 pi->flags |= HAS_TRACEQ;
3164         }
3165
3166         /* all ok */
3167         setbit(&sc->open_device_map, pi->port_id);
3168         PORT_LOCK(pi);
3169         ifp->if_drv_flags |= IFF_DRV_RUNNING;
3170         PORT_UNLOCK(pi);
3171
3172         callout_reset(&pi->tick, hz, cxgbe_tick, pi);
3173 done:
3174         if (rc != 0)
3175                 cxgbe_uninit_synchronized(pi);
3176
3177         return (rc);
3178 }
3179
3180 /*
3181  * Idempotent.
3182  */
3183 static int
3184 cxgbe_uninit_synchronized(struct port_info *pi)
3185 {
3186         struct adapter *sc = pi->adapter;
3187         struct ifnet *ifp = pi->ifp;
3188         int rc;
3189
3190         ASSERT_SYNCHRONIZED_OP(sc);
3191
3192         /*
3193          * Disable the VI so that all its data in either direction is discarded
3194          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
3195          * tick) intact as the TP can deliver negative advice or data that it's
3196          * holding in its RAM (for an offloaded connection) even after the VI is
3197          * disabled.
3198          */
3199         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
3200         if (rc) {
3201                 if_printf(ifp, "disable_vi failed: %d\n", rc);
3202                 return (rc);
3203         }
3204
3205         clrbit(&sc->open_device_map, pi->port_id);
3206         PORT_LOCK(pi);
3207         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3208         PORT_UNLOCK(pi);
3209
3210         pi->link_cfg.link_ok = 0;
3211         pi->link_cfg.speed = 0;
3212         pi->linkdnrc = -1;
3213         t4_os_link_changed(sc, pi->port_id, 0, -1);
3214
3215         return (0);
3216 }
3217
3218 /*
3219  * It is ok for this function to fail midway and return right away.  t4_detach
3220  * will walk the entire sc->irq list and clean up whatever is valid.
3221  */
3222 static int
3223 setup_intr_handlers(struct adapter *sc)
3224 {
3225         int rc, rid, p, q;
3226         char s[8];
3227         struct irq *irq;
3228         struct port_info *pi;
3229         struct sge_rxq *rxq;
3230 #ifdef TCP_OFFLOAD
3231         struct sge_ofld_rxq *ofld_rxq;
3232 #endif
3233 #ifdef DEV_NETMAP
3234         struct sge_nm_rxq *nm_rxq;
3235 #endif
3236
3237         /*
3238          * Setup interrupts.
3239          */
3240         irq = &sc->irq[0];
3241         rid = sc->intr_type == INTR_INTX ? 0 : 1;
3242         if (sc->intr_count == 1)
3243                 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
3244
3245         /* Multiple interrupts. */
3246         KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
3247             ("%s: too few intr.", __func__));
3248
3249         /* The first one is always error intr */
3250         rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
3251         if (rc != 0)
3252                 return (rc);
3253         irq++;
3254         rid++;
3255
3256         /* The second one is always the firmware event queue */
3257         rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
3258         if (rc != 0)
3259                 return (rc);
3260         irq++;
3261         rid++;
3262
3263         for_each_port(sc, p) {
3264                 pi = sc->port[p];
3265
3266                 if (pi->flags & INTR_RXQ) {
3267                         for_each_rxq(pi, q, rxq) {
3268                                 snprintf(s, sizeof(s), "%d.%d", p, q);
3269                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
3270                                     s);
3271                                 if (rc != 0)
3272                                         return (rc);
3273                                 irq++;
3274                                 rid++;
3275                         }
3276                 }
3277 #ifdef TCP_OFFLOAD
3278                 if (pi->flags & INTR_OFLD_RXQ) {
3279                         for_each_ofld_rxq(pi, q, ofld_rxq) {
3280                                 snprintf(s, sizeof(s), "%d,%d", p, q);
3281                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr,
3282                                     ofld_rxq, s);
3283                                 if (rc != 0)
3284                                         return (rc);
3285                                 irq++;
3286                                 rid++;
3287                         }
3288                 }
3289 #endif
3290 #ifdef DEV_NETMAP
3291                 if (pi->flags & INTR_NM_RXQ) {
3292                         for_each_nm_rxq(pi, q, nm_rxq) {
3293                                 snprintf(s, sizeof(s), "%d-%d", p, q);
3294                                 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr,
3295                                     nm_rxq, s);
3296                                 if (rc != 0)
3297                                         return (rc);
3298                                 irq++;
3299                                 rid++;
3300                         }
3301                 }
3302 #endif
3303         }
3304         MPASS(irq == &sc->irq[sc->intr_count]);
3305
3306         return (0);
3307 }
3308
3309 int
3310 adapter_full_init(struct adapter *sc)
3311 {
3312         int rc, i;
3313
3314         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3315         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
3316             ("%s: FULL_INIT_DONE already", __func__));
3317
3318         /*
3319          * queues that belong to the adapter (not any particular port).
3320          */
3321         rc = t4_setup_adapter_queues(sc);
3322         if (rc != 0)
3323                 goto done;
3324
3325         for (i = 0; i < nitems(sc->tq); i++) {
3326                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
3327                     taskqueue_thread_enqueue, &sc->tq[i]);
3328                 if (sc->tq[i] == NULL) {
3329                         device_printf(sc->dev,
3330                             "failed to allocate task queue %d\n", i);
3331                         rc = ENOMEM;
3332                         goto done;
3333                 }
3334                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
3335                     device_get_nameunit(sc->dev), i);
3336         }
3337
3338         t4_intr_enable(sc);
3339         sc->flags |= FULL_INIT_DONE;
3340 done:
3341         if (rc != 0)
3342                 adapter_full_uninit(sc);
3343
3344         return (rc);
3345 }
3346
3347 int
3348 adapter_full_uninit(struct adapter *sc)
3349 {
3350         int i;
3351
3352         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3353
3354         t4_teardown_adapter_queues(sc);
3355
3356         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
3357                 taskqueue_free(sc->tq[i]);
3358                 sc->tq[i] = NULL;
3359         }
3360
3361         sc->flags &= ~FULL_INIT_DONE;
3362
3363         return (0);
3364 }
3365
3366 int
3367 port_full_init(struct port_info *pi)
3368 {
3369         struct adapter *sc = pi->adapter;
3370         struct ifnet *ifp = pi->ifp;
3371         uint16_t *rss;
3372         struct sge_rxq *rxq;
3373         int rc, i, j;
3374
3375         ASSERT_SYNCHRONIZED_OP(sc);
3376         KASSERT((pi->flags & PORT_INIT_DONE) == 0,
3377             ("%s: PORT_INIT_DONE already", __func__));
3378
3379         sysctl_ctx_init(&pi->ctx);
3380         pi->flags |= PORT_SYSCTL_CTX;
3381
3382         /*
3383          * Allocate tx/rx/fl queues for this port.
3384          */
3385         rc = t4_setup_port_queues(pi);
3386         if (rc != 0)
3387                 goto done;      /* error message displayed already */
3388
3389         /*
3390          * Setup RSS for this port.  Save a copy of the RSS table for later use.
3391          */
3392         rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
3393         for (i = 0; i < pi->rss_size;) {
3394                 for_each_rxq(pi, j, rxq) {
3395                         rss[i++] = rxq->iq.abs_id;
3396                         if (i == pi->rss_size)
3397                                 break;
3398                 }
3399         }
3400
3401         rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
3402             pi->rss_size);
3403         if (rc != 0) {
3404                 if_printf(ifp, "rss_config failed: %d\n", rc);
3405                 goto done;
3406         }
3407
3408         pi->rss = rss;
3409         pi->flags |= PORT_INIT_DONE;
3410 done:
3411         if (rc != 0)
3412                 port_full_uninit(pi);
3413
3414         return (rc);
3415 }
3416
3417 /*
3418  * Idempotent.
3419  */
3420 int
3421 port_full_uninit(struct port_info *pi)
3422 {
3423         struct adapter *sc = pi->adapter;
3424         int i;
3425         struct sge_rxq *rxq;
3426         struct sge_txq *txq;
3427 #ifdef TCP_OFFLOAD
3428         struct sge_ofld_rxq *ofld_rxq;
3429         struct sge_wrq *ofld_txq;
3430 #endif
3431
3432         if (pi->flags & PORT_INIT_DONE) {
3433
3434                 /* Need to quiesce queues.  XXX: ctrl queues? */
3435
3436                 for_each_txq(pi, i, txq) {
3437                         quiesce_eq(sc, &txq->eq);
3438                 }
3439
3440 #ifdef TCP_OFFLOAD
3441                 for_each_ofld_txq(pi, i, ofld_txq) {
3442                         quiesce_eq(sc, &ofld_txq->eq);
3443                 }
3444 #endif
3445
3446                 for_each_rxq(pi, i, rxq) {
3447                         quiesce_iq(sc, &rxq->iq);
3448                         quiesce_fl(sc, &rxq->fl);
3449                 }
3450
3451 #ifdef TCP_OFFLOAD
3452                 for_each_ofld_rxq(pi, i, ofld_rxq) {
3453                         quiesce_iq(sc, &ofld_rxq->iq);
3454                         quiesce_fl(sc, &ofld_rxq->fl);
3455                 }
3456 #endif
3457                 free(pi->rss, M_CXGBE);
3458         }
3459
3460         t4_teardown_port_queues(pi);
3461         pi->flags &= ~PORT_INIT_DONE;
3462
3463         return (0);
3464 }
3465
3466 static void
3467 quiesce_eq(struct adapter *sc, struct sge_eq *eq)
3468 {
3469         EQ_LOCK(eq);
3470         eq->flags |= EQ_DOOMED;
3471
3472         /*
3473          * Wait for the response to a credit flush if one's
3474          * pending.
3475          */
3476         while (eq->flags & EQ_CRFLUSHED)
3477                 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0);
3478         EQ_UNLOCK(eq);
3479
3480         callout_drain(&eq->tx_callout); /* XXX: iffy */
3481         pause("callout", 10);           /* Still iffy */
3482
3483         taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task);
3484 }
3485
3486 static void
3487 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
3488 {
3489         (void) sc;      /* unused */
3490
3491         /* Synchronize with the interrupt handler */
3492         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
3493                 pause("iqfree", 1);
3494 }
3495
3496 static void
3497 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
3498 {
3499         mtx_lock(&sc->sfl_lock);
3500         FL_LOCK(fl);
3501         fl->flags |= FL_DOOMED;
3502         FL_UNLOCK(fl);
3503         mtx_unlock(&sc->sfl_lock);
3504
3505         callout_drain(&sc->sfl_callout);
3506         KASSERT((fl->flags & FL_STARVING) == 0,
3507             ("%s: still starving", __func__));
3508 }
3509
3510 static int
3511 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
3512     driver_intr_t *handler, void *arg, char *name)
3513 {
3514         int rc;
3515
3516         irq->rid = rid;
3517         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
3518             RF_SHAREABLE | RF_ACTIVE);
3519         if (irq->res == NULL) {
3520                 device_printf(sc->dev,
3521                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
3522                 return (ENOMEM);
3523         }
3524
3525         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
3526             NULL, handler, arg, &irq->tag);
3527         if (rc != 0) {
3528                 device_printf(sc->dev,
3529                     "failed to setup interrupt for rid %d, name %s: %d\n",
3530                     rid, name, rc);
3531         } else if (name)
3532                 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
3533
3534         return (rc);
3535 }
3536
3537 static int
3538 t4_free_irq(struct adapter *sc, struct irq *irq)
3539 {
3540         if (irq->tag)
3541                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
3542         if (irq->res)
3543                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
3544
3545         bzero(irq, sizeof(*irq));
3546
3547         return (0);
3548 }
3549
3550 static void
3551 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
3552     unsigned int end)
3553 {
3554         uint32_t *p = (uint32_t *)(buf + start);
3555
3556         for ( ; start <= end; start += sizeof(uint32_t))
3557                 *p++ = t4_read_reg(sc, start);
3558 }
3559
3560 static void
3561 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
3562 {
3563         int i, n;
3564         const unsigned int *reg_ranges;
3565         static const unsigned int t4_reg_ranges[] = {
3566                 0x1008, 0x1108,
3567                 0x1180, 0x11b4,
3568                 0x11fc, 0x123c,
3569                 0x1300, 0x173c,
3570                 0x1800, 0x18fc,
3571                 0x3000, 0x30d8,
3572                 0x30e0, 0x5924,
3573                 0x5960, 0x59d4,
3574                 0x5a00, 0x5af8,
3575                 0x6000, 0x6098,
3576                 0x6100, 0x6150,
3577                 0x6200, 0x6208,
3578                 0x6240, 0x6248,
3579                 0x6280, 0x6338,
3580                 0x6370, 0x638c,
3581                 0x6400, 0x643c,
3582                 0x6500, 0x6524,
3583                 0x6a00, 0x6a38,
3584                 0x6a60, 0x6a78,
3585                 0x6b00, 0x6b84,
3586                 0x6bf0, 0x6c84,
3587                 0x6cf0, 0x6d84,
3588                 0x6df0, 0x6e84,
3589                 0x6ef0, 0x6f84,
3590                 0x6ff0, 0x7084,
3591                 0x70f0, 0x7184,
3592                 0x71f0, 0x7284,
3593                 0x72f0, 0x7384,
3594                 0x73f0, 0x7450,
3595                 0x7500, 0x7530,
3596                 0x7600, 0x761c,
3597                 0x7680, 0x76cc,
3598                 0x7700, 0x7798,
3599                 0x77c0, 0x77fc,
3600                 0x7900, 0x79fc,
3601                 0x7b00, 0x7c38,
3602                 0x7d00, 0x7efc,
3603                 0x8dc0, 0x8e1c,
3604                 0x8e30, 0x8e78,
3605                 0x8ea0, 0x8f6c,
3606                 0x8fc0, 0x9074,
3607                 0x90fc, 0x90fc,
3608                 0x9400, 0x9458,
3609                 0x9600, 0x96bc,
3610                 0x9800, 0x9808,
3611                 0x9820, 0x983c,
3612                 0x9850, 0x9864,
3613                 0x9c00, 0x9c6c,
3614                 0x9c80, 0x9cec,
3615                 0x9d00, 0x9d6c,
3616                 0x9d80, 0x9dec,
3617                 0x9e00, 0x9e6c,
3618                 0x9e80, 0x9eec,
3619                 0x9f00, 0x9f6c,
3620                 0x9f80, 0x9fec,
3621                 0xd004, 0xd03c,
3622                 0xdfc0, 0xdfe0,
3623                 0xe000, 0xea7c,
3624                 0xf000, 0x11110,
3625                 0x11118, 0x11190,
3626                 0x19040, 0x1906c,
3627                 0x19078, 0x19080,
3628                 0x1908c, 0x19124,
3629                 0x19150, 0x191b0,
3630                 0x191d0, 0x191e8,
3631                 0x19238, 0x1924c,
3632                 0x193f8, 0x19474,
3633                 0x19490, 0x194f8,
3634                 0x19800, 0x19f30,
3635                 0x1a000, 0x1a06c,
3636                 0x1a0b0, 0x1a120,
3637                 0x1a128, 0x1a138,
3638                 0x1a190, 0x1a1c4,
3639                 0x1a1fc, 0x1a1fc,
3640                 0x1e040, 0x1e04c,
3641                 0x1e284, 0x1e28c,
3642                 0x1e2c0, 0x1e2c0,
3643                 0x1e2e0, 0x1e2e0,
3644                 0x1e300, 0x1e384,
3645                 0x1e3c0, 0x1e3c8,
3646                 0x1e440, 0x1e44c,
3647                 0x1e684, 0x1e68c,
3648                 0x1e6c0, 0x1e6c0,
3649                 0x1e6e0, 0x1e6e0,
3650                 0x1e700, 0x1e784,
3651                 0x1e7c0, 0x1e7c8,
3652                 0x1e840, 0x1e84c,
3653                 0x1ea84, 0x1ea8c,
3654                 0x1eac0, 0x1eac0,
3655                 0x1eae0, 0x1eae0,
3656                 0x1eb00, 0x1eb84,
3657                 0x1ebc0, 0x1ebc8,
3658                 0x1ec40, 0x1ec4c,
3659                 0x1ee84, 0x1ee8c,
3660                 0x1eec0, 0x1eec0,
3661                 0x1eee0, 0x1eee0,
3662                 0x1ef00, 0x1ef84,
3663                 0x1efc0, 0x1efc8,
3664                 0x1f040, 0x1f04c,
3665                 0x1f284, 0x1f28c,
3666                 0x1f2c0, 0x1f2c0,
3667                 0x1f2e0, 0x1f2e0,
3668                 0x1f300, 0x1f384,
3669                 0x1f3c0, 0x1f3c8,
3670                 0x1f440, 0x1f44c,
3671                 0x1f684, 0x1f68c,
3672                 0x1f6c0, 0x1f6c0,
3673                 0x1f6e0, 0x1f6e0,
3674                 0x1f700, 0x1f784,
3675                 0x1f7c0, 0x1f7c8,
3676                 0x1f840, 0x1f84c,
3677                 0x1fa84, 0x1fa8c,
3678                 0x1fac0, 0x1fac0,
3679                 0x1fae0, 0x1fae0,
3680                 0x1fb00, 0x1fb84,
3681                 0x1fbc0, 0x1fbc8,
3682                 0x1fc40, 0x1fc4c,
3683                 0x1fe84, 0x1fe8c,
3684                 0x1fec0, 0x1fec0,
3685                 0x1fee0, 0x1fee0,
3686                 0x1ff00, 0x1ff84,
3687                 0x1ffc0, 0x1ffc8,
3688                 0x20000, 0x2002c,
3689                 0x20100, 0x2013c,
3690                 0x20190, 0x201c8,
3691                 0x20200, 0x20318,
3692                 0x20400, 0x20528,
3693                 0x20540, 0x20614,
3694                 0x21000, 0x21040,
3695                 0x2104c, 0x21060,
3696                 0x210c0, 0x210ec,
3697                 0x21200, 0x21268,
3698                 0x21270, 0x21284,
3699                 0x212fc, 0x21388,
3700                 0x21400, 0x21404,
3701                 0x21500, 0x21518,
3702                 0x2152c, 0x2153c,
3703                 0x21550, 0x21554,
3704                 0x21600, 0x21600,
3705                 0x21608, 0x21628,
3706                 0x21630, 0x2163c,
3707                 0x21700, 0x2171c,
3708                 0x21780, 0x2178c,
3709                 0x21800, 0x21c38,
3710                 0x21c80, 0x21d7c,
3711                 0x21e00, 0x21e04,
3712                 0x22000, 0x2202c,
3713                 0x22100, 0x2213c,
3714                 0x22190, 0x221c8,
3715                 0x22200, 0x22318,
3716                 0x22400, 0x22528,
3717                 0x22540, 0x22614,
3718                 0x23000, 0x23040,
3719                 0x2304c, 0x23060,
3720                 0x230c0, 0x230ec,
3721                 0x23200, 0x23268,
3722                 0x23270, 0x23284,
3723                 0x232fc, 0x23388,
3724                 0x23400, 0x23404,
3725                 0x23500, 0x23518,
3726                 0x2352c, 0x2353c,
3727                 0x23550, 0x23554,
3728                 0x23600, 0x23600,
3729                 0x23608, 0x23628,
3730                 0x23630, 0x2363c,
3731                 0x23700, 0x2371c,
3732                 0x23780, 0x2378c,
3733                 0x23800, 0x23c38,
3734                 0x23c80, 0x23d7c,
3735                 0x23e00, 0x23e04,
3736                 0x24000, 0x2402c,
3737                 0x24100, 0x2413c,
3738                 0x24190, 0x241c8,
3739                 0x24200, 0x24318,
3740                 0x24400, 0x24528,
3741                 0x24540, 0x24614,
3742                 0x25000, 0x25040,
3743                 0x2504c, 0x25060,
3744                 0x250c0, 0x250ec,
3745                 0x25200, 0x25268,
3746                 0x25270, 0x25284,
3747                 0x252fc, 0x25388,
3748                 0x25400, 0x25404,
3749                 0x25500, 0x25518,
3750                 0x2552c, 0x2553c,
3751                 0x25550, 0x25554,
3752                 0x25600, 0x25600,
3753                 0x25608, 0x25628,
3754                 0x25630, 0x2563c,
3755                 0x25700, 0x2571c,
3756                 0x25780, 0x2578c,
3757                 0x25800, 0x25c38,
3758                 0x25c80, 0x25d7c,
3759                 0x25e00, 0x25e04,
3760                 0x26000, 0x2602c,
3761                 0x26100, 0x2613c,
3762                 0x26190, 0x261c8,
3763                 0x26200, 0x26318,
3764                 0x26400, 0x26528,
3765                 0x26540, 0x26614,
3766                 0x27000, 0x27040,
3767                 0x2704c, 0x27060,
3768                 0x270c0, 0x270ec,
3769                 0x27200, 0x27268,
3770                 0x27270, 0x27284,
3771                 0x272fc, 0x27388,
3772                 0x27400, 0x27404,
3773                 0x27500, 0x27518,
3774                 0x2752c, 0x2753c,
3775                 0x27550, 0x27554,
3776                 0x27600, 0x27600,
3777                 0x27608, 0x27628,
3778                 0x27630, 0x2763c,
3779                 0x27700, 0x2771c,
3780                 0x27780, 0x2778c,
3781                 0x27800, 0x27c38,
3782                 0x27c80, 0x27d7c,
3783                 0x27e00, 0x27e04
3784         };
3785         static const unsigned int t5_reg_ranges[] = {
3786                 0x1008, 0x1148,
3787                 0x1180, 0x11b4,
3788                 0x11fc, 0x123c,
3789                 0x1280, 0x173c,
3790                 0x1800, 0x18fc,
3791                 0x3000, 0x3028,
3792                 0x3060, 0x30d8,
3793                 0x30e0, 0x30fc,
3794                 0x3140, 0x357c,
3795                 0x35a8, 0x35cc,
3796                 0x35ec, 0x35ec,
3797                 0x3600, 0x5624,
3798                 0x56cc, 0x575c,
3799                 0x580c, 0x5814,
3800                 0x5890, 0x58bc,
3801                 0x5940, 0x59dc,
3802                 0x59fc, 0x5a18,
3803                 0x5a60, 0x5a9c,
3804                 0x5b94, 0x5bfc,
3805                 0x6000, 0x6040,
3806                 0x6058, 0x614c,
3807                 0x7700, 0x7798,
3808                 0x77c0, 0x78fc,
3809                 0x7b00, 0x7c54,
3810                 0x7d00, 0x7efc,
3811                 0x8dc0, 0x8de0,
3812                 0x8df8, 0x8e84,
3813                 0x8ea0, 0x8f84,
3814                 0x8fc0, 0x90f8,
3815                 0x9400, 0x9470,
3816                 0x9600, 0x96f4,
3817                 0x9800, 0x9808,
3818                 0x9820, 0x983c,
3819                 0x9850, 0x9864,
3820                 0x9c00, 0x9c6c,
3821                 0x9c80, 0x9cec,
3822                 0x9d00, 0x9d6c,
3823                 0x9d80, 0x9dec,
3824                 0x9e00, 0x9e6c,
3825                 0x9e80, 0x9eec,
3826                 0x9f00, 0x9f6c,
3827                 0x9f80, 0xa020,
3828                 0xd004, 0xd03c,
3829                 0xdfc0, 0xdfe0,
3830                 0xe000, 0x11088,
3831                 0x1109c, 0x11110,
3832                 0x11118, 0x1117c,
3833                 0x11190, 0x11204,
3834                 0x19040, 0x1906c,
3835                 0x19078, 0x19080,
3836                 0x1908c, 0x19124,
3837                 0x19150, 0x191b0,
3838                 0x191d0, 0x191e8,
3839                 0x19238, 0x19290,
3840                 0x193f8, 0x19474,
3841                 0x19490, 0x194cc,
3842                 0x194f0, 0x194f8,
3843                 0x19c00, 0x19c60,
3844                 0x19c94, 0x19e10,
3845                 0x19e50, 0x19f34,
3846                 0x19f40, 0x19f50,
3847                 0x19f90, 0x19fe4,
3848                 0x1a000, 0x1a06c,
3849                 0x1a0b0, 0x1a120,
3850                 0x1a128, 0x1a138,
3851                 0x1a190, 0x1a1c4,
3852                 0x1a1fc, 0x1a1fc,
3853                 0x1e008, 0x1e00c,
3854                 0x1e040, 0x1e04c,
3855                 0x1e284, 0x1e290,
3856                 0x1e2c0, 0x1e2c0,
3857                 0x1e2e0, 0x1e2e0,
3858                 0x1e300, 0x1e384,
3859                 0x1e3c0, 0x1e3c8,
3860                 0x1e408, 0x1e40c,
3861                 0x1e440, 0x1e44c,
3862                 0x1e684, 0x1e690,
3863                 0x1e6c0, 0x1e6c0,
3864                 0x1e6e0, 0x1e6e0,
3865                 0x1e700, 0x1e784,
3866                 0x1e7c0, 0x1e7c8,
3867                 0x1e808, 0x1e80c,
3868                 0x1e840, 0x1e84c,
3869                 0x1ea84, 0x1ea90,
3870                 0x1eac0, 0x1eac0,
3871                 0x1eae0, 0x1eae0,
3872                 0x1eb00, 0x1eb84,
3873                 0x1ebc0, 0x1ebc8,
3874                 0x1ec08, 0x1ec0c,
3875                 0x1ec40, 0x1ec4c,
3876                 0x1ee84, 0x1ee90,
3877                 0x1eec0, 0x1eec0,
3878                 0x1eee0, 0x1eee0,
3879                 0x1ef00, 0x1ef84,
3880                 0x1efc0, 0x1efc8,
3881                 0x1f008, 0x1f00c,
3882                 0x1f040, 0x1f04c,
3883                 0x1f284, 0x1f290,
3884                 0x1f2c0, 0x1f2c0,
3885                 0x1f2e0, 0x1f2e0,
3886                 0x1f300, 0x1f384,
3887                 0x1f3c0, 0x1f3c8,
3888                 0x1f408, 0x1f40c,
3889                 0x1f440, 0x1f44c,
3890                 0x1f684, 0x1f690,
3891                 0x1f6c0, 0x1f6c0,
3892                 0x1f6e0, 0x1f6e0,
3893                 0x1f700, 0x1f784,
3894                 0x1f7c0, 0x1f7c8,
3895                 0x1f808, 0x1f80c,
3896                 0x1f840, 0x1f84c,
3897                 0x1fa84, 0x1fa90,
3898                 0x1fac0, 0x1fac0,
3899                 0x1fae0, 0x1fae0,
3900                 0x1fb00, 0x1fb84,
3901                 0x1fbc0, 0x1fbc8,
3902                 0x1fc08, 0x1fc0c,
3903                 0x1fc40, 0x1fc4c,
3904                 0x1fe84, 0x1fe90,
3905                 0x1fec0, 0x1fec0,
3906                 0x1fee0, 0x1fee0,
3907                 0x1ff00, 0x1ff84,
3908                 0x1ffc0, 0x1ffc8,
3909                 0x30000, 0x30030,
3910                 0x30100, 0x30144,
3911                 0x30190, 0x301d0,
3912                 0x30200, 0x30318,
3913                 0x30400, 0x3052c,
3914                 0x30540, 0x3061c,
3915                 0x30800, 0x30834,
3916                 0x308c0, 0x30908,
3917                 0x30910, 0x309ac,
3918                 0x30a00, 0x30a2c,
3919                 0x30a44, 0x30a50,
3920                 0x30a74, 0x30c24,
3921                 0x30d00, 0x30d00,
3922                 0x30d08, 0x30d14,
3923                 0x30d1c, 0x30d20,
3924                 0x30d3c, 0x30d50,
3925                 0x31200, 0x3120c,
3926                 0x31220, 0x31220,
3927                 0x31240, 0x31240,
3928                 0x31600, 0x3160c,
3929                 0x31a00, 0x31a1c,
3930                 0x31e00, 0x31e20,
3931                 0x31e38, 0x31e3c,
3932                 0x31e80, 0x31e80,
3933                 0x31e88, 0x31ea8,
3934                 0x31eb0, 0x31eb4,
3935                 0x31ec8, 0x31ed4,
3936                 0x31fb8, 0x32004,
3937                 0x32200, 0x32200,
3938                 0x32208, 0x32240,
3939                 0x32248, 0x32280,
3940                 0x32288, 0x322c0,
3941                 0x322c8, 0x322fc,
3942                 0x32600, 0x32630,
3943                 0x32a00, 0x32abc,
3944                 0x32b00, 0x32b70,
3945                 0x33000, 0x33048,
3946                 0x33060, 0x3309c,
3947                 0x330f0, 0x33148,
3948                 0x33160, 0x3319c,
3949                 0x331f0, 0x332e4,
3950                 0x332f8, 0x333e4,
3951                 0x333f8, 0x33448,
3952                 0x33460, 0x3349c,
3953                 0x334f0, 0x33548,
3954                 0x33560, 0x3359c,
3955                 0x335f0, 0x336e4,
3956                 0x336f8, 0x337e4,
3957                 0x337f8, 0x337fc,
3958                 0x33814, 0x33814,
3959                 0x3382c, 0x3382c,
3960                 0x33880, 0x3388c,
3961                 0x338e8, 0x338ec,
3962                 0x33900, 0x33948,
3963                 0x33960, 0x3399c,
3964                 0x339f0, 0x33ae4,
3965                 0x33af8, 0x33b10,
3966                 0x33b28, 0x33b28,
3967                 0x33b3c, 0x33b50,
3968                 0x33bf0, 0x33c10,
3969                 0x33c28, 0x33c28,
3970                 0x33c3c, 0x33c50,
3971                 0x33cf0, 0x33cfc,
3972                 0x34000, 0x34030,
3973                 0x34100, 0x34144,
3974                 0x34190, 0x341d0,
3975                 0x34200, 0x34318,
3976                 0x34400, 0x3452c,
3977                 0x34540, 0x3461c,
3978                 0x34800, 0x34834,
3979                 0x348c0, 0x34908,
3980                 0x34910, 0x349ac,
3981                 0x34a00, 0x34a2c,
3982                 0x34a44, 0x34a50,
3983                 0x34a74, 0x34c24,
3984                 0x34d00, 0x34d00,
3985                 0x34d08, 0x34d14,
3986                 0x34d1c, 0x34d20,
3987                 0x34d3c, 0x34d50,
3988                 0x35200, 0x3520c,
3989                 0x35220, 0x35220,
3990                 0x35240, 0x35240,
3991                 0x35600, 0x3560c,
3992                 0x35a00, 0x35a1c,
3993                 0x35e00, 0x35e20,
3994                 0x35e38, 0x35e3c,
3995                 0x35e80, 0x35e80,
3996                 0x35e88, 0x35ea8,
3997                 0x35eb0, 0x35eb4,
3998                 0x35ec8, 0x35ed4,
3999                 0x35fb8, 0x36004,
4000                 0x36200, 0x36200,
4001                 0x36208, 0x36240,
4002                 0x36248, 0x36280,
4003                 0x36288, 0x362c0,
4004                 0x362c8, 0x362fc,
4005                 0x36600, 0x36630,
4006                 0x36a00, 0x36abc,
4007                 0x36b00, 0x36b70,
4008                 0x37000, 0x37048,
4009                 0x37060, 0x3709c,
4010                 0x370f0, 0x37148,
4011                 0x37160, 0x3719c,
4012                 0x371f0, 0x372e4,
4013                 0x372f8, 0x373e4,
4014                 0x373f8, 0x37448,
4015                 0x37460, 0x3749c,
4016                 0x374f0, 0x37548,
4017                 0x37560, 0x3759c,
4018                 0x375f0, 0x376e4,
4019                 0x376f8, 0x377e4,
4020                 0x377f8, 0x377fc,
4021                 0x37814, 0x37814,
4022                 0x3782c, 0x3782c,
4023                 0x37880, 0x3788c,
4024                 0x378e8, 0x378ec,
4025                 0x37900, 0x37948,
4026                 0x37960, 0x3799c,
4027                 0x379f0, 0x37ae4,
4028                 0x37af8, 0x37b10,
4029                 0x37b28, 0x37b28,
4030                 0x37b3c, 0x37b50,
4031                 0x37bf0, 0x37c10,
4032                 0x37c28, 0x37c28,
4033                 0x37c3c, 0x37c50,
4034                 0x37cf0, 0x37cfc,
4035                 0x38000, 0x38030,
4036                 0x38100, 0x38144,
4037                 0x38190, 0x381d0,
4038                 0x38200, 0x38318,
4039                 0x38400, 0x3852c,
4040                 0x38540, 0x3861c,
4041                 0x38800, 0x38834,
4042                 0x388c0, 0x38908,
4043                 0x38910, 0x389ac,
4044                 0x38a00, 0x38a2c,
4045                 0x38a44, 0x38a50,
4046                 0x38a74, 0x38c24,
4047                 0x38d00, 0x38d00,
4048                 0x38d08, 0x38d14,
4049                 0x38d1c, 0x38d20,
4050                 0x38d3c, 0x38d50,
4051                 0x39200, 0x3920c,
4052                 0x39220, 0x39220,
4053                 0x39240, 0x39240,
4054                 0x39600, 0x3960c,
4055                 0x39a00, 0x39a1c,
4056                 0x39e00, 0x39e20,
4057                 0x39e38, 0x39e3c,
4058                 0x39e80, 0x39e80,
4059                 0x39e88, 0x39ea8,
4060                 0x39eb0, 0x39eb4,
4061                 0x39ec8, 0x39ed4,
4062                 0x39fb8, 0x3a004,
4063                 0x3a200, 0x3a200,
4064                 0x3a208, 0x3a240,
4065                 0x3a248, 0x3a280,
4066                 0x3a288, 0x3a2c0,
4067                 0x3a2c8, 0x3a2fc,
4068                 0x3a600, 0x3a630,
4069                 0x3aa00, 0x3aabc,
4070                 0x3ab00, 0x3ab70,
4071                 0x3b000, 0x3b048,
4072                 0x3b060, 0x3b09c,
4073                 0x3b0f0, 0x3b148,
4074                 0x3b160, 0x3b19c,
4075                 0x3b1f0, 0x3b2e4,
4076                 0x3b2f8, 0x3b3e4,
4077                 0x3b3f8, 0x3b448,
4078                 0x3b460, 0x3b49c,
4079                 0x3b4f0, 0x3b548,
4080                 0x3b560, 0x3b59c,
4081                 0x3b5f0, 0x3b6e4,
4082                 0x3b6f8, 0x3b7e4,
4083                 0x3b7f8, 0x3b7fc,
4084                 0x3b814, 0x3b814,
4085                 0x3b82c, 0x3b82c,
4086                 0x3b880, 0x3b88c,
4087                 0x3b8e8, 0x3b8ec,
4088                 0x3b900, 0x3b948,
4089                 0x3b960, 0x3b99c,
4090                 0x3b9f0, 0x3bae4,
4091                 0x3baf8, 0x3bb10,
4092                 0x3bb28, 0x3bb28,
4093                 0x3bb3c, 0x3bb50,
4094                 0x3bbf0, 0x3bc10,
4095                 0x3bc28, 0x3bc28,
4096                 0x3bc3c, 0x3bc50,
4097                 0x3bcf0, 0x3bcfc,
4098                 0x3c000, 0x3c030,
4099                 0x3c100, 0x3c144,
4100                 0x3c190, 0x3c1d0,
4101                 0x3c200, 0x3c318,
4102                 0x3c400, 0x3c52c,
4103                 0x3c540, 0x3c61c,
4104                 0x3c800, 0x3c834,
4105                 0x3c8c0, 0x3c908,
4106                 0x3c910, 0x3c9ac,
4107                 0x3ca00, 0x3ca2c,
4108                 0x3ca44, 0x3ca50,
4109                 0x3ca74, 0x3cc24,
4110                 0x3cd00, 0x3cd00,
4111                 0x3cd08, 0x3cd14,
4112                 0x3cd1c, 0x3cd20,
4113                 0x3cd3c, 0x3cd50,
4114                 0x3d200, 0x3d20c,
4115                 0x3d220, 0x3d220,
4116                 0x3d240, 0x3d240,
4117                 0x3d600, 0x3d60c,
4118                 0x3da00, 0x3da1c,
4119                 0x3de00, 0x3de20,
4120                 0x3de38, 0x3de3c,
4121                 0x3de80, 0x3de80,
4122                 0x3de88, 0x3dea8,
4123                 0x3deb0, 0x3deb4,
4124                 0x3dec8, 0x3ded4,
4125                 0x3dfb8, 0x3e004,
4126                 0x3e200, 0x3e200,
4127                 0x3e208, 0x3e240,
4128                 0x3e248, 0x3e280,
4129                 0x3e288, 0x3e2c0,
4130                 0x3e2c8, 0x3e2fc,
4131                 0x3e600, 0x3e630,
4132                 0x3ea00, 0x3eabc,
4133                 0x3eb00, 0x3eb70,
4134                 0x3f000, 0x3f048,
4135                 0x3f060, 0x3f09c,
4136                 0x3f0f0, 0x3f148,
4137                 0x3f160, 0x3f19c,
4138                 0x3f1f0, 0x3f2e4,
4139                 0x3f2f8, 0x3f3e4,
4140                 0x3f3f8, 0x3f448,
4141                 0x3f460, 0x3f49c,
4142                 0x3f4f0, 0x3f548,
4143                 0x3f560, 0x3f59c,
4144                 0x3f5f0, 0x3f6e4,
4145                 0x3f6f8, 0x3f7e4,
4146                 0x3f7f8, 0x3f7fc,
4147                 0x3f814, 0x3f814,
4148                 0x3f82c, 0x3f82c,
4149                 0x3f880, 0x3f88c,
4150                 0x3f8e8, 0x3f8ec,
4151                 0x3f900, 0x3f948,
4152                 0x3f960, 0x3f99c,
4153                 0x3f9f0, 0x3fae4,
4154                 0x3faf8, 0x3fb10,
4155                 0x3fb28, 0x3fb28,
4156                 0x3fb3c, 0x3fb50,
4157                 0x3fbf0, 0x3fc10,
4158                 0x3fc28, 0x3fc28,
4159                 0x3fc3c, 0x3fc50,
4160                 0x3fcf0, 0x3fcfc,
4161                 0x40000, 0x4000c,
4162                 0x40040, 0x40068,
4163                 0x4007c, 0x40144,
4164                 0x40180, 0x4018c,
4165                 0x40200, 0x40298,
4166                 0x402ac, 0x4033c,
4167                 0x403f8, 0x403fc,
4168                 0x41304, 0x413c4,
4169                 0x41400, 0x4141c,
4170                 0x41480, 0x414d0,
4171                 0x44000, 0x44078,
4172                 0x440c0, 0x44278,
4173                 0x442c0, 0x44478,
4174                 0x444c0, 0x44678,
4175                 0x446c0, 0x44878,
4176                 0x448c0, 0x449fc,
4177                 0x45000, 0x45068,
4178                 0x45080, 0x45084,
4179                 0x450a0, 0x450b0,
4180                 0x45200, 0x45268,
4181                 0x45280, 0x45284,
4182                 0x452a0, 0x452b0,
4183                 0x460c0, 0x460e4,
4184                 0x47000, 0x4708c,
4185                 0x47200, 0x47250,
4186                 0x47400, 0x47420,
4187                 0x47600, 0x47618,
4188                 0x47800, 0x47814,
4189                 0x48000, 0x4800c,
4190                 0x48040, 0x48068,
4191                 0x4807c, 0x48144,
4192                 0x48180, 0x4818c,
4193                 0x48200, 0x48298,
4194                 0x482ac, 0x4833c,
4195                 0x483f8, 0x483fc,
4196                 0x49304, 0x493c4,
4197                 0x49400, 0x4941c,
4198                 0x49480, 0x494d0,
4199                 0x4c000, 0x4c078,
4200                 0x4c0c0, 0x4c278,
4201                 0x4c2c0, 0x4c478,
4202                 0x4c4c0, 0x4c678,
4203                 0x4c6c0, 0x4c878,
4204                 0x4c8c0, 0x4c9fc,
4205                 0x4d000, 0x4d068,
4206                 0x4d080, 0x4d084,
4207                 0x4d0a0, 0x4d0b0,
4208                 0x4d200, 0x4d268,
4209                 0x4d280, 0x4d284,
4210                 0x4d2a0, 0x4d2b0,
4211                 0x4e0c0, 0x4e0e4,
4212                 0x4f000, 0x4f08c,
4213                 0x4f200, 0x4f250,
4214                 0x4f400, 0x4f420,
4215                 0x4f600, 0x4f618,
4216                 0x4f800, 0x4f814,
4217                 0x50000, 0x500cc,
4218                 0x50400, 0x50400,
4219                 0x50800, 0x508cc,
4220                 0x50c00, 0x50c00,
4221                 0x51000, 0x5101c,
4222                 0x51300, 0x51308,
4223         };
4224
4225         if (is_t4(sc)) {
4226                 reg_ranges = &t4_reg_ranges[0];
4227                 n = nitems(t4_reg_ranges);
4228         } else {
4229                 reg_ranges = &t5_reg_ranges[0];
4230                 n = nitems(t5_reg_ranges);
4231         }
4232
4233         regs->version = chip_id(sc) | chip_rev(sc) << 10;
4234         for (i = 0; i < n; i += 2)
4235                 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
4236 }
4237
4238 static void
4239 cxgbe_tick(void *arg)
4240 {
4241         struct port_info *pi = arg;
4242         struct adapter *sc = pi->adapter;
4243         struct ifnet *ifp = pi->ifp;
4244         struct sge_txq *txq;
4245         int i, drops;
4246         struct port_stats *s = &pi->stats;
4247
4248         PORT_LOCK(pi);
4249         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4250                 PORT_UNLOCK(pi);
4251                 return; /* without scheduling another callout */
4252         }
4253
4254         t4_get_port_stats(sc, pi->tx_chan, s);
4255
4256         ifp->if_opackets = s->tx_frames - s->tx_pause;
4257         ifp->if_ipackets = s->rx_frames - s->rx_pause;
4258         ifp->if_obytes = s->tx_octets - s->tx_pause * 64;
4259         ifp->if_ibytes = s->rx_octets - s->rx_pause * 64;
4260         ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause;
4261         ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause;
4262         ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
4263             s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
4264             s->rx_trunc3;
4265         for (i = 0; i < 4; i++) {
4266                 if (pi->rx_chan_map & (1 << i)) {
4267                         uint32_t v;
4268
4269                         /*
4270                          * XXX: indirect reads from the same ADDR/DATA pair can
4271                          * race with each other.
4272                          */
4273                         t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
4274                             1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4275                         ifp->if_iqdrops += v;
4276                 }
4277         }
4278
4279         drops = s->tx_drop;
4280         for_each_txq(pi, i, txq)
4281                 drops += txq->br->br_drops;
4282         ifp->if_snd.ifq_drops = drops;
4283
4284         ifp->if_oerrors = s->tx_error_frames;
4285         ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long +
4286             s->rx_fcs_err + s->rx_len_err;
4287
4288         callout_schedule(&pi->tick, hz);
4289         PORT_UNLOCK(pi);
4290 }
4291
4292 static void
4293 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4294 {
4295         struct ifnet *vlan;
4296
4297         if (arg != ifp || ifp->if_type != IFT_ETHER)
4298                 return;
4299
4300         vlan = VLAN_DEVAT(ifp, vid);
4301         VLAN_SETCOOKIE(vlan, ifp);
4302 }
4303
4304 static int
4305 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4306 {
4307
4308 #ifdef INVARIANTS
4309         panic("%s: opcode 0x%02x on iq %p with payload %p",
4310             __func__, rss->opcode, iq, m);
4311 #else
4312         log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
4313             __func__, rss->opcode, iq, m);
4314         m_freem(m);
4315 #endif
4316         return (EDOOFUS);
4317 }
4318
4319 int
4320 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
4321 {
4322         uintptr_t *loc, new;
4323
4324         if (opcode >= nitems(sc->cpl_handler))
4325                 return (EINVAL);
4326
4327         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
4328         loc = (uintptr_t *) &sc->cpl_handler[opcode];
4329         atomic_store_rel_ptr(loc, new);
4330
4331         return (0);
4332 }
4333
4334 static int
4335 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
4336 {
4337
4338 #ifdef INVARIANTS
4339         panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
4340 #else
4341         log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
4342             __func__, iq, ctrl);
4343 #endif
4344         return (EDOOFUS);
4345 }
4346
4347 int
4348 t4_register_an_handler(struct adapter *sc, an_handler_t h)
4349 {
4350         uintptr_t *loc, new;
4351
4352         new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
4353         loc = (uintptr_t *) &sc->an_handler;
4354         atomic_store_rel_ptr(loc, new);
4355
4356         return (0);
4357 }
4358
4359 static int
4360 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
4361 {
4362         const struct cpl_fw6_msg *cpl =
4363             __containerof(rpl, struct cpl_fw6_msg, data[0]);
4364
4365 #ifdef INVARIANTS
4366         panic("%s: fw_msg type %d", __func__, cpl->type);
4367 #else
4368         log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
4369 #endif
4370         return (EDOOFUS);
4371 }
4372
4373 int
4374 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
4375 {
4376         uintptr_t *loc, new;
4377
4378         if (type >= nitems(sc->fw_msg_handler))
4379                 return (EINVAL);
4380
4381         /*
4382          * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
4383          * handler dispatch table.  Reject any attempt to install a handler for
4384          * this subtype.
4385          */
4386         if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
4387                 return (EINVAL);
4388
4389         new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
4390         loc = (uintptr_t *) &sc->fw_msg_handler[type];
4391         atomic_store_rel_ptr(loc, new);
4392
4393         return (0);
4394 }
4395
4396 static int
4397 t4_sysctls(struct adapter *sc)
4398 {
4399         struct sysctl_ctx_list *ctx;
4400         struct sysctl_oid *oid;
4401         struct sysctl_oid_list *children, *c0;
4402         static char *caps[] = {
4403                 "\20\1PPP\2QFC\3DCBX",                  /* caps[0] linkcaps */
4404                 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL"        /* caps[1] niccaps */
4405                     "\6HASHFILTER\7ETHOFLD",
4406                 "\20\1TOE",                             /* caps[2] toecaps */
4407                 "\20\1RDDP\2RDMAC",                     /* caps[3] rdmacaps */
4408                 "\20\1INITIATOR_PDU\2TARGET_PDU"        /* caps[4] iscsicaps */
4409                     "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
4410                     "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
4411                 "\20\1INITIATOR\2TARGET\3CTRL_OFLD"     /* caps[5] fcoecaps */
4412                     "\4PO_INITIAOR\5PO_TARGET"
4413         };
4414         static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
4415
4416         ctx = device_get_sysctl_ctx(sc->dev);
4417
4418         /*
4419          * dev.t4nex.X.
4420          */
4421         oid = device_get_sysctl_tree(sc->dev);
4422         c0 = children = SYSCTL_CHILDREN(oid);
4423
4424         sc->sc_do_rxcopy = 1;
4425         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
4426             &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
4427
4428         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
4429             sc->params.nports, "# of ports");
4430
4431         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
4432             NULL, chip_rev(sc), "chip hardware revision");
4433
4434         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
4435             CTLFLAG_RD, sc->fw_version, 0, "firmware version");
4436
4437         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
4438             CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
4439
4440         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
4441             sc->cfcsum, "config file checksum");
4442
4443         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
4444             CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
4445             sysctl_bitfield, "A", "available doorbells");
4446
4447         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
4448             CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
4449             sysctl_bitfield, "A", "available link capabilities");
4450
4451         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
4452             CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
4453             sysctl_bitfield, "A", "available NIC capabilities");
4454
4455         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
4456             CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
4457             sysctl_bitfield, "A", "available TCP offload capabilities");
4458
4459         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
4460             CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
4461             sysctl_bitfield, "A", "available RDMA capabilities");
4462
4463         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
4464             CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
4465             sysctl_bitfield, "A", "available iSCSI capabilities");
4466
4467         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
4468             CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
4469             sysctl_bitfield, "A", "available FCoE capabilities");
4470
4471         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
4472             sc->params.vpd.cclk, "core clock frequency (in KHz)");
4473
4474         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
4475             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
4476             sizeof(sc->sge.timer_val), sysctl_int_array, "A",
4477             "interrupt holdoff timer values (us)");
4478
4479         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
4480             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
4481             sizeof(sc->sge.counter_val), sysctl_int_array, "A",
4482             "interrupt holdoff packet counter values");
4483
4484         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
4485             NULL, sc->tids.nftids, "number of filters");
4486
4487         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
4488             CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
4489             "chip temperature (in Celsius)");
4490
4491         t4_sge_sysctls(sc, ctx, children);
4492
4493         sc->lro_timeout = 100;
4494         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
4495             &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
4496
4497 #ifdef SBUF_DRAIN
4498         /*
4499          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
4500          */
4501         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
4502             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
4503             "logs and miscellaneous information");
4504         children = SYSCTL_CHILDREN(oid);
4505
4506         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
4507             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4508             sysctl_cctrl, "A", "congestion control");
4509
4510         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
4511             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4512             sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
4513
4514         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
4515             CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
4516             sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
4517
4518         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
4519             CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
4520             sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
4521
4522         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
4523             CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
4524             sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
4525
4526         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
4527             CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
4528             sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
4529
4530         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
4531             CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
4532             sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
4533
4534         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
4535             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4536             sysctl_cim_la, "A", "CIM logic analyzer");
4537
4538         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
4539             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4540             sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
4541
4542         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
4543             CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
4544             sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
4545
4546         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
4547             CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
4548             sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
4549
4550         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
4551             CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
4552             sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
4553
4554         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
4555             CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
4556             sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
4557
4558         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
4559             CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
4560             sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
4561
4562         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
4563             CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
4564             sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
4565
4566         if (is_t5(sc)) {
4567                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
4568                     CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
4569                     sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
4570
4571                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
4572                     CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
4573                     sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
4574         }
4575
4576         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
4577             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4578             sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
4579
4580         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
4581             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4582             sysctl_cim_qcfg, "A", "CIM queue configuration");
4583
4584         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
4585             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4586             sysctl_cpl_stats, "A", "CPL statistics");
4587
4588         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
4589             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4590             sysctl_ddp_stats, "A", "DDP statistics");
4591
4592         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
4593             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4594             sysctl_devlog, "A", "firmware's device log");
4595
4596         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
4597             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4598             sysctl_fcoe_stats, "A", "FCoE statistics");
4599
4600         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
4601             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4602             sysctl_hw_sched, "A", "hardware scheduler ");
4603
4604         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
4605             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4606             sysctl_l2t, "A", "hardware L2 table");
4607
4608         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
4609             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4610             sysctl_lb_stats, "A", "loopback statistics");
4611
4612         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
4613             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4614             sysctl_meminfo, "A", "memory regions");
4615
4616         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
4617             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4618             sysctl_mps_tcam, "A", "MPS TCAM entries");
4619
4620         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
4621             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4622             sysctl_path_mtus, "A", "path MTUs");
4623
4624         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
4625             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4626             sysctl_pm_stats, "A", "PM statistics");
4627
4628         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
4629             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4630             sysctl_rdma_stats, "A", "RDMA statistics");
4631
4632         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
4633             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4634             sysctl_tcp_stats, "A", "TCP statistics");
4635
4636         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
4637             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4638             sysctl_tids, "A", "TID information");
4639
4640         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
4641             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4642             sysctl_tp_err_stats, "A", "TP error statistics");
4643
4644         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
4645             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4646             sysctl_tp_la, "A", "TP logic analyzer");
4647
4648         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
4649             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4650             sysctl_tx_rate, "A", "Tx rate");
4651
4652         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
4653             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4654             sysctl_ulprx_la, "A", "ULPRX logic analyzer");
4655
4656         if (is_t5(sc)) {
4657                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
4658                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4659                     sysctl_wcwr_stats, "A", "write combined work requests");
4660         }
4661 #endif
4662
4663 #ifdef TCP_OFFLOAD
4664         if (is_offload(sc)) {
4665                 /*
4666                  * dev.t4nex.X.toe.
4667                  */
4668                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
4669                     NULL, "TOE parameters");
4670                 children = SYSCTL_CHILDREN(oid);
4671
4672                 sc->tt.sndbuf = 256 * 1024;
4673                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
4674                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
4675
4676                 sc->tt.ddp = 0;
4677                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
4678                     &sc->tt.ddp, 0, "DDP allowed");
4679
4680                 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
4681                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
4682                     &sc->tt.indsz, 0, "DDP max indicate size allowed");
4683
4684                 sc->tt.ddp_thres =
4685                     G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
4686                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
4687                     &sc->tt.ddp_thres, 0, "DDP threshold");
4688
4689                 sc->tt.rx_coalesce = 1;
4690                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
4691                     CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
4692         }
4693 #endif
4694
4695
4696         return (0);
4697 }
4698
4699 static int
4700 cxgbe_sysctls(struct port_info *pi)
4701 {
4702         struct sysctl_ctx_list *ctx;
4703         struct sysctl_oid *oid;
4704         struct sysctl_oid_list *children;
4705         struct adapter *sc = pi->adapter;
4706
4707         ctx = device_get_sysctl_ctx(pi->dev);
4708
4709         /*
4710          * dev.cxgbe.X.
4711          */
4712         oid = device_get_sysctl_tree(pi->dev);
4713         children = SYSCTL_CHILDREN(oid);
4714
4715         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
4716            CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
4717         if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
4718                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
4719                     CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
4720                     "PHY temperature (in Celsius)");
4721                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
4722                     CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
4723                     "PHY firmware version");
4724         }
4725         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
4726             &pi->nrxq, 0, "# of rx queues");
4727         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
4728             &pi->ntxq, 0, "# of tx queues");
4729         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
4730             &pi->first_rxq, 0, "index of first rx queue");
4731         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
4732             &pi->first_txq, 0, "index of first tx queue");
4733         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
4734             CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU",
4735             "Reserve queue 0 for non-flowid packets");
4736
4737 #ifdef TCP_OFFLOAD
4738         if (is_offload(sc)) {
4739                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
4740                     &pi->nofldrxq, 0,
4741                     "# of rx queues for offloaded TCP connections");
4742                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
4743                     &pi->nofldtxq, 0,
4744                     "# of tx queues for offloaded TCP connections");
4745                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
4746                     CTLFLAG_RD, &pi->first_ofld_rxq, 0,
4747                     "index of first TOE rx queue");
4748                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
4749                     CTLFLAG_RD, &pi->first_ofld_txq, 0,
4750                     "index of first TOE tx queue");
4751         }
4752 #endif
4753 #ifdef DEV_NETMAP
4754         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
4755             &pi->nnmrxq, 0, "# of rx queues for netmap");
4756         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
4757             &pi->nnmtxq, 0, "# of tx queues for netmap");
4758         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
4759             CTLFLAG_RD, &pi->first_nm_rxq, 0,
4760             "index of first netmap rx queue");
4761         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
4762             CTLFLAG_RD, &pi->first_nm_txq, 0,
4763             "index of first netmap tx queue");
4764 #endif
4765
4766         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
4767             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
4768             "holdoff timer index");
4769         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
4770             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
4771             "holdoff packet counter index");
4772
4773         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
4774             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
4775             "rx queue size");
4776         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
4777             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
4778             "tx queue size");
4779
4780         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
4781             CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings,
4782             "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
4783
4784         /*
4785          * dev.cxgbe.X.stats.
4786          */
4787         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
4788             NULL, "port statistics");
4789         children = SYSCTL_CHILDREN(oid);
4790
4791 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
4792         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
4793             CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
4794             sysctl_handle_t4_reg64, "QU", desc)
4795
4796         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
4797             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
4798         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
4799             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
4800         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
4801             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
4802         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
4803             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
4804         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
4805             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
4806         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
4807             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
4808         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
4809             "# of tx frames in this range",
4810             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
4811         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
4812             "# of tx frames in this range",
4813             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
4814         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
4815             "# of tx frames in this range",
4816             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
4817         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
4818             "# of tx frames in this range",
4819             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
4820         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
4821             "# of tx frames in this range",
4822             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
4823         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
4824             "# of tx frames in this range",
4825             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
4826         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
4827             "# of tx frames in this range",
4828             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
4829         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
4830             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
4831         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
4832             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
4833         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
4834             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
4835         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
4836             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
4837         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
4838             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
4839         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
4840             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
4841         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
4842             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
4843         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
4844             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
4845         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
4846             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
4847         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
4848             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
4849
4850         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
4851             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
4852         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
4853             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
4854         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
4855             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
4856         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
4857             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
4858         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
4859             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
4860         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
4861             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
4862         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
4863             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
4864         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
4865             "# of frames received with bad FCS",
4866             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
4867         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
4868             "# of frames received with length error",
4869             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
4870         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
4871             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
4872         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
4873             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
4874         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
4875             "# of rx frames in this range",
4876             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
4877         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
4878             "# of rx frames in this range",
4879             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
4880         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
4881             "# of rx frames in this range",
4882             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
4883         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
4884             "# of rx frames in this range",
4885             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
4886         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
4887             "# of rx frames in this range",
4888             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
4889         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
4890             "# of rx frames in this range",
4891             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
4892         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
4893             "# of rx frames in this range",
4894             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
4895         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
4896             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
4897         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
4898             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
4899         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
4900             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
4901         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
4902             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
4903         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
4904             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
4905         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
4906             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
4907         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
4908             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
4909         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
4910             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
4911         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
4912             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
4913
4914 #undef SYSCTL_ADD_T4_REG64
4915
4916 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
4917         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
4918             &pi->stats.name, desc)
4919
4920         /* We get these from port_stats and they may be stale by upto 1s */
4921         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
4922             "# drops due to buffer-group 0 overflows");
4923         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
4924             "# drops due to buffer-group 1 overflows");
4925         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
4926             "# drops due to buffer-group 2 overflows");
4927         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
4928             "# drops due to buffer-group 3 overflows");
4929         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
4930             "# of buffer-group 0 truncated packets");
4931         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
4932             "# of buffer-group 1 truncated packets");
4933         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
4934             "# of buffer-group 2 truncated packets");
4935         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
4936             "# of buffer-group 3 truncated packets");
4937
4938 #undef SYSCTL_ADD_T4_PORTSTAT
4939
4940         return (0);
4941 }
4942
4943 static int
4944 sysctl_int_array(SYSCTL_HANDLER_ARGS)
4945 {
4946         int rc, *i;
4947         struct sbuf sb;
4948
4949         sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
4950         for (i = arg1; arg2; arg2 -= sizeof(int), i++)
4951                 sbuf_printf(&sb, "%d ", *i);
4952         sbuf_trim(&sb);
4953         sbuf_finish(&sb);
4954         rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
4955         sbuf_delete(&sb);
4956         return (rc);
4957 }
4958
4959 static int
4960 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
4961 {
4962         int rc;
4963         struct sbuf *sb;
4964
4965         rc = sysctl_wire_old_buffer(req, 0);
4966         if (rc != 0)
4967                 return(rc);
4968
4969         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
4970         if (sb == NULL)
4971                 return (ENOMEM);
4972
4973         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
4974         rc = sbuf_finish(sb);
4975         sbuf_delete(sb);
4976
4977         return (rc);
4978 }
4979
4980 static int
4981 sysctl_btphy(SYSCTL_HANDLER_ARGS)
4982 {
4983         struct port_info *pi = arg1;
4984         int op = arg2;
4985         struct adapter *sc = pi->adapter;
4986         u_int v;
4987         int rc;
4988
4989         rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt");
4990         if (rc)
4991                 return (rc);
4992         /* XXX: magic numbers */
4993         rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
4994             &v);
4995         end_synchronized_op(sc, 0);
4996         if (rc)
4997                 return (rc);
4998         if (op == 0)
4999                 v /= 256;
5000
5001         rc = sysctl_handle_int(oidp, &v, 0, req);
5002         return (rc);
5003 }
5004
5005 static int
5006 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
5007 {
5008         struct port_info *pi = arg1;
5009         int rc, val;
5010
5011         val = pi->rsrv_noflowq;
5012         rc = sysctl_handle_int(oidp, &val, 0, req);
5013         if (rc != 0 || req->newptr == NULL)
5014                 return (rc);
5015
5016         if ((val >= 1) && (pi->ntxq > 1))
5017                 pi->rsrv_noflowq = 1;
5018         else
5019                 pi->rsrv_noflowq = 0;
5020
5021         return (rc);
5022 }
5023
5024 static int
5025 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
5026 {
5027         struct port_info *pi = arg1;
5028         struct adapter *sc = pi->adapter;
5029         int idx, rc, i;
5030         struct sge_rxq *rxq;
5031 #ifdef TCP_OFFLOAD
5032         struct sge_ofld_rxq *ofld_rxq;
5033 #endif
5034         uint8_t v;
5035
5036         idx = pi->tmr_idx;
5037
5038         rc = sysctl_handle_int(oidp, &idx, 0, req);
5039         if (rc != 0 || req->newptr == NULL)
5040                 return (rc);
5041
5042         if (idx < 0 || idx >= SGE_NTIMERS)
5043                 return (EINVAL);
5044
5045         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5046             "t4tmr");
5047         if (rc)
5048                 return (rc);
5049
5050         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
5051         for_each_rxq(pi, i, rxq) {
5052 #ifdef atomic_store_rel_8
5053                 atomic_store_rel_8(&rxq->iq.intr_params, v);
5054 #else
5055                 rxq->iq.intr_params = v;
5056 #endif
5057         }
5058 #ifdef TCP_OFFLOAD
5059         for_each_ofld_rxq(pi, i, ofld_rxq) {
5060 #ifdef atomic_store_rel_8
5061                 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
5062 #else
5063                 ofld_rxq->iq.intr_params = v;
5064 #endif
5065         }
5066 #endif
5067         pi->tmr_idx = idx;
5068
5069         end_synchronized_op(sc, LOCK_HELD);
5070         return (0);
5071 }
5072
5073 static int
5074 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
5075 {
5076         struct port_info *pi = arg1;
5077         struct adapter *sc = pi->adapter;
5078         int idx, rc;
5079
5080         idx = pi->pktc_idx;
5081
5082         rc = sysctl_handle_int(oidp, &idx, 0, req);
5083         if (rc != 0 || req->newptr == NULL)
5084                 return (rc);
5085
5086         if (idx < -1 || idx >= SGE_NCOUNTERS)
5087                 return (EINVAL);
5088
5089         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5090             "t4pktc");
5091         if (rc)
5092                 return (rc);
5093
5094         if (pi->flags & PORT_INIT_DONE)
5095                 rc = EBUSY; /* cannot be changed once the queues are created */
5096         else
5097                 pi->pktc_idx = idx;
5098
5099         end_synchronized_op(sc, LOCK_HELD);
5100         return (rc);
5101 }
5102
5103 static int
5104 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
5105 {
5106         struct port_info *pi = arg1;
5107         struct adapter *sc = pi->adapter;
5108         int qsize, rc;
5109
5110         qsize = pi->qsize_rxq;
5111
5112         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5113         if (rc != 0 || req->newptr == NULL)
5114                 return (rc);
5115
5116         if (qsize < 128 || (qsize & 7))
5117                 return (EINVAL);
5118
5119         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5120             "t4rxqs");
5121         if (rc)
5122                 return (rc);
5123
5124         if (pi->flags & PORT_INIT_DONE)
5125                 rc = EBUSY; /* cannot be changed once the queues are created */
5126         else
5127                 pi->qsize_rxq = qsize;
5128
5129         end_synchronized_op(sc, LOCK_HELD);
5130         return (rc);
5131 }
5132
5133 static int
5134 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
5135 {
5136         struct port_info *pi = arg1;
5137         struct adapter *sc = pi->adapter;
5138         int qsize, rc;
5139
5140         qsize = pi->qsize_txq;
5141
5142         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5143         if (rc != 0 || req->newptr == NULL)
5144                 return (rc);
5145
5146         /* bufring size must be powerof2 */
5147         if (qsize < 128 || !powerof2(qsize))
5148                 return (EINVAL);
5149
5150         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5151             "t4txqs");
5152         if (rc)
5153                 return (rc);
5154
5155         if (pi->flags & PORT_INIT_DONE)
5156                 rc = EBUSY; /* cannot be changed once the queues are created */
5157         else
5158                 pi->qsize_txq = qsize;
5159
5160         end_synchronized_op(sc, LOCK_HELD);
5161         return (rc);
5162 }
5163
5164 static int
5165 sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
5166 {
5167         struct port_info *pi = arg1;
5168         struct adapter *sc = pi->adapter;
5169         struct link_config *lc = &pi->link_cfg;
5170         int rc;
5171
5172         if (req->newptr == NULL) {
5173                 struct sbuf *sb;
5174                 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX";
5175
5176                 rc = sysctl_wire_old_buffer(req, 0);
5177                 if (rc != 0)
5178                         return(rc);
5179
5180                 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5181                 if (sb == NULL)
5182                         return (ENOMEM);
5183
5184                 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits);
5185                 rc = sbuf_finish(sb);
5186                 sbuf_delete(sb);
5187         } else {
5188                 char s[2];
5189                 int n;
5190
5191                 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX));
5192                 s[1] = 0;
5193
5194                 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
5195                 if (rc != 0)
5196                         return(rc);
5197
5198                 if (s[1] != 0)
5199                         return (EINVAL);
5200                 if (s[0] < '0' || s[0] > '9')
5201                         return (EINVAL);        /* not a number */
5202                 n = s[0] - '0';
5203                 if (n & ~(PAUSE_TX | PAUSE_RX))
5204                         return (EINVAL);        /* some other bit is set too */
5205
5206                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE");
5207                 if (rc)
5208                         return (rc);
5209                 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) {
5210                         int link_ok = lc->link_ok;
5211
5212                         lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX);
5213                         lc->requested_fc |= n;
5214                         rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc);
5215                         lc->link_ok = link_ok;  /* restore */
5216                 }
5217                 end_synchronized_op(sc, 0);
5218         }
5219
5220         return (rc);
5221 }
5222
5223 static int
5224 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
5225 {
5226         struct adapter *sc = arg1;
5227         int reg = arg2;
5228         uint64_t val;
5229
5230         val = t4_read_reg64(sc, reg);
5231
5232         return (sysctl_handle_64(oidp, &val, 0, req));
5233 }
5234
5235 static int
5236 sysctl_temperature(SYSCTL_HANDLER_ARGS)
5237 {
5238         struct adapter *sc = arg1;
5239         int rc, t;
5240         uint32_t param, val;
5241
5242         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
5243         if (rc)
5244                 return (rc);
5245         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5246             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5247             V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
5248         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
5249         end_synchronized_op(sc, 0);
5250         if (rc)
5251                 return (rc);
5252
5253         /* unknown is returned as 0 but we display -1 in that case */
5254         t = val == 0 ? -1 : val;
5255
5256         rc = sysctl_handle_int(oidp, &t, 0, req);
5257         return (rc);
5258 }
5259
5260 #ifdef SBUF_DRAIN
5261 static int
5262 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
5263 {
5264         struct adapter *sc = arg1;
5265         struct sbuf *sb;
5266         int rc, i;
5267         uint16_t incr[NMTUS][NCCTRL_WIN];
5268         static const char *dec_fac[] = {
5269                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
5270                 "0.9375"
5271         };
5272
5273         rc = sysctl_wire_old_buffer(req, 0);
5274         if (rc != 0)
5275                 return (rc);
5276
5277         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5278         if (sb == NULL)
5279                 return (ENOMEM);
5280
5281         t4_read_cong_tbl(sc, incr);
5282
5283         for (i = 0; i < NCCTRL_WIN; ++i) {
5284                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
5285                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
5286                     incr[5][i], incr[6][i], incr[7][i]);
5287                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
5288                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
5289                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
5290                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
5291         }
5292
5293         rc = sbuf_finish(sb);
5294         sbuf_delete(sb);
5295
5296         return (rc);
5297 }
5298
5299 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
5300         "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",   /* ibq's */
5301         "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
5302         "SGE0-RX", "SGE1-RX"    /* additional obq's (T5 onwards) */
5303 };
5304
5305 static int
5306 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
5307 {
5308         struct adapter *sc = arg1;
5309         struct sbuf *sb;
5310         int rc, i, n, qid = arg2;
5311         uint32_t *buf, *p;
5312         char *qtype;
5313         u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
5314
5315         KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
5316             ("%s: bad qid %d\n", __func__, qid));
5317
5318         if (qid < CIM_NUM_IBQ) {
5319                 /* inbound queue */
5320                 qtype = "IBQ";
5321                 n = 4 * CIM_IBQ_SIZE;
5322                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5323                 rc = t4_read_cim_ibq(sc, qid, buf, n);
5324         } else {
5325                 /* outbound queue */
5326                 qtype = "OBQ";
5327                 qid -= CIM_NUM_IBQ;
5328                 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
5329                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5330                 rc = t4_read_cim_obq(sc, qid, buf, n);
5331         }
5332
5333         if (rc < 0) {
5334                 rc = -rc;
5335                 goto done;
5336         }
5337         n = rc * sizeof(uint32_t);      /* rc has # of words actually read */
5338
5339         rc = sysctl_wire_old_buffer(req, 0);
5340         if (rc != 0)
5341                 goto done;
5342
5343         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5344         if (sb == NULL) {
5345                 rc = ENOMEM;
5346                 goto done;
5347         }
5348
5349         sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
5350         for (i = 0, p = buf; i < n; i += 16, p += 4)
5351                 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
5352                     p[2], p[3]);
5353
5354         rc = sbuf_finish(sb);
5355         sbuf_delete(sb);
5356 done:
5357         free(buf, M_CXGBE);
5358         return (rc);
5359 }
5360
5361 static int
5362 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
5363 {
5364         struct adapter *sc = arg1;
5365         u_int cfg;
5366         struct sbuf *sb;
5367         uint32_t *buf, *p;
5368         int rc;
5369
5370         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
5371         if (rc != 0)
5372                 return (rc);
5373
5374         rc = sysctl_wire_old_buffer(req, 0);
5375         if (rc != 0)
5376                 return (rc);
5377
5378         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5379         if (sb == NULL)
5380                 return (ENOMEM);
5381
5382         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
5383             M_ZERO | M_WAITOK);
5384
5385         rc = -t4_cim_read_la(sc, buf, NULL);
5386         if (rc != 0)
5387                 goto done;
5388
5389         sbuf_printf(sb, "Status   Data      PC%s",
5390             cfg & F_UPDBGLACAPTPCONLY ? "" :
5391             "     LS0Stat  LS0Addr             LS0Data");
5392
5393         KASSERT((sc->params.cim_la_size & 7) == 0,
5394             ("%s: p will walk off the end of buf", __func__));
5395
5396         for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
5397                 if (cfg & F_UPDBGLACAPTPCONLY) {
5398                         sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
5399                             p[6], p[7]);
5400                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
5401                             (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
5402                             p[4] & 0xff, p[5] >> 8);
5403                         sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
5404                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5405                             p[1] & 0xf, p[2] >> 4);
5406                 } else {
5407                         sbuf_printf(sb,
5408                             "\n  %02x   %x%07x %x%07x %08x %08x "
5409                             "%08x%08x%08x%08x",
5410                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5411                             p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
5412                             p[6], p[7]);
5413                 }
5414         }
5415
5416         rc = sbuf_finish(sb);
5417         sbuf_delete(sb);
5418 done:
5419         free(buf, M_CXGBE);
5420         return (rc);
5421 }
5422
5423 static int
5424 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
5425 {
5426         struct adapter *sc = arg1;
5427         u_int i;
5428         struct sbuf *sb;
5429         uint32_t *buf, *p;
5430         int rc;
5431
5432         rc = sysctl_wire_old_buffer(req, 0);
5433         if (rc != 0)
5434                 return (rc);
5435
5436         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5437         if (sb == NULL)
5438                 return (ENOMEM);
5439
5440         buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
5441             M_ZERO | M_WAITOK);
5442
5443         t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
5444         p = buf;
5445
5446         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5447                 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
5448                     p[1], p[0]);
5449         }
5450
5451         sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
5452         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5453                 sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
5454                     (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
5455                     (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
5456                     (p[1] >> 2) | ((p[2] & 3) << 30),
5457                     (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
5458                     p[0] & 1);
5459         }
5460
5461         rc = sbuf_finish(sb);
5462         sbuf_delete(sb);
5463         free(buf, M_CXGBE);
5464         return (rc);
5465 }
5466
5467 static int
5468 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
5469 {
5470         struct adapter *sc = arg1;
5471         u_int i;
5472         struct sbuf *sb;
5473         uint32_t *buf, *p;
5474         int rc;
5475
5476         rc = sysctl_wire_old_buffer(req, 0);
5477         if (rc != 0)
5478                 return (rc);
5479
5480         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5481         if (sb == NULL)
5482                 return (ENOMEM);
5483
5484         buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
5485             M_ZERO | M_WAITOK);
5486
5487         t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
5488         p = buf;
5489
5490         sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
5491         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5492                 sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
5493                     (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
5494                     p[4], p[3], p[2], p[1], p[0]);
5495         }
5496
5497         sbuf_printf(sb, "\n\nCntl ID               Data");
5498         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5499                 sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
5500                     (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
5501         }
5502
5503         rc = sbuf_finish(sb);
5504         sbuf_delete(sb);
5505         free(buf, M_CXGBE);
5506         return (rc);
5507 }
5508
5509 static int
5510 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
5511 {
5512         struct adapter *sc = arg1;
5513         struct sbuf *sb;
5514         int rc, i;
5515         uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5516         uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5517         uint16_t thres[CIM_NUM_IBQ];
5518         uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
5519         uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
5520         u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
5521
5522         if (is_t4(sc)) {
5523                 cim_num_obq = CIM_NUM_OBQ;
5524                 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
5525                 obq_rdaddr = A_UP_OBQ_0_REALADDR;
5526         } else {
5527                 cim_num_obq = CIM_NUM_OBQ_T5;
5528                 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
5529                 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
5530         }
5531         nq = CIM_NUM_IBQ + cim_num_obq;
5532
5533         rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
5534         if (rc == 0)
5535                 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
5536         if (rc != 0)
5537                 return (rc);
5538
5539         t4_read_cimq_cfg(sc, base, size, thres);
5540
5541         rc = sysctl_wire_old_buffer(req, 0);
5542         if (rc != 0)
5543                 return (rc);
5544
5545         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5546         if (sb == NULL)
5547                 return (ENOMEM);
5548
5549         sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
5550
5551         for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
5552                 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
5553                     qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
5554                     G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5555                     G_QUEREMFLITS(p[2]) * 16);
5556         for ( ; i < nq; i++, p += 4, wr += 2)
5557                 sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
5558                     base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
5559                     wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5560                     G_QUEREMFLITS(p[2]) * 16);
5561
5562         rc = sbuf_finish(sb);
5563         sbuf_delete(sb);
5564
5565         return (rc);
5566 }
5567
5568 static int
5569 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
5570 {
5571         struct adapter *sc = arg1;
5572         struct sbuf *sb;
5573         int rc;
5574         struct tp_cpl_stats stats;
5575
5576         rc = sysctl_wire_old_buffer(req, 0);
5577         if (rc != 0)
5578                 return (rc);
5579
5580         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5581         if (sb == NULL)
5582                 return (ENOMEM);
5583
5584         t4_tp_get_cpl_stats(sc, &stats);
5585
5586         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
5587             "channel 3\n");
5588         sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
5589                    stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
5590         sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
5591                    stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
5592
5593         rc = sbuf_finish(sb);
5594         sbuf_delete(sb);
5595
5596         return (rc);
5597 }
5598
5599 static int
5600 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
5601 {
5602         struct adapter *sc = arg1;
5603         struct sbuf *sb;
5604         int rc;
5605         struct tp_usm_stats stats;
5606
5607         rc = sysctl_wire_old_buffer(req, 0);
5608         if (rc != 0)
5609                 return(rc);
5610
5611         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5612         if (sb == NULL)
5613                 return (ENOMEM);
5614
5615         t4_get_usm_stats(sc, &stats);
5616
5617         sbuf_printf(sb, "Frames: %u\n", stats.frames);
5618         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
5619         sbuf_printf(sb, "Drops:  %u", stats.drops);
5620
5621         rc = sbuf_finish(sb);
5622         sbuf_delete(sb);
5623
5624         return (rc);
5625 }
5626
5627 const char *devlog_level_strings[] = {
5628         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
5629         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
5630         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
5631         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
5632         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
5633         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
5634 };
5635
5636 const char *devlog_facility_strings[] = {
5637         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
5638         [FW_DEVLOG_FACILITY_CF]         = "CF",
5639         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
5640         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
5641         [FW_DEVLOG_FACILITY_RES]        = "RES",
5642         [FW_DEVLOG_FACILITY_HW]         = "HW",
5643         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
5644         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
5645         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
5646         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
5647         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
5648         [FW_DEVLOG_FACILITY_VI]         = "VI",
5649         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
5650         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
5651         [FW_DEVLOG_FACILITY_TM]         = "TM",
5652         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
5653         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
5654         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
5655         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
5656         [FW_DEVLOG_FACILITY_RI]         = "RI",
5657         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
5658         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
5659         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
5660         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE"
5661 };
5662
5663 static int
5664 sysctl_devlog(SYSCTL_HANDLER_ARGS)
5665 {
5666         struct adapter *sc = arg1;
5667         struct devlog_params *dparams = &sc->params.devlog;
5668         struct fw_devlog_e *buf, *e;
5669         int i, j, rc, nentries, first = 0, m;
5670         struct sbuf *sb;
5671         uint64_t ftstamp = UINT64_MAX;
5672
5673         if (dparams->start == 0) {
5674                 dparams->memtype = FW_MEMTYPE_EDC0;
5675                 dparams->start = 0x84000;
5676                 dparams->size = 32768;
5677         }
5678
5679         nentries = dparams->size / sizeof(struct fw_devlog_e);
5680
5681         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
5682         if (buf == NULL)
5683                 return (ENOMEM);
5684
5685         m = fwmtype_to_hwmtype(dparams->memtype);
5686         rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
5687         if (rc != 0)
5688                 goto done;
5689
5690         for (i = 0; i < nentries; i++) {
5691                 e = &buf[i];
5692
5693                 if (e->timestamp == 0)
5694                         break;  /* end */
5695
5696                 e->timestamp = be64toh(e->timestamp);
5697                 e->seqno = be32toh(e->seqno);
5698                 for (j = 0; j < 8; j++)
5699                         e->params[j] = be32toh(e->params[j]);
5700
5701                 if (e->timestamp < ftstamp) {
5702                         ftstamp = e->timestamp;
5703                         first = i;
5704                 }
5705         }
5706
5707         if (buf[first].timestamp == 0)
5708                 goto done;      /* nothing in the log */
5709
5710         rc = sysctl_wire_old_buffer(req, 0);
5711         if (rc != 0)
5712                 goto done;
5713
5714         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5715         if (sb == NULL) {
5716                 rc = ENOMEM;
5717                 goto done;
5718         }
5719         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
5720             "Seq#", "Tstamp", "Level", "Facility", "Message");
5721
5722         i = first;
5723         do {
5724                 e = &buf[i];
5725                 if (e->timestamp == 0)
5726                         break;  /* end */
5727
5728                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
5729                     e->seqno, e->timestamp,
5730                     (e->level < nitems(devlog_level_strings) ?
5731                         devlog_level_strings[e->level] : "UNKNOWN"),
5732                     (e->facility < nitems(devlog_facility_strings) ?
5733                         devlog_facility_strings[e->facility] : "UNKNOWN"));
5734                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
5735                     e->params[2], e->params[3], e->params[4],
5736                     e->params[5], e->params[6], e->params[7]);
5737
5738                 if (++i == nentries)
5739                         i = 0;
5740         } while (i != first);
5741
5742         rc = sbuf_finish(sb);
5743         sbuf_delete(sb);
5744 done:
5745         free(buf, M_CXGBE);
5746         return (rc);
5747 }
5748
5749 static int
5750 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
5751 {
5752         struct adapter *sc = arg1;
5753         struct sbuf *sb;
5754         int rc;
5755         struct tp_fcoe_stats stats[4];
5756
5757         rc = sysctl_wire_old_buffer(req, 0);
5758         if (rc != 0)
5759                 return (rc);
5760
5761         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5762         if (sb == NULL)
5763                 return (ENOMEM);
5764
5765         t4_get_fcoe_stats(sc, 0, &stats[0]);
5766         t4_get_fcoe_stats(sc, 1, &stats[1]);
5767         t4_get_fcoe_stats(sc, 2, &stats[2]);
5768         t4_get_fcoe_stats(sc, 3, &stats[3]);
5769
5770         sbuf_printf(sb, "                   channel 0        channel 1        "
5771             "channel 2        channel 3\n");
5772         sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
5773             stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
5774             stats[3].octetsDDP);
5775         sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
5776             stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
5777         sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
5778             stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
5779             stats[3].framesDrop);
5780
5781         rc = sbuf_finish(sb);
5782         sbuf_delete(sb);
5783
5784         return (rc);
5785 }
5786
5787 static int
5788 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
5789 {
5790         struct adapter *sc = arg1;
5791         struct sbuf *sb;
5792         int rc, i;
5793         unsigned int map, kbps, ipg, mode;
5794         unsigned int pace_tab[NTX_SCHED];
5795
5796         rc = sysctl_wire_old_buffer(req, 0);
5797         if (rc != 0)
5798                 return (rc);
5799
5800         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5801         if (sb == NULL)
5802                 return (ENOMEM);
5803
5804         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
5805         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
5806         t4_read_pace_tbl(sc, pace_tab);
5807
5808         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
5809             "Class IPG (0.1 ns)   Flow IPG (us)");
5810
5811         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
5812                 t4_get_tx_sched(sc, i, &kbps, &ipg);
5813                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
5814                     (mode & (1 << i)) ? "flow" : "class", map & 3);
5815                 if (kbps)
5816                         sbuf_printf(sb, "%9u     ", kbps);
5817                 else
5818                         sbuf_printf(sb, " disabled     ");
5819
5820                 if (ipg)
5821                         sbuf_printf(sb, "%13u        ", ipg);
5822                 else
5823                         sbuf_printf(sb, "     disabled        ");
5824
5825                 if (pace_tab[i])
5826                         sbuf_printf(sb, "%10u", pace_tab[i]);
5827                 else
5828                         sbuf_printf(sb, "  disabled");
5829         }
5830
5831         rc = sbuf_finish(sb);
5832         sbuf_delete(sb);
5833
5834         return (rc);
5835 }
5836
5837 static int
5838 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
5839 {
5840         struct adapter *sc = arg1;
5841         struct sbuf *sb;
5842         int rc, i, j;
5843         uint64_t *p0, *p1;
5844         struct lb_port_stats s[2];
5845         static const char *stat_name[] = {
5846                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
5847                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
5848                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
5849                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
5850                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
5851                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
5852                 "BG2FramesTrunc:", "BG3FramesTrunc:"
5853         };
5854
5855         rc = sysctl_wire_old_buffer(req, 0);
5856         if (rc != 0)
5857                 return (rc);
5858
5859         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5860         if (sb == NULL)
5861                 return (ENOMEM);
5862
5863         memset(s, 0, sizeof(s));
5864
5865         for (i = 0; i < 4; i += 2) {
5866                 t4_get_lb_stats(sc, i, &s[0]);
5867                 t4_get_lb_stats(sc, i + 1, &s[1]);
5868
5869                 p0 = &s[0].octets;
5870                 p1 = &s[1].octets;
5871                 sbuf_printf(sb, "%s                       Loopback %u"
5872                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
5873
5874                 for (j = 0; j < nitems(stat_name); j++)
5875                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
5876                                    *p0++, *p1++);
5877         }
5878
5879         rc = sbuf_finish(sb);
5880         sbuf_delete(sb);
5881
5882         return (rc);
5883 }
5884
5885 static int
5886 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
5887 {
5888         int rc = 0;
5889         struct port_info *pi = arg1;
5890         struct sbuf *sb;
5891         static const char *linkdnreasons[] = {
5892                 "non-specific", "remote fault", "autoneg failed", "reserved3",
5893                 "PHY overheated", "unknown", "rx los", "reserved7"
5894         };
5895
5896         rc = sysctl_wire_old_buffer(req, 0);
5897         if (rc != 0)
5898                 return(rc);
5899         sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
5900         if (sb == NULL)
5901                 return (ENOMEM);
5902
5903         if (pi->linkdnrc < 0)
5904                 sbuf_printf(sb, "n/a");
5905         else if (pi->linkdnrc < nitems(linkdnreasons))
5906                 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
5907         else
5908                 sbuf_printf(sb, "%d", pi->linkdnrc);
5909
5910         rc = sbuf_finish(sb);
5911         sbuf_delete(sb);
5912
5913         return (rc);
5914 }
5915
5916 struct mem_desc {
5917         unsigned int base;
5918         unsigned int limit;
5919         unsigned int idx;
5920 };
5921
5922 static int
5923 mem_desc_cmp(const void *a, const void *b)
5924 {
5925         return ((const struct mem_desc *)a)->base -
5926                ((const struct mem_desc *)b)->base;
5927 }
5928
5929 static void
5930 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
5931     unsigned int to)
5932 {
5933         unsigned int size;
5934
5935         size = to - from + 1;
5936         if (size == 0)
5937                 return;
5938
5939         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
5940         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
5941 }
5942
5943 static int
5944 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
5945 {
5946         struct adapter *sc = arg1;
5947         struct sbuf *sb;
5948         int rc, i, n;
5949         uint32_t lo, hi, used, alloc;
5950         static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
5951         static const char *region[] = {
5952                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
5953                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
5954                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
5955                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
5956                 "RQUDP region:", "PBL region:", "TXPBL region:",
5957                 "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
5958                 "On-chip queues:"
5959         };
5960         struct mem_desc avail[4];
5961         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
5962         struct mem_desc *md = mem;
5963
5964         rc = sysctl_wire_old_buffer(req, 0);
5965         if (rc != 0)
5966                 return (rc);
5967
5968         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5969         if (sb == NULL)
5970                 return (ENOMEM);
5971
5972         for (i = 0; i < nitems(mem); i++) {
5973                 mem[i].limit = 0;
5974                 mem[i].idx = i;
5975         }
5976
5977         /* Find and sort the populated memory ranges */
5978         i = 0;
5979         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
5980         if (lo & F_EDRAM0_ENABLE) {
5981                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
5982                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
5983                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
5984                 avail[i].idx = 0;
5985                 i++;
5986         }
5987         if (lo & F_EDRAM1_ENABLE) {
5988                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
5989                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
5990                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
5991                 avail[i].idx = 1;
5992                 i++;
5993         }
5994         if (lo & F_EXT_MEM_ENABLE) {
5995                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
5996                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
5997                 avail[i].limit = avail[i].base +
5998                     (G_EXT_MEM_SIZE(hi) << 20);
5999                 avail[i].idx = is_t4(sc) ? 2 : 3;       /* Call it MC for T4 */
6000                 i++;
6001         }
6002         if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
6003                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
6004                 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
6005                 avail[i].limit = avail[i].base +
6006                     (G_EXT_MEM1_SIZE(hi) << 20);
6007                 avail[i].idx = 4;
6008                 i++;
6009         }
6010         if (!i)                                    /* no memory available */
6011                 return 0;
6012         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
6013
6014         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
6015         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
6016         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
6017         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
6018         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
6019         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
6020         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
6021         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
6022         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
6023
6024         /* the next few have explicit upper bounds */
6025         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
6026         md->limit = md->base - 1 +
6027                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
6028                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
6029         md++;
6030
6031         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
6032         md->limit = md->base - 1 +
6033                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
6034                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
6035         md++;
6036
6037         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6038                 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
6039                 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
6040                 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
6041         } else {
6042                 md->base = 0;
6043                 md->idx = nitems(region);  /* hide it */
6044         }
6045         md++;
6046
6047 #define ulp_region(reg) \
6048         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
6049         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
6050
6051         ulp_region(RX_ISCSI);
6052         ulp_region(RX_TDDP);
6053         ulp_region(TX_TPT);
6054         ulp_region(RX_STAG);
6055         ulp_region(RX_RQ);
6056         ulp_region(RX_RQUDP);
6057         ulp_region(RX_PBL);
6058         ulp_region(TX_PBL);
6059 #undef ulp_region
6060
6061         md->base = 0;
6062         md->idx = nitems(region);
6063         if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
6064                 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
6065                 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
6066                     A_SGE_DBVFIFO_SIZE))) << 2) - 1;
6067         }
6068         md++;
6069
6070         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
6071         md->limit = md->base + sc->tids.ntids - 1;
6072         md++;
6073         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
6074         md->limit = md->base + sc->tids.ntids - 1;
6075         md++;
6076
6077         md->base = sc->vres.ocq.start;
6078         if (sc->vres.ocq.size)
6079                 md->limit = md->base + sc->vres.ocq.size - 1;
6080         else
6081                 md->idx = nitems(region);  /* hide it */
6082         md++;
6083
6084         /* add any address-space holes, there can be up to 3 */
6085         for (n = 0; n < i - 1; n++)
6086                 if (avail[n].limit < avail[n + 1].base)
6087                         (md++)->base = avail[n].limit;
6088         if (avail[n].limit)
6089                 (md++)->base = avail[n].limit;
6090
6091         n = md - mem;
6092         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
6093
6094         for (lo = 0; lo < i; lo++)
6095                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
6096                                 avail[lo].limit - 1);
6097
6098         sbuf_printf(sb, "\n");
6099         for (i = 0; i < n; i++) {
6100                 if (mem[i].idx >= nitems(region))
6101                         continue;                        /* skip holes */
6102                 if (!mem[i].limit)
6103                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
6104                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
6105                                 mem[i].limit);
6106         }
6107
6108         sbuf_printf(sb, "\n");
6109         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
6110         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
6111         mem_region_show(sb, "uP RAM:", lo, hi);
6112
6113         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
6114         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
6115         mem_region_show(sb, "uP Extmem2:", lo, hi);
6116
6117         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
6118         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
6119                    G_PMRXMAXPAGE(lo),
6120                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
6121                    (lo & F_PMRXNUMCHN) ? 2 : 1);
6122
6123         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
6124         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
6125         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
6126                    G_PMTXMAXPAGE(lo),
6127                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
6128                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
6129         sbuf_printf(sb, "%u p-structs\n",
6130                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
6131
6132         for (i = 0; i < 4; i++) {
6133                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
6134                 if (is_t4(sc)) {
6135                         used = G_USED(lo);
6136                         alloc = G_ALLOC(lo);
6137                 } else {
6138                         used = G_T5_USED(lo);
6139                         alloc = G_T5_ALLOC(lo);
6140                 }
6141                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
6142                            i, used, alloc);
6143         }
6144         for (i = 0; i < 4; i++) {
6145                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
6146                 if (is_t4(sc)) {
6147                         used = G_USED(lo);
6148                         alloc = G_ALLOC(lo);
6149                 } else {
6150                         used = G_T5_USED(lo);
6151                         alloc = G_T5_ALLOC(lo);
6152                 }
6153                 sbuf_printf(sb,
6154                            "\nLoopback %d using %u pages out of %u allocated",
6155                            i, used, alloc);
6156         }
6157
6158         rc = sbuf_finish(sb);
6159         sbuf_delete(sb);
6160
6161         return (rc);
6162 }
6163
6164 static inline void
6165 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
6166 {
6167         *mask = x | y;
6168         y = htobe64(y);
6169         memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
6170 }
6171
6172 static int
6173 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
6174 {
6175         struct adapter *sc = arg1;
6176         struct sbuf *sb;
6177         int rc, i, n;
6178
6179         rc = sysctl_wire_old_buffer(req, 0);
6180         if (rc != 0)
6181                 return (rc);
6182
6183         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6184         if (sb == NULL)
6185                 return (ENOMEM);
6186
6187         sbuf_printf(sb,
6188             "Idx  Ethernet address     Mask     Vld Ports PF"
6189             "  VF              Replication             P0 P1 P2 P3  ML");
6190         n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
6191             NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
6192         for (i = 0; i < n; i++) {
6193                 uint64_t tcamx, tcamy, mask;
6194                 uint32_t cls_lo, cls_hi;
6195                 uint8_t addr[ETHER_ADDR_LEN];
6196
6197                 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
6198                 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
6199                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
6200                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
6201
6202                 if (tcamx & tcamy)
6203                         continue;
6204
6205                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
6206                 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
6207                            "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
6208                            addr[3], addr[4], addr[5], (uintmax_t)mask,
6209                            (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
6210                            G_PORTMAP(cls_hi), G_PF(cls_lo),
6211                            (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
6212
6213                 if (cls_lo & F_REPLICATE) {
6214                         struct fw_ldst_cmd ldst_cmd;
6215
6216                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
6217                         ldst_cmd.op_to_addrspace =
6218                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
6219                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
6220                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
6221                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
6222                         ldst_cmd.u.mps.fid_ctl =
6223                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
6224                                 V_FW_LDST_CMD_CTL(i));
6225
6226                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
6227                             "t4mps");
6228                         if (rc)
6229                                 break;
6230                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
6231                             sizeof(ldst_cmd), &ldst_cmd);
6232                         end_synchronized_op(sc, 0);
6233
6234                         if (rc != 0) {
6235                                 sbuf_printf(sb,
6236                                     " ------------ error %3u ------------", rc);
6237                                 rc = 0;
6238                         } else {
6239                                 sbuf_printf(sb, " %08x %08x %08x %08x",
6240                                     be32toh(ldst_cmd.u.mps.rplc127_96),
6241                                     be32toh(ldst_cmd.u.mps.rplc95_64),
6242                                     be32toh(ldst_cmd.u.mps.rplc63_32),
6243                                     be32toh(ldst_cmd.u.mps.rplc31_0));
6244                         }
6245                 } else
6246                         sbuf_printf(sb, "%36s", "");
6247
6248                 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
6249                     G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
6250                     G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
6251         }
6252
6253         if (rc)
6254                 (void) sbuf_finish(sb);
6255         else
6256                 rc = sbuf_finish(sb);
6257         sbuf_delete(sb);
6258
6259         return (rc);
6260 }
6261
6262 static int
6263 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
6264 {
6265         struct adapter *sc = arg1;
6266         struct sbuf *sb;
6267         int rc;
6268         uint16_t mtus[NMTUS];
6269
6270         rc = sysctl_wire_old_buffer(req, 0);
6271         if (rc != 0)
6272                 return (rc);
6273
6274         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6275         if (sb == NULL)
6276                 return (ENOMEM);
6277
6278         t4_read_mtu_tbl(sc, mtus, NULL);
6279
6280         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
6281             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
6282             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
6283             mtus[14], mtus[15]);
6284
6285         rc = sbuf_finish(sb);
6286         sbuf_delete(sb);
6287
6288         return (rc);
6289 }
6290
6291 static int
6292 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
6293 {
6294         struct adapter *sc = arg1;
6295         struct sbuf *sb;
6296         int rc, i;
6297         uint32_t cnt[PM_NSTATS];
6298         uint64_t cyc[PM_NSTATS];
6299         static const char *rx_stats[] = {
6300                 "Read:", "Write bypass:", "Write mem:", "Flush:"
6301         };
6302         static const char *tx_stats[] = {
6303                 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
6304         };
6305
6306         rc = sysctl_wire_old_buffer(req, 0);
6307         if (rc != 0)
6308                 return (rc);
6309
6310         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6311         if (sb == NULL)
6312                 return (ENOMEM);
6313
6314         t4_pmtx_get_stats(sc, cnt, cyc);
6315         sbuf_printf(sb, "                Tx pcmds             Tx bytes");
6316         for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
6317                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
6318                     cyc[i]);
6319
6320         t4_pmrx_get_stats(sc, cnt, cyc);
6321         sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
6322         for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
6323                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
6324                     cyc[i]);
6325
6326         rc = sbuf_finish(sb);
6327         sbuf_delete(sb);
6328
6329         return (rc);
6330 }
6331
6332 static int
6333 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
6334 {
6335         struct adapter *sc = arg1;
6336         struct sbuf *sb;
6337         int rc;
6338         struct tp_rdma_stats stats;
6339
6340         rc = sysctl_wire_old_buffer(req, 0);
6341         if (rc != 0)
6342                 return (rc);
6343
6344         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6345         if (sb == NULL)
6346                 return (ENOMEM);
6347
6348         t4_tp_get_rdma_stats(sc, &stats);
6349         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
6350         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
6351
6352         rc = sbuf_finish(sb);
6353         sbuf_delete(sb);
6354
6355         return (rc);
6356 }
6357
6358 static int
6359 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
6360 {
6361         struct adapter *sc = arg1;
6362         struct sbuf *sb;
6363         int rc;
6364         struct tp_tcp_stats v4, v6;
6365
6366         rc = sysctl_wire_old_buffer(req, 0);
6367         if (rc != 0)
6368                 return (rc);
6369
6370         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6371         if (sb == NULL)
6372                 return (ENOMEM);
6373
6374         t4_tp_get_tcp_stats(sc, &v4, &v6);
6375         sbuf_printf(sb,
6376             "                                IP                 IPv6\n");
6377         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
6378             v4.tcpOutRsts, v6.tcpOutRsts);
6379         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
6380             v4.tcpInSegs, v6.tcpInSegs);
6381         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
6382             v4.tcpOutSegs, v6.tcpOutSegs);
6383         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
6384             v4.tcpRetransSegs, v6.tcpRetransSegs);
6385
6386         rc = sbuf_finish(sb);
6387         sbuf_delete(sb);
6388
6389         return (rc);
6390 }
6391
6392 static int
6393 sysctl_tids(SYSCTL_HANDLER_ARGS)
6394 {
6395         struct adapter *sc = arg1;
6396         struct sbuf *sb;
6397         int rc;
6398         struct tid_info *t = &sc->tids;
6399
6400         rc = sysctl_wire_old_buffer(req, 0);
6401         if (rc != 0)
6402                 return (rc);
6403
6404         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6405         if (sb == NULL)
6406                 return (ENOMEM);
6407
6408         if (t->natids) {
6409                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
6410                     t->atids_in_use);
6411         }
6412
6413         if (t->ntids) {
6414                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6415                         uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
6416
6417                         if (b) {
6418                                 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
6419                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6420                                     t->ntids - 1);
6421                         } else {
6422                                 sbuf_printf(sb, "TID range: %u-%u",
6423                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6424                                     t->ntids - 1);
6425                         }
6426                 } else
6427                         sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
6428                 sbuf_printf(sb, ", in use: %u\n",
6429                     atomic_load_acq_int(&t->tids_in_use));
6430         }
6431
6432         if (t->nstids) {
6433                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
6434                     t->stid_base + t->nstids - 1, t->stids_in_use);
6435         }
6436
6437         if (t->nftids) {
6438                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
6439                     t->ftid_base + t->nftids - 1);
6440         }
6441
6442         if (t->netids) {
6443                 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
6444                     t->etid_base + t->netids - 1);
6445         }
6446
6447         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
6448             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
6449             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
6450
6451         rc = sbuf_finish(sb);
6452         sbuf_delete(sb);
6453
6454         return (rc);
6455 }
6456
6457 static int
6458 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
6459 {
6460         struct adapter *sc = arg1;
6461         struct sbuf *sb;
6462         int rc;
6463         struct tp_err_stats stats;
6464
6465         rc = sysctl_wire_old_buffer(req, 0);
6466         if (rc != 0)
6467                 return (rc);
6468
6469         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6470         if (sb == NULL)
6471                 return (ENOMEM);
6472
6473         t4_tp_get_err_stats(sc, &stats);
6474
6475         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
6476                       "channel 3\n");
6477         sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
6478             stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
6479             stats.macInErrs[3]);
6480         sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
6481             stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
6482             stats.hdrInErrs[3]);
6483         sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
6484             stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
6485             stats.tcpInErrs[3]);
6486         sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
6487             stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
6488             stats.tcp6InErrs[3]);
6489         sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
6490             stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
6491             stats.tnlCongDrops[3]);
6492         sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
6493             stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
6494             stats.tnlTxDrops[3]);
6495         sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
6496             stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
6497             stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
6498         sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
6499             stats.ofldChanDrops[0], stats.ofldChanDrops[1],
6500             stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
6501         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
6502             stats.ofldNoNeigh, stats.ofldCongDefer);
6503
6504         rc = sbuf_finish(sb);
6505         sbuf_delete(sb);
6506
6507         return (rc);
6508 }
6509
6510 struct field_desc {
6511         const char *name;
6512         u_int start;
6513         u_int width;
6514 };
6515
6516 static void
6517 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
6518 {
6519         char buf[32];
6520         int line_size = 0;
6521
6522         while (f->name) {
6523                 uint64_t mask = (1ULL << f->width) - 1;
6524                 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
6525                     ((uintmax_t)v >> f->start) & mask);
6526
6527                 if (line_size + len >= 79) {
6528                         line_size = 8;
6529                         sbuf_printf(sb, "\n        ");
6530                 }
6531                 sbuf_printf(sb, "%s ", buf);
6532                 line_size += len + 1;
6533                 f++;
6534         }
6535         sbuf_printf(sb, "\n");
6536 }
6537
6538 static struct field_desc tp_la0[] = {
6539         { "RcfOpCodeOut", 60, 4 },
6540         { "State", 56, 4 },
6541         { "WcfState", 52, 4 },
6542         { "RcfOpcSrcOut", 50, 2 },
6543         { "CRxError", 49, 1 },
6544         { "ERxError", 48, 1 },
6545         { "SanityFailed", 47, 1 },
6546         { "SpuriousMsg", 46, 1 },
6547         { "FlushInputMsg", 45, 1 },
6548         { "FlushInputCpl", 44, 1 },
6549         { "RssUpBit", 43, 1 },
6550         { "RssFilterHit", 42, 1 },
6551         { "Tid", 32, 10 },
6552         { "InitTcb", 31, 1 },
6553         { "LineNumber", 24, 7 },
6554         { "Emsg", 23, 1 },
6555         { "EdataOut", 22, 1 },
6556         { "Cmsg", 21, 1 },
6557         { "CdataOut", 20, 1 },
6558         { "EreadPdu", 19, 1 },
6559         { "CreadPdu", 18, 1 },
6560         { "TunnelPkt", 17, 1 },
6561         { "RcfPeerFin", 16, 1 },
6562         { "RcfReasonOut", 12, 4 },
6563         { "TxCchannel", 10, 2 },
6564         { "RcfTxChannel", 8, 2 },
6565         { "RxEchannel", 6, 2 },
6566         { "RcfRxChannel", 5, 1 },
6567         { "RcfDataOutSrdy", 4, 1 },
6568         { "RxDvld", 3, 1 },
6569         { "RxOoDvld", 2, 1 },
6570         { "RxCongestion", 1, 1 },
6571         { "TxCongestion", 0, 1 },
6572         { NULL }
6573 };
6574
6575 static struct field_desc tp_la1[] = {
6576         { "CplCmdIn", 56, 8 },
6577         { "CplCmdOut", 48, 8 },
6578         { "ESynOut", 47, 1 },
6579         { "EAckOut", 46, 1 },
6580         { "EFinOut", 45, 1 },
6581         { "ERstOut", 44, 1 },
6582         { "SynIn", 43, 1 },
6583         { "AckIn", 42, 1 },
6584         { "FinIn", 41, 1 },
6585         { "RstIn", 40, 1 },
6586         { "DataIn", 39, 1 },
6587         { "DataInVld", 38, 1 },
6588         { "PadIn", 37, 1 },
6589         { "RxBufEmpty", 36, 1 },
6590         { "RxDdp", 35, 1 },
6591         { "RxFbCongestion", 34, 1 },
6592         { "TxFbCongestion", 33, 1 },
6593         { "TxPktSumSrdy", 32, 1 },
6594         { "RcfUlpType", 28, 4 },
6595         { "Eread", 27, 1 },
6596         { "Ebypass", 26, 1 },
6597         { "Esave", 25, 1 },
6598         { "Static0", 24, 1 },
6599         { "Cread", 23, 1 },
6600         { "Cbypass", 22, 1 },
6601         { "Csave", 21, 1 },
6602         { "CPktOut", 20, 1 },
6603         { "RxPagePoolFull", 18, 2 },
6604         { "RxLpbkPkt", 17, 1 },
6605         { "TxLpbkPkt", 16, 1 },
6606         { "RxVfValid", 15, 1 },
6607         { "SynLearned", 14, 1 },
6608         { "SetDelEntry", 13, 1 },
6609         { "SetInvEntry", 12, 1 },
6610         { "CpcmdDvld", 11, 1 },
6611         { "CpcmdSave", 10, 1 },
6612         { "RxPstructsFull", 8, 2 },
6613         { "EpcmdDvld", 7, 1 },
6614         { "EpcmdFlush", 6, 1 },
6615         { "EpcmdTrimPrefix", 5, 1 },
6616         { "EpcmdTrimPostfix", 4, 1 },
6617         { "ERssIp4Pkt", 3, 1 },
6618         { "ERssIp6Pkt", 2, 1 },
6619         { "ERssTcpUdpPkt", 1, 1 },
6620         { "ERssFceFipPkt", 0, 1 },
6621         { NULL }
6622 };
6623
6624 static struct field_desc tp_la2[] = {
6625         { "CplCmdIn", 56, 8 },
6626         { "MpsVfVld", 55, 1 },
6627         { "MpsPf", 52, 3 },
6628         { "MpsVf", 44, 8 },
6629         { "SynIn", 43, 1 },
6630         { "AckIn", 42, 1 },
6631         { "FinIn", 41, 1 },
6632         { "RstIn", 40, 1 },
6633         { "DataIn", 39, 1 },
6634         { "DataInVld", 38, 1 },
6635         { "PadIn", 37, 1 },
6636         { "RxBufEmpty", 36, 1 },
6637         { "RxDdp", 35, 1 },
6638         { "RxFbCongestion", 34, 1 },
6639         { "TxFbCongestion", 33, 1 },
6640         { "TxPktSumSrdy", 32, 1 },
6641         { "RcfUlpType", 28, 4 },
6642         { "Eread", 27, 1 },
6643         { "Ebypass", 26, 1 },
6644         { "Esave", 25, 1 },
6645         { "Static0", 24, 1 },
6646         { "Cread", 23, 1 },
6647         { "Cbypass", 22, 1 },
6648         { "Csave", 21, 1 },
6649         { "CPktOut", 20, 1 },
6650         { "RxPagePoolFull", 18, 2 },
6651         { "RxLpbkPkt", 17, 1 },
6652         { "TxLpbkPkt", 16, 1 },
6653         { "RxVfValid", 15, 1 },
6654         { "SynLearned", 14, 1 },
6655         { "SetDelEntry", 13, 1 },
6656         { "SetInvEntry", 12, 1 },
6657         { "CpcmdDvld", 11, 1 },
6658         { "CpcmdSave", 10, 1 },
6659         { "RxPstructsFull", 8, 2 },
6660         { "EpcmdDvld", 7, 1 },
6661         { "EpcmdFlush", 6, 1 },
6662         { "EpcmdTrimPrefix", 5, 1 },
6663         { "EpcmdTrimPostfix", 4, 1 },
6664         { "ERssIp4Pkt", 3, 1 },
6665         { "ERssIp6Pkt", 2, 1 },
6666         { "ERssTcpUdpPkt", 1, 1 },
6667         { "ERssFceFipPkt", 0, 1 },
6668         { NULL }
6669 };
6670
6671 static void
6672 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
6673 {
6674
6675         field_desc_show(sb, *p, tp_la0);
6676 }
6677
6678 static void
6679 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
6680 {
6681
6682         if (idx)
6683                 sbuf_printf(sb, "\n");
6684         field_desc_show(sb, p[0], tp_la0);
6685         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6686                 field_desc_show(sb, p[1], tp_la0);
6687 }
6688
6689 static void
6690 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
6691 {
6692
6693         if (idx)
6694                 sbuf_printf(sb, "\n");
6695         field_desc_show(sb, p[0], tp_la0);
6696         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6697                 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
6698 }
6699
6700 static int
6701 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
6702 {
6703         struct adapter *sc = arg1;
6704         struct sbuf *sb;
6705         uint64_t *buf, *p;
6706         int rc;
6707         u_int i, inc;
6708         void (*show_func)(struct sbuf *, uint64_t *, int);
6709
6710         rc = sysctl_wire_old_buffer(req, 0);
6711         if (rc != 0)
6712                 return (rc);
6713
6714         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6715         if (sb == NULL)
6716                 return (ENOMEM);
6717
6718         buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
6719
6720         t4_tp_read_la(sc, buf, NULL);
6721         p = buf;
6722
6723         switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
6724         case 2:
6725                 inc = 2;
6726                 show_func = tp_la_show2;
6727                 break;
6728         case 3:
6729                 inc = 2;
6730                 show_func = tp_la_show3;
6731                 break;
6732         default:
6733                 inc = 1;
6734                 show_func = tp_la_show;
6735         }
6736
6737         for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
6738                 (*show_func)(sb, p, i);
6739
6740         rc = sbuf_finish(sb);
6741         sbuf_delete(sb);
6742         free(buf, M_CXGBE);
6743         return (rc);
6744 }
6745
6746 static int
6747 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
6748 {
6749         struct adapter *sc = arg1;
6750         struct sbuf *sb;
6751         int rc;
6752         u64 nrate[NCHAN], orate[NCHAN];
6753
6754         rc = sysctl_wire_old_buffer(req, 0);
6755         if (rc != 0)
6756                 return (rc);
6757
6758         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6759         if (sb == NULL)
6760                 return (ENOMEM);
6761
6762         t4_get_chan_txrate(sc, nrate, orate);
6763         sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
6764                  "channel 3\n");
6765         sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
6766             nrate[0], nrate[1], nrate[2], nrate[3]);
6767         sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
6768             orate[0], orate[1], orate[2], orate[3]);
6769
6770         rc = sbuf_finish(sb);
6771         sbuf_delete(sb);
6772
6773         return (rc);
6774 }
6775
6776 static int
6777 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
6778 {
6779         struct adapter *sc = arg1;
6780         struct sbuf *sb;
6781         uint32_t *buf, *p;
6782         int rc, i;
6783
6784         rc = sysctl_wire_old_buffer(req, 0);
6785         if (rc != 0)
6786                 return (rc);
6787
6788         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6789         if (sb == NULL)
6790                 return (ENOMEM);
6791
6792         buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
6793             M_ZERO | M_WAITOK);
6794
6795         t4_ulprx_read_la(sc, buf);
6796         p = buf;
6797
6798         sbuf_printf(sb, "      Pcmd        Type   Message"
6799             "                Data");
6800         for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
6801                 sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
6802                     p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
6803         }
6804
6805         rc = sbuf_finish(sb);
6806         sbuf_delete(sb);
6807         free(buf, M_CXGBE);
6808         return (rc);
6809 }
6810
6811 static int
6812 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
6813 {
6814         struct adapter *sc = arg1;
6815         struct sbuf *sb;
6816         int rc, v;
6817
6818         rc = sysctl_wire_old_buffer(req, 0);
6819         if (rc != 0)
6820                 return (rc);
6821
6822         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6823         if (sb == NULL)
6824                 return (ENOMEM);
6825
6826         v = t4_read_reg(sc, A_SGE_STAT_CFG);
6827         if (G_STATSOURCE_T5(v) == 7) {
6828                 if (G_STATMODE(v) == 0) {
6829                         sbuf_printf(sb, "total %d, incomplete %d",
6830                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6831                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6832                 } else if (G_STATMODE(v) == 1) {
6833                         sbuf_printf(sb, "total %d, data overflow %d",
6834                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6835                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6836                 }
6837         }
6838         rc = sbuf_finish(sb);
6839         sbuf_delete(sb);
6840
6841         return (rc);
6842 }
6843 #endif
6844
6845 static inline void
6846 txq_start(struct ifnet *ifp, struct sge_txq *txq)
6847 {
6848         struct buf_ring *br;
6849         struct mbuf *m;
6850
6851         TXQ_LOCK_ASSERT_OWNED(txq);
6852
6853         br = txq->br;
6854         m = txq->m ? txq->m : drbr_dequeue(ifp, br);
6855         if (m)
6856                 t4_eth_tx(ifp, txq, m);
6857 }
6858
6859 void
6860 t4_tx_callout(void *arg)
6861 {
6862         struct sge_eq *eq = arg;
6863         struct adapter *sc;
6864
6865         if (EQ_TRYLOCK(eq) == 0)
6866                 goto reschedule;
6867
6868         if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) {
6869                 EQ_UNLOCK(eq);
6870 reschedule:
6871                 if (__predict_true(!(eq->flags && EQ_DOOMED)))
6872                         callout_schedule(&eq->tx_callout, 1);
6873                 return;
6874         }
6875
6876         EQ_LOCK_ASSERT_OWNED(eq);
6877
6878         if (__predict_true((eq->flags & EQ_DOOMED) == 0)) {
6879
6880                 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
6881                         struct sge_txq *txq = arg;
6882                         struct port_info *pi = txq->ifp->if_softc;
6883
6884                         sc = pi->adapter;
6885                 } else {
6886                         struct sge_wrq *wrq = arg;
6887
6888                         sc = wrq->adapter;
6889                 }
6890
6891                 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task);
6892         }
6893
6894         EQ_UNLOCK(eq);
6895 }
6896
6897 void
6898 t4_tx_task(void *arg, int count)
6899 {
6900         struct sge_eq *eq = arg;
6901
6902         EQ_LOCK(eq);
6903         if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
6904                 struct sge_txq *txq = arg;
6905                 txq_start(txq->ifp, txq);
6906         } else {
6907                 struct sge_wrq *wrq = arg;
6908                 t4_wrq_tx_locked(wrq->adapter, wrq, NULL);
6909         }
6910         EQ_UNLOCK(eq);
6911 }
6912
6913 static uint32_t
6914 fconf_to_mode(uint32_t fconf)
6915 {
6916         uint32_t mode;
6917
6918         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
6919             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
6920
6921         if (fconf & F_FRAGMENTATION)
6922                 mode |= T4_FILTER_IP_FRAGMENT;
6923
6924         if (fconf & F_MPSHITTYPE)
6925                 mode |= T4_FILTER_MPS_HIT_TYPE;
6926
6927         if (fconf & F_MACMATCH)
6928                 mode |= T4_FILTER_MAC_IDX;
6929
6930         if (fconf & F_ETHERTYPE)
6931                 mode |= T4_FILTER_ETH_TYPE;
6932
6933         if (fconf & F_PROTOCOL)
6934                 mode |= T4_FILTER_IP_PROTO;
6935
6936         if (fconf & F_TOS)
6937                 mode |= T4_FILTER_IP_TOS;
6938
6939         if (fconf & F_VLAN)
6940                 mode |= T4_FILTER_VLAN;
6941
6942         if (fconf & F_VNIC_ID)
6943                 mode |= T4_FILTER_VNIC;
6944
6945         if (fconf & F_PORT)
6946                 mode |= T4_FILTER_PORT;
6947
6948         if (fconf & F_FCOE)
6949                 mode |= T4_FILTER_FCoE;
6950
6951         return (mode);
6952 }
6953
6954 static uint32_t
6955 mode_to_fconf(uint32_t mode)
6956 {
6957         uint32_t fconf = 0;
6958
6959         if (mode & T4_FILTER_IP_FRAGMENT)
6960                 fconf |= F_FRAGMENTATION;
6961
6962         if (mode & T4_FILTER_MPS_HIT_TYPE)
6963                 fconf |= F_MPSHITTYPE;
6964
6965         if (mode & T4_FILTER_MAC_IDX)
6966                 fconf |= F_MACMATCH;
6967
6968         if (mode & T4_FILTER_ETH_TYPE)
6969                 fconf |= F_ETHERTYPE;
6970
6971         if (mode & T4_FILTER_IP_PROTO)
6972                 fconf |= F_PROTOCOL;
6973
6974         if (mode & T4_FILTER_IP_TOS)
6975                 fconf |= F_TOS;
6976
6977         if (mode & T4_FILTER_VLAN)
6978                 fconf |= F_VLAN;
6979
6980         if (mode & T4_FILTER_VNIC)
6981                 fconf |= F_VNIC_ID;
6982
6983         if (mode & T4_FILTER_PORT)
6984                 fconf |= F_PORT;
6985
6986         if (mode & T4_FILTER_FCoE)
6987                 fconf |= F_FCOE;
6988
6989         return (fconf);
6990 }
6991
6992 static uint32_t
6993 fspec_to_fconf(struct t4_filter_specification *fs)
6994 {
6995         uint32_t fconf = 0;
6996
6997         if (fs->val.frag || fs->mask.frag)
6998                 fconf |= F_FRAGMENTATION;
6999
7000         if (fs->val.matchtype || fs->mask.matchtype)
7001                 fconf |= F_MPSHITTYPE;
7002
7003         if (fs->val.macidx || fs->mask.macidx)
7004                 fconf |= F_MACMATCH;
7005
7006         if (fs->val.ethtype || fs->mask.ethtype)
7007                 fconf |= F_ETHERTYPE;
7008
7009         if (fs->val.proto || fs->mask.proto)
7010                 fconf |= F_PROTOCOL;
7011
7012         if (fs->val.tos || fs->mask.tos)
7013                 fconf |= F_TOS;
7014
7015         if (fs->val.vlan_vld || fs->mask.vlan_vld)
7016                 fconf |= F_VLAN;
7017
7018         if (fs->val.vnic_vld || fs->mask.vnic_vld)
7019                 fconf |= F_VNIC_ID;
7020
7021         if (fs->val.iport || fs->mask.iport)
7022                 fconf |= F_PORT;
7023
7024         if (fs->val.fcoe || fs->mask.fcoe)
7025                 fconf |= F_FCOE;
7026
7027         return (fconf);
7028 }
7029
7030 static int
7031 get_filter_mode(struct adapter *sc, uint32_t *mode)
7032 {
7033         int rc;
7034         uint32_t fconf;
7035
7036         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7037             "t4getfm");
7038         if (rc)
7039                 return (rc);
7040
7041         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
7042             A_TP_VLAN_PRI_MAP);
7043
7044         if (sc->params.tp.vlan_pri_map != fconf) {
7045                 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
7046                     device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
7047                     fconf);
7048                 sc->params.tp.vlan_pri_map = fconf;
7049         }
7050
7051         *mode = fconf_to_mode(sc->params.tp.vlan_pri_map);
7052
7053         end_synchronized_op(sc, LOCK_HELD);
7054         return (0);
7055 }
7056
7057 static int
7058 set_filter_mode(struct adapter *sc, uint32_t mode)
7059 {
7060         uint32_t fconf;
7061         int rc;
7062
7063         fconf = mode_to_fconf(mode);
7064
7065         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7066             "t4setfm");
7067         if (rc)
7068                 return (rc);
7069
7070         if (sc->tids.ftids_in_use > 0) {
7071                 rc = EBUSY;
7072                 goto done;
7073         }
7074
7075 #ifdef TCP_OFFLOAD
7076         if (sc->offload_map) {
7077                 rc = EBUSY;
7078                 goto done;
7079         }
7080 #endif
7081
7082 #ifdef notyet
7083         rc = -t4_set_filter_mode(sc, fconf);
7084         if (rc == 0)
7085                 sc->filter_mode = fconf;
7086 #else
7087         rc = ENOTSUP;
7088 #endif
7089
7090 done:
7091         end_synchronized_op(sc, LOCK_HELD);
7092         return (rc);
7093 }
7094
7095 static inline uint64_t
7096 get_filter_hits(struct adapter *sc, uint32_t fid)
7097 {
7098         uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
7099         uint64_t hits;
7100
7101         memwin_info(sc, 0, &mw_base, NULL);
7102         off = position_memwin(sc, 0,
7103             tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
7104         if (is_t4(sc)) {
7105                 hits = t4_read_reg64(sc, mw_base + off + 16);
7106                 hits = be64toh(hits);
7107         } else {
7108                 hits = t4_read_reg(sc, mw_base + off + 24);
7109                 hits = be32toh(hits);
7110         }
7111
7112         return (hits);
7113 }
7114
7115 static int
7116 get_filter(struct adapter *sc, struct t4_filter *t)
7117 {
7118         int i, rc, nfilters = sc->tids.nftids;
7119         struct filter_entry *f;
7120
7121         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7122             "t4getf");
7123         if (rc)
7124                 return (rc);
7125
7126         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
7127             t->idx >= nfilters) {
7128                 t->idx = 0xffffffff;
7129                 goto done;
7130         }
7131
7132         f = &sc->tids.ftid_tab[t->idx];
7133         for (i = t->idx; i < nfilters; i++, f++) {
7134                 if (f->valid) {
7135                         t->idx = i;
7136                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
7137                         t->smtidx = f->smtidx;
7138                         if (f->fs.hitcnts)
7139                                 t->hits = get_filter_hits(sc, t->idx);
7140                         else
7141                                 t->hits = UINT64_MAX;
7142                         t->fs = f->fs;
7143
7144                         goto done;
7145                 }
7146         }
7147
7148         t->idx = 0xffffffff;
7149 done:
7150         end_synchronized_op(sc, LOCK_HELD);
7151         return (0);
7152 }
7153
7154 static int
7155 set_filter(struct adapter *sc, struct t4_filter *t)
7156 {
7157         unsigned int nfilters, nports;
7158         struct filter_entry *f;
7159         int i, rc;
7160
7161         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
7162         if (rc)
7163                 return (rc);
7164
7165         nfilters = sc->tids.nftids;
7166         nports = sc->params.nports;
7167
7168         if (nfilters == 0) {
7169                 rc = ENOTSUP;
7170                 goto done;
7171         }
7172
7173         if (!(sc->flags & FULL_INIT_DONE)) {
7174                 rc = EAGAIN;
7175                 goto done;
7176         }
7177
7178         if (t->idx >= nfilters) {
7179                 rc = EINVAL;
7180                 goto done;
7181         }
7182
7183         /* Validate against the global filter mode */
7184         if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
7185             sc->params.tp.vlan_pri_map) {
7186                 rc = E2BIG;
7187                 goto done;
7188         }
7189
7190         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
7191                 rc = EINVAL;
7192                 goto done;
7193         }
7194
7195         if (t->fs.val.iport >= nports) {
7196                 rc = EINVAL;
7197                 goto done;
7198         }
7199
7200         /* Can't specify an iq if not steering to it */
7201         if (!t->fs.dirsteer && t->fs.iq) {
7202                 rc = EINVAL;
7203                 goto done;
7204         }
7205
7206         /* IPv6 filter idx must be 4 aligned */
7207         if (t->fs.type == 1 &&
7208             ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
7209                 rc = EINVAL;
7210                 goto done;
7211         }
7212
7213         if (sc->tids.ftid_tab == NULL) {
7214                 KASSERT(sc->tids.ftids_in_use == 0,
7215                     ("%s: no memory allocated but filters_in_use > 0",
7216                     __func__));
7217
7218                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
7219                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
7220                 if (sc->tids.ftid_tab == NULL) {
7221                         rc = ENOMEM;
7222                         goto done;
7223                 }
7224                 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
7225         }
7226
7227         for (i = 0; i < 4; i++) {
7228                 f = &sc->tids.ftid_tab[t->idx + i];
7229
7230                 if (f->pending || f->valid) {
7231                         rc = EBUSY;
7232                         goto done;
7233                 }
7234                 if (f->locked) {
7235                         rc = EPERM;
7236                         goto done;
7237                 }
7238
7239                 if (t->fs.type == 0)
7240                         break;
7241         }
7242
7243         f = &sc->tids.ftid_tab[t->idx];
7244         f->fs = t->fs;
7245
7246         rc = set_filter_wr(sc, t->idx);
7247 done:
7248         end_synchronized_op(sc, 0);
7249
7250         if (rc == 0) {
7251                 mtx_lock(&sc->tids.ftid_lock);
7252                 for (;;) {
7253                         if (f->pending == 0) {
7254                                 rc = f->valid ? 0 : EIO;
7255                                 break;
7256                         }
7257
7258                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7259                             PCATCH, "t4setfw", 0)) {
7260                                 rc = EINPROGRESS;
7261                                 break;
7262                         }
7263                 }
7264                 mtx_unlock(&sc->tids.ftid_lock);
7265         }
7266         return (rc);
7267 }
7268
7269 static int
7270 del_filter(struct adapter *sc, struct t4_filter *t)
7271 {
7272         unsigned int nfilters;
7273         struct filter_entry *f;
7274         int rc;
7275
7276         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
7277         if (rc)
7278                 return (rc);
7279
7280         nfilters = sc->tids.nftids;
7281
7282         if (nfilters == 0) {
7283                 rc = ENOTSUP;
7284                 goto done;
7285         }
7286
7287         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
7288             t->idx >= nfilters) {
7289                 rc = EINVAL;
7290                 goto done;
7291         }
7292
7293         if (!(sc->flags & FULL_INIT_DONE)) {
7294                 rc = EAGAIN;
7295                 goto done;
7296         }
7297
7298         f = &sc->tids.ftid_tab[t->idx];
7299
7300         if (f->pending) {
7301                 rc = EBUSY;
7302                 goto done;
7303         }
7304         if (f->locked) {
7305                 rc = EPERM;
7306                 goto done;
7307         }
7308
7309         if (f->valid) {
7310                 t->fs = f->fs;  /* extra info for the caller */
7311                 rc = del_filter_wr(sc, t->idx);
7312         }
7313
7314 done:
7315         end_synchronized_op(sc, 0);
7316
7317         if (rc == 0) {
7318                 mtx_lock(&sc->tids.ftid_lock);
7319                 for (;;) {
7320                         if (f->pending == 0) {
7321                                 rc = f->valid ? EIO : 0;
7322                                 break;
7323                         }
7324
7325                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7326                             PCATCH, "t4delfw", 0)) {
7327                                 rc = EINPROGRESS;
7328                                 break;
7329                         }
7330                 }
7331                 mtx_unlock(&sc->tids.ftid_lock);
7332         }
7333
7334         return (rc);
7335 }
7336
7337 static void
7338 clear_filter(struct filter_entry *f)
7339 {
7340         if (f->l2t)
7341                 t4_l2t_release(f->l2t);
7342
7343         bzero(f, sizeof (*f));
7344 }
7345
7346 static int
7347 set_filter_wr(struct adapter *sc, int fidx)
7348 {
7349         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7350         struct wrqe *wr;
7351         struct fw_filter_wr *fwr;
7352         unsigned int ftid;
7353
7354         ASSERT_SYNCHRONIZED_OP(sc);
7355
7356         if (f->fs.newdmac || f->fs.newvlan) {
7357                 /* This filter needs an L2T entry; allocate one. */
7358                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
7359                 if (f->l2t == NULL)
7360                         return (EAGAIN);
7361                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
7362                     f->fs.dmac)) {
7363                         t4_l2t_release(f->l2t);
7364                         f->l2t = NULL;
7365                         return (ENOMEM);
7366                 }
7367         }
7368
7369         ftid = sc->tids.ftid_base + fidx;
7370
7371         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7372         if (wr == NULL)
7373                 return (ENOMEM);
7374
7375         fwr = wrtod(wr);
7376         bzero(fwr, sizeof (*fwr));
7377
7378         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
7379         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
7380         fwr->tid_to_iq =
7381             htobe32(V_FW_FILTER_WR_TID(ftid) |
7382                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
7383                 V_FW_FILTER_WR_NOREPLY(0) |
7384                 V_FW_FILTER_WR_IQ(f->fs.iq));
7385         fwr->del_filter_to_l2tix =
7386             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
7387                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
7388                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
7389                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
7390                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
7391                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
7392                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
7393                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
7394                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
7395                     f->fs.newvlan == VLAN_REWRITE) |
7396                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
7397                     f->fs.newvlan == VLAN_REWRITE) |
7398                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
7399                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
7400                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
7401                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
7402         fwr->ethtype = htobe16(f->fs.val.ethtype);
7403         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
7404         fwr->frag_to_ovlan_vldm =
7405             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
7406                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
7407                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
7408                 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
7409                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
7410                 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
7411         fwr->smac_sel = 0;
7412         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
7413             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
7414         fwr->maci_to_matchtypem =
7415             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
7416                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
7417                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
7418                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
7419                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
7420                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
7421                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
7422                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
7423         fwr->ptcl = f->fs.val.proto;
7424         fwr->ptclm = f->fs.mask.proto;
7425         fwr->ttyp = f->fs.val.tos;
7426         fwr->ttypm = f->fs.mask.tos;
7427         fwr->ivlan = htobe16(f->fs.val.vlan);
7428         fwr->ivlanm = htobe16(f->fs.mask.vlan);
7429         fwr->ovlan = htobe16(f->fs.val.vnic);
7430         fwr->ovlanm = htobe16(f->fs.mask.vnic);
7431         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
7432         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
7433         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
7434         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
7435         fwr->lp = htobe16(f->fs.val.dport);
7436         fwr->lpm = htobe16(f->fs.mask.dport);
7437         fwr->fp = htobe16(f->fs.val.sport);
7438         fwr->fpm = htobe16(f->fs.mask.sport);
7439         if (f->fs.newsmac)
7440                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
7441
7442         f->pending = 1;
7443         sc->tids.ftids_in_use++;
7444
7445         t4_wrq_tx(sc, wr);
7446         return (0);
7447 }
7448
7449 static int
7450 del_filter_wr(struct adapter *sc, int fidx)
7451 {
7452         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7453         struct wrqe *wr;
7454         struct fw_filter_wr *fwr;
7455         unsigned int ftid;
7456
7457         ftid = sc->tids.ftid_base + fidx;
7458
7459         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7460         if (wr == NULL)
7461                 return (ENOMEM);
7462         fwr = wrtod(wr);
7463         bzero(fwr, sizeof (*fwr));
7464
7465         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
7466
7467         f->pending = 1;
7468         t4_wrq_tx(sc, wr);
7469         return (0);
7470 }
7471
7472 int
7473 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
7474 {
7475         struct adapter *sc = iq->adapter;
7476         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
7477         unsigned int idx = GET_TID(rpl);
7478         unsigned int rc;
7479         struct filter_entry *f;
7480
7481         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
7482             rss->opcode));
7483
7484         if (is_ftid(sc, idx)) {
7485
7486                 idx -= sc->tids.ftid_base;
7487                 f = &sc->tids.ftid_tab[idx];
7488                 rc = G_COOKIE(rpl->cookie);
7489
7490                 mtx_lock(&sc->tids.ftid_lock);
7491                 if (rc == FW_FILTER_WR_FLT_ADDED) {
7492                         KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
7493                             __func__, idx));
7494                         f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
7495                         f->pending = 0;  /* asynchronous setup completed */
7496                         f->valid = 1;
7497                 } else {
7498                         if (rc != FW_FILTER_WR_FLT_DELETED) {
7499                                 /* Add or delete failed, display an error */
7500                                 log(LOG_ERR,
7501                                     "filter %u setup failed with error %u\n",
7502                                     idx, rc);
7503                         }
7504
7505                         clear_filter(f);
7506                         sc->tids.ftids_in_use--;
7507                 }
7508                 wakeup(&sc->tids.ftid_tab);
7509                 mtx_unlock(&sc->tids.ftid_lock);
7510         }
7511
7512         return (0);
7513 }
7514
7515 static int
7516 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
7517 {
7518         int rc;
7519
7520         if (cntxt->cid > M_CTXTQID)
7521                 return (EINVAL);
7522
7523         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
7524             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
7525                 return (EINVAL);
7526
7527         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
7528         if (rc)
7529                 return (rc);
7530
7531         if (sc->flags & FW_OK) {
7532                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
7533                     &cntxt->data[0]);
7534                 if (rc == 0)
7535                         goto done;
7536         }
7537
7538         /*
7539          * Read via firmware failed or wasn't even attempted.  Read directly via
7540          * the backdoor.
7541          */
7542         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
7543 done:
7544         end_synchronized_op(sc, 0);
7545         return (rc);
7546 }
7547
7548 static int
7549 load_fw(struct adapter *sc, struct t4_data *fw)
7550 {
7551         int rc;
7552         uint8_t *fw_data;
7553
7554         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
7555         if (rc)
7556                 return (rc);
7557
7558         if (sc->flags & FULL_INIT_DONE) {
7559                 rc = EBUSY;
7560                 goto done;
7561         }
7562
7563         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
7564         if (fw_data == NULL) {
7565                 rc = ENOMEM;
7566                 goto done;
7567         }
7568
7569         rc = copyin(fw->data, fw_data, fw->len);
7570         if (rc == 0)
7571                 rc = -t4_load_fw(sc, fw_data, fw->len);
7572
7573         free(fw_data, M_CXGBE);
7574 done:
7575         end_synchronized_op(sc, 0);
7576         return (rc);
7577 }
7578
7579 static int
7580 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
7581 {
7582         uint32_t addr, off, remaining, i, n;
7583         uint32_t *buf, *b;
7584         uint32_t mw_base, mw_aperture;
7585         int rc;
7586         uint8_t *dst;
7587
7588         rc = validate_mem_range(sc, mr->addr, mr->len);
7589         if (rc != 0)
7590                 return (rc);
7591
7592         memwin_info(sc, win, &mw_base, &mw_aperture);
7593         buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
7594         addr = mr->addr;
7595         remaining = mr->len;
7596         dst = (void *)mr->data;
7597
7598         while (remaining) {
7599                 off = position_memwin(sc, win, addr);
7600
7601                 /* number of bytes that we'll copy in the inner loop */
7602                 n = min(remaining, mw_aperture - off);
7603                 for (i = 0; i < n; i += 4)
7604                         *b++ = t4_read_reg(sc, mw_base + off + i);
7605
7606                 rc = copyout(buf, dst, n);
7607                 if (rc != 0)
7608                         break;
7609
7610                 b = buf;
7611                 dst += n;
7612                 remaining -= n;
7613                 addr += n;
7614         }
7615
7616         free(buf, M_CXGBE);
7617         return (rc);
7618 }
7619
7620 static int
7621 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
7622 {
7623         int rc;
7624
7625         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
7626                 return (EINVAL);
7627
7628         if (i2cd->len > sizeof(i2cd->data))
7629                 return (EFBIG);
7630
7631         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
7632         if (rc)
7633                 return (rc);
7634         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
7635             i2cd->offset, i2cd->len, &i2cd->data[0]);
7636         end_synchronized_op(sc, 0);
7637
7638         return (rc);
7639 }
7640
7641 static int
7642 in_range(int val, int lo, int hi)
7643 {
7644
7645         return (val < 0 || (val <= hi && val >= lo));
7646 }
7647
7648 static int
7649 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
7650 {
7651         int fw_subcmd, fw_type, rc;
7652
7653         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
7654         if (rc)
7655                 return (rc);
7656
7657         if (!(sc->flags & FULL_INIT_DONE)) {
7658                 rc = EAGAIN;
7659                 goto done;
7660         }
7661
7662         /*
7663          * Translate the cxgbetool parameters into T4 firmware parameters.  (The
7664          * sub-command and type are in common locations.)
7665          */
7666         if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
7667                 fw_subcmd = FW_SCHED_SC_CONFIG;
7668         else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
7669                 fw_subcmd = FW_SCHED_SC_PARAMS;
7670         else {
7671                 rc = EINVAL;
7672                 goto done;
7673         }
7674         if (p->type == SCHED_CLASS_TYPE_PACKET)
7675                 fw_type = FW_SCHED_TYPE_PKTSCHED;
7676         else {
7677                 rc = EINVAL;
7678                 goto done;
7679         }
7680
7681         if (fw_subcmd == FW_SCHED_SC_CONFIG) {
7682                 /* Vet our parameters ..*/
7683                 if (p->u.config.minmax < 0) {
7684                         rc = EINVAL;
7685                         goto done;
7686                 }
7687
7688                 /* And pass the request to the firmware ...*/
7689                 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1);
7690                 goto done;
7691         }
7692
7693         if (fw_subcmd == FW_SCHED_SC_PARAMS) {
7694                 int fw_level;
7695                 int fw_mode;
7696                 int fw_rateunit;
7697                 int fw_ratemode;
7698
7699                 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
7700                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
7701                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
7702                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
7703                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
7704                         fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
7705                 else {
7706                         rc = EINVAL;
7707                         goto done;
7708                 }
7709
7710                 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
7711                         fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
7712                 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
7713                         fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
7714                 else {
7715                         rc = EINVAL;
7716                         goto done;
7717                 }
7718
7719                 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
7720                         fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
7721                 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
7722                         fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
7723                 else {
7724                         rc = EINVAL;
7725                         goto done;
7726                 }
7727
7728                 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
7729                         fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
7730                 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
7731                         fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
7732                 else {
7733                         rc = EINVAL;
7734                         goto done;
7735                 }
7736
7737                 /* Vet our parameters ... */
7738                 if (!in_range(p->u.params.channel, 0, 3) ||
7739                     !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
7740                     !in_range(p->u.params.minrate, 0, 10000000) ||
7741                     !in_range(p->u.params.maxrate, 0, 10000000) ||
7742                     !in_range(p->u.params.weight, 0, 100)) {
7743                         rc = ERANGE;
7744                         goto done;
7745                 }
7746
7747                 /*
7748                  * Translate any unset parameters into the firmware's
7749                  * nomenclature and/or fail the call if the parameters
7750                  * are required ...
7751                  */
7752                 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
7753                     p->u.params.channel < 0 || p->u.params.cl < 0) {
7754                         rc = EINVAL;
7755                         goto done;
7756                 }
7757                 if (p->u.params.minrate < 0)
7758                         p->u.params.minrate = 0;
7759                 if (p->u.params.maxrate < 0) {
7760                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7761                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7762                                 rc = EINVAL;
7763                                 goto done;
7764                         } else
7765                                 p->u.params.maxrate = 0;
7766                 }
7767                 if (p->u.params.weight < 0) {
7768                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
7769                                 rc = EINVAL;
7770                                 goto done;
7771                         } else
7772                                 p->u.params.weight = 0;
7773                 }
7774                 if (p->u.params.pktsize < 0) {
7775                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7776                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7777                                 rc = EINVAL;
7778                                 goto done;
7779                         } else
7780                                 p->u.params.pktsize = 0;
7781                 }
7782
7783                 /* See what the firmware thinks of the request ... */
7784                 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
7785                     fw_rateunit, fw_ratemode, p->u.params.channel,
7786                     p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
7787                     p->u.params.weight, p->u.params.pktsize, 1);
7788                 goto done;
7789         }
7790
7791         rc = EINVAL;
7792 done:
7793         end_synchronized_op(sc, 0);
7794         return (rc);
7795 }
7796
7797 static int
7798 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
7799 {
7800         struct port_info *pi = NULL;
7801         struct sge_txq *txq;
7802         uint32_t fw_mnem, fw_queue, fw_class;
7803         int i, rc;
7804
7805         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
7806         if (rc)
7807                 return (rc);
7808
7809         if (!(sc->flags & FULL_INIT_DONE)) {
7810                 rc = EAGAIN;
7811                 goto done;
7812         }
7813
7814         if (p->port >= sc->params.nports) {
7815                 rc = EINVAL;
7816                 goto done;
7817         }
7818
7819         pi = sc->port[p->port];
7820         if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
7821                 rc = EINVAL;
7822                 goto done;
7823         }
7824
7825         /*
7826          * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
7827          * Scheduling Class in this case).
7828          */
7829         fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
7830             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
7831         fw_class = p->cl < 0 ? 0xffffffff : p->cl;
7832
7833         /*
7834          * If op.queue is non-negative, then we're only changing the scheduling
7835          * on a single specified TX queue.
7836          */
7837         if (p->queue >= 0) {
7838                 txq = &sc->sge.txq[pi->first_txq + p->queue];
7839                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7840                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7841                     &fw_class);
7842                 goto done;
7843         }
7844
7845         /*
7846          * Change the scheduling on all the TX queues for the
7847          * interface.
7848          */
7849         for_each_txq(pi, i, txq) {
7850                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7851                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7852                     &fw_class);
7853                 if (rc)
7854                         goto done;
7855         }
7856
7857         rc = 0;
7858 done:
7859         end_synchronized_op(sc, 0);
7860         return (rc);
7861 }
7862
7863 int
7864 t4_os_find_pci_capability(struct adapter *sc, int cap)
7865 {
7866         int i;
7867
7868         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
7869 }
7870
7871 int
7872 t4_os_pci_save_state(struct adapter *sc)
7873 {
7874         device_t dev;
7875         struct pci_devinfo *dinfo;
7876
7877         dev = sc->dev;
7878         dinfo = device_get_ivars(dev);
7879
7880         pci_cfg_save(dev, dinfo, 0);
7881         return (0);
7882 }
7883
7884 int
7885 t4_os_pci_restore_state(struct adapter *sc)
7886 {
7887         device_t dev;
7888         struct pci_devinfo *dinfo;
7889
7890         dev = sc->dev;
7891         dinfo = device_get_ivars(dev);
7892
7893         pci_cfg_restore(dev, dinfo);
7894         return (0);
7895 }
7896
7897 void
7898 t4_os_portmod_changed(const struct adapter *sc, int idx)
7899 {
7900         struct port_info *pi = sc->port[idx];
7901         static const char *mod_str[] = {
7902                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
7903         };
7904
7905         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
7906                 if_printf(pi->ifp, "transceiver unplugged.\n");
7907         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
7908                 if_printf(pi->ifp, "unknown transceiver inserted.\n");
7909         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
7910                 if_printf(pi->ifp, "unsupported transceiver inserted.\n");
7911         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
7912                 if_printf(pi->ifp, "%s transceiver inserted.\n",
7913                     mod_str[pi->mod_type]);
7914         } else {
7915                 if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
7916                     pi->mod_type);
7917         }
7918 }
7919
7920 void
7921 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
7922 {
7923         struct port_info *pi = sc->port[idx];
7924         struct ifnet *ifp = pi->ifp;
7925
7926         if (link_stat) {
7927                 pi->linkdnrc = -1;
7928                 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
7929                 if_link_state_change(ifp, LINK_STATE_UP);
7930         } else {
7931                 if (reason >= 0)
7932                         pi->linkdnrc = reason;
7933                 if_link_state_change(ifp, LINK_STATE_DOWN);
7934         }
7935 }
7936
7937 void
7938 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
7939 {
7940         struct adapter *sc;
7941
7942         sx_slock(&t4_list_lock);
7943         SLIST_FOREACH(sc, &t4_list, link) {
7944                 /*
7945                  * func should not make any assumptions about what state sc is
7946                  * in - the only guarantee is that sc->sc_lock is a valid lock.
7947                  */
7948                 func(sc, arg);
7949         }
7950         sx_sunlock(&t4_list_lock);
7951 }
7952
7953 static int
7954 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
7955 {
7956        return (0);
7957 }
7958
7959 static int
7960 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
7961 {
7962        return (0);
7963 }
7964
7965 static int
7966 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
7967     struct thread *td)
7968 {
7969         int rc;
7970         struct adapter *sc = dev->si_drv1;
7971
7972         rc = priv_check(td, PRIV_DRIVER);
7973         if (rc != 0)
7974                 return (rc);
7975
7976         switch (cmd) {
7977         case CHELSIO_T4_GETREG: {
7978                 struct t4_reg *edata = (struct t4_reg *)data;
7979
7980                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
7981                         return (EFAULT);
7982
7983                 if (edata->size == 4)
7984                         edata->val = t4_read_reg(sc, edata->addr);
7985                 else if (edata->size == 8)
7986                         edata->val = t4_read_reg64(sc, edata->addr);
7987                 else
7988                         return (EINVAL);
7989
7990                 break;
7991         }
7992         case CHELSIO_T4_SETREG: {
7993                 struct t4_reg *edata = (struct t4_reg *)data;
7994
7995                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
7996                         return (EFAULT);
7997
7998                 if (edata->size == 4) {
7999                         if (edata->val & 0xffffffff00000000)
8000                                 return (EINVAL);
8001                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
8002                 } else if (edata->size == 8)
8003                         t4_write_reg64(sc, edata->addr, edata->val);
8004                 else
8005                         return (EINVAL);
8006                 break;
8007         }
8008         case CHELSIO_T4_REGDUMP: {
8009                 struct t4_regdump *regs = (struct t4_regdump *)data;
8010                 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
8011                 uint8_t *buf;
8012
8013                 if (regs->len < reglen) {
8014                         regs->len = reglen; /* hint to the caller */
8015                         return (ENOBUFS);
8016                 }
8017
8018                 regs->len = reglen;
8019                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
8020                 t4_get_regs(sc, regs, buf);
8021                 rc = copyout(buf, regs->data, reglen);
8022                 free(buf, M_CXGBE);
8023                 break;
8024         }
8025         case CHELSIO_T4_GET_FILTER_MODE:
8026                 rc = get_filter_mode(sc, (uint32_t *)data);
8027                 break;
8028         case CHELSIO_T4_SET_FILTER_MODE:
8029                 rc = set_filter_mode(sc, *(uint32_t *)data);
8030                 break;
8031         case CHELSIO_T4_GET_FILTER:
8032                 rc = get_filter(sc, (struct t4_filter *)data);
8033                 break;
8034         case CHELSIO_T4_SET_FILTER:
8035                 rc = set_filter(sc, (struct t4_filter *)data);
8036                 break;
8037         case CHELSIO_T4_DEL_FILTER:
8038                 rc = del_filter(sc, (struct t4_filter *)data);
8039                 break;
8040         case CHELSIO_T4_GET_SGE_CONTEXT:
8041                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
8042                 break;
8043         case CHELSIO_T4_LOAD_FW:
8044                 rc = load_fw(sc, (struct t4_data *)data);
8045                 break;
8046         case CHELSIO_T4_GET_MEM:
8047                 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
8048                 break;
8049         case CHELSIO_T4_GET_I2C:
8050                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
8051                 break;
8052         case CHELSIO_T4_CLEAR_STATS: {
8053                 int i;
8054                 u_int port_id = *(uint32_t *)data;
8055                 struct port_info *pi;
8056
8057                 if (port_id >= sc->params.nports)
8058                         return (EINVAL);
8059                 pi = sc->port[port_id];
8060
8061                 /* MAC stats */
8062                 t4_clr_port_stats(sc, pi->tx_chan);
8063
8064                 if (pi->flags & PORT_INIT_DONE) {
8065                         struct sge_rxq *rxq;
8066                         struct sge_txq *txq;
8067                         struct sge_wrq *wrq;
8068
8069                         for_each_rxq(pi, i, rxq) {
8070 #if defined(INET) || defined(INET6)
8071                                 rxq->lro.lro_queued = 0;
8072                                 rxq->lro.lro_flushed = 0;
8073 #endif
8074                                 rxq->rxcsum = 0;
8075                                 rxq->vlan_extraction = 0;
8076                         }
8077
8078                         for_each_txq(pi, i, txq) {
8079                                 txq->txcsum = 0;
8080                                 txq->tso_wrs = 0;
8081                                 txq->vlan_insertion = 0;
8082                                 txq->imm_wrs = 0;
8083                                 txq->sgl_wrs = 0;
8084                                 txq->txpkt_wrs = 0;
8085                                 txq->txpkts_wrs = 0;
8086                                 txq->txpkts_pkts = 0;
8087                                 txq->br->br_drops = 0;
8088                                 txq->no_dmamap = 0;
8089                                 txq->no_desc = 0;
8090                         }
8091
8092 #ifdef TCP_OFFLOAD
8093                         /* nothing to clear for each ofld_rxq */
8094
8095                         for_each_ofld_txq(pi, i, wrq) {
8096                                 wrq->tx_wrs = 0;
8097                                 wrq->no_desc = 0;
8098                         }
8099 #endif
8100                         wrq = &sc->sge.ctrlq[pi->port_id];
8101                         wrq->tx_wrs = 0;
8102                         wrq->no_desc = 0;
8103                 }
8104                 break;
8105         }
8106         case CHELSIO_T4_SCHED_CLASS:
8107                 rc = set_sched_class(sc, (struct t4_sched_params *)data);
8108                 break;
8109         case CHELSIO_T4_SCHED_QUEUE:
8110                 rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
8111                 break;
8112         case CHELSIO_T4_GET_TRACER:
8113                 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
8114                 break;
8115         case CHELSIO_T4_SET_TRACER:
8116                 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
8117                 break;
8118         default:
8119                 rc = EINVAL;
8120         }
8121
8122         return (rc);
8123 }
8124
8125 #ifdef TCP_OFFLOAD
8126 void
8127 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask,
8128     const unsigned int *pgsz_order)
8129 {
8130         struct port_info *pi = ifp->if_softc;
8131         struct adapter *sc = pi->adapter;
8132
8133         t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
8134         t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |
8135                 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) |
8136                 V_HPZ3(pgsz_order[3]));
8137 }
8138
8139 static int
8140 toe_capability(struct port_info *pi, int enable)
8141 {
8142         int rc;
8143         struct adapter *sc = pi->adapter;
8144
8145         ASSERT_SYNCHRONIZED_OP(sc);
8146
8147         if (!is_offload(sc))
8148                 return (ENODEV);
8149
8150         if (enable) {
8151                 if (!(sc->flags & FULL_INIT_DONE)) {
8152                         rc = cxgbe_init_synchronized(pi);
8153                         if (rc)
8154                                 return (rc);
8155                 }
8156
8157                 if (isset(&sc->offload_map, pi->port_id))
8158                         return (0);
8159
8160                 if (!(sc->flags & TOM_INIT_DONE)) {
8161                         rc = t4_activate_uld(sc, ULD_TOM);
8162                         if (rc == EAGAIN) {
8163                                 log(LOG_WARNING,
8164                                     "You must kldload t4_tom.ko before trying "
8165                                     "to enable TOE on a cxgbe interface.\n");
8166                         }
8167                         if (rc != 0)
8168                                 return (rc);
8169                         KASSERT(sc->tom_softc != NULL,
8170                             ("%s: TOM activated but softc NULL", __func__));
8171                         KASSERT(sc->flags & TOM_INIT_DONE,
8172                             ("%s: TOM activated but flag not set", __func__));
8173                 }
8174
8175                 setbit(&sc->offload_map, pi->port_id);
8176         } else {
8177                 if (!isset(&sc->offload_map, pi->port_id))
8178                         return (0);
8179
8180                 KASSERT(sc->flags & TOM_INIT_DONE,
8181                     ("%s: TOM never initialized?", __func__));
8182                 clrbit(&sc->offload_map, pi->port_id);
8183         }
8184
8185         return (0);
8186 }
8187
8188 /*
8189  * Add an upper layer driver to the global list.
8190  */
8191 int
8192 t4_register_uld(struct uld_info *ui)
8193 {
8194         int rc = 0;
8195         struct uld_info *u;
8196
8197         sx_xlock(&t4_uld_list_lock);
8198         SLIST_FOREACH(u, &t4_uld_list, link) {
8199             if (u->uld_id == ui->uld_id) {
8200                     rc = EEXIST;
8201                     goto done;
8202             }
8203         }
8204
8205         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
8206         ui->refcount = 0;
8207 done:
8208         sx_xunlock(&t4_uld_list_lock);
8209         return (rc);
8210 }
8211
8212 int
8213 t4_unregister_uld(struct uld_info *ui)
8214 {
8215         int rc = EINVAL;
8216         struct uld_info *u;
8217
8218         sx_xlock(&t4_uld_list_lock);
8219
8220         SLIST_FOREACH(u, &t4_uld_list, link) {
8221             if (u == ui) {
8222                     if (ui->refcount > 0) {
8223                             rc = EBUSY;
8224                             goto done;
8225                     }
8226
8227                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
8228                     rc = 0;
8229                     goto done;
8230             }
8231         }
8232 done:
8233         sx_xunlock(&t4_uld_list_lock);
8234         return (rc);
8235 }
8236
8237 int
8238 t4_activate_uld(struct adapter *sc, int id)
8239 {
8240         int rc = EAGAIN;
8241         struct uld_info *ui;
8242
8243         ASSERT_SYNCHRONIZED_OP(sc);
8244
8245         sx_slock(&t4_uld_list_lock);
8246
8247         SLIST_FOREACH(ui, &t4_uld_list, link) {
8248                 if (ui->uld_id == id) {
8249                         rc = ui->activate(sc);
8250                         if (rc == 0)
8251                                 ui->refcount++;
8252                         goto done;
8253                 }
8254         }
8255 done:
8256         sx_sunlock(&t4_uld_list_lock);
8257
8258         return (rc);
8259 }
8260
8261 int
8262 t4_deactivate_uld(struct adapter *sc, int id)
8263 {
8264         int rc = EINVAL;
8265         struct uld_info *ui;
8266
8267         ASSERT_SYNCHRONIZED_OP(sc);
8268
8269         sx_slock(&t4_uld_list_lock);
8270
8271         SLIST_FOREACH(ui, &t4_uld_list, link) {
8272                 if (ui->uld_id == id) {
8273                         rc = ui->deactivate(sc);
8274                         if (rc == 0)
8275                                 ui->refcount--;
8276                         goto done;
8277                 }
8278         }
8279 done:
8280         sx_sunlock(&t4_uld_list_lock);
8281
8282         return (rc);
8283 }
8284 #endif
8285
8286 /*
8287  * Come up with reasonable defaults for some of the tunables, provided they're
8288  * not set by the user (in which case we'll use the values as is).
8289  */
8290 static void
8291 tweak_tunables(void)
8292 {
8293         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
8294
8295         if (t4_ntxq10g < 1)
8296                 t4_ntxq10g = min(nc, NTXQ_10G);
8297
8298         if (t4_ntxq1g < 1)
8299                 t4_ntxq1g = min(nc, NTXQ_1G);
8300
8301         if (t4_nrxq10g < 1)
8302                 t4_nrxq10g = min(nc, NRXQ_10G);
8303
8304         if (t4_nrxq1g < 1)
8305                 t4_nrxq1g = min(nc, NRXQ_1G);
8306
8307 #ifdef TCP_OFFLOAD
8308         if (t4_nofldtxq10g < 1)
8309                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
8310
8311         if (t4_nofldtxq1g < 1)
8312                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
8313
8314         if (t4_nofldrxq10g < 1)
8315                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
8316
8317         if (t4_nofldrxq1g < 1)
8318                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
8319
8320         if (t4_toecaps_allowed == -1)
8321                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
8322 #else
8323         if (t4_toecaps_allowed == -1)
8324                 t4_toecaps_allowed = 0;
8325 #endif
8326
8327 #ifdef DEV_NETMAP
8328         if (t4_nnmtxq10g < 1)
8329                 t4_nnmtxq10g = min(nc, NNMTXQ_10G);
8330
8331         if (t4_nnmtxq1g < 1)
8332                 t4_nnmtxq1g = min(nc, NNMTXQ_1G);
8333
8334         if (t4_nnmrxq10g < 1)
8335                 t4_nnmrxq10g = min(nc, NNMRXQ_10G);
8336
8337         if (t4_nnmrxq1g < 1)
8338                 t4_nnmrxq1g = min(nc, NNMRXQ_1G);
8339 #endif
8340
8341         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
8342                 t4_tmr_idx_10g = TMR_IDX_10G;
8343
8344         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
8345                 t4_pktc_idx_10g = PKTC_IDX_10G;
8346
8347         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
8348                 t4_tmr_idx_1g = TMR_IDX_1G;
8349
8350         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
8351                 t4_pktc_idx_1g = PKTC_IDX_1G;
8352
8353         if (t4_qsize_txq < 128)
8354                 t4_qsize_txq = 128;
8355
8356         if (t4_qsize_rxq < 128)
8357                 t4_qsize_rxq = 128;
8358         while (t4_qsize_rxq & 7)
8359                 t4_qsize_rxq++;
8360
8361         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
8362 }
8363
8364 static struct sx mlu;   /* mod load unload */
8365 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
8366
8367 static int
8368 mod_event(module_t mod, int cmd, void *arg)
8369 {
8370         int rc = 0;
8371         static int loaded = 0;
8372
8373         switch (cmd) {
8374         case MOD_LOAD:
8375                 sx_xlock(&mlu);
8376                 if (loaded++ == 0) {
8377                         t4_sge_modload();
8378                         sx_init(&t4_list_lock, "T4/T5 adapters");
8379                         SLIST_INIT(&t4_list);
8380 #ifdef TCP_OFFLOAD
8381                         sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
8382                         SLIST_INIT(&t4_uld_list);
8383 #endif
8384                         t4_tracer_modload();
8385                         tweak_tunables();
8386                 }
8387                 sx_xunlock(&mlu);
8388                 break;
8389
8390         case MOD_UNLOAD:
8391                 sx_xlock(&mlu);
8392                 if (--loaded == 0) {
8393                         int tries;
8394
8395                         sx_slock(&t4_list_lock);
8396                         if (!SLIST_EMPTY(&t4_list)) {
8397                                 rc = EBUSY;
8398                                 sx_sunlock(&t4_list_lock);
8399                                 goto done_unload;
8400                         }
8401 #ifdef TCP_OFFLOAD
8402                         sx_slock(&t4_uld_list_lock);
8403                         if (!SLIST_EMPTY(&t4_uld_list)) {
8404                                 rc = EBUSY;
8405                                 sx_sunlock(&t4_uld_list_lock);
8406                                 sx_sunlock(&t4_list_lock);
8407                                 goto done_unload;
8408                         }
8409 #endif
8410                         tries = 0;
8411                         while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
8412                                 uprintf("%ju clusters with custom free routine "
8413                                     "still is use.\n", t4_sge_extfree_refs());
8414                                 pause("t4unload", 2 * hz);
8415                         }
8416 #ifdef TCP_OFFLOAD
8417                         sx_sunlock(&t4_uld_list_lock);
8418 #endif
8419                         sx_sunlock(&t4_list_lock);
8420
8421                         if (t4_sge_extfree_refs() == 0) {
8422                                 t4_tracer_modunload();
8423 #ifdef TCP_OFFLOAD
8424                                 sx_destroy(&t4_uld_list_lock);
8425 #endif
8426                                 sx_destroy(&t4_list_lock);
8427                                 t4_sge_modunload();
8428                                 loaded = 0;
8429                         } else {
8430                                 rc = EBUSY;
8431                                 loaded++;       /* undo earlier decrement */
8432                         }
8433                 }
8434 done_unload:
8435                 sx_xunlock(&mlu);
8436                 break;
8437         }
8438
8439         return (rc);
8440 }
8441
8442 static devclass_t t4_devclass, t5_devclass;
8443 static devclass_t cxgbe_devclass, cxl_devclass;
8444
8445 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
8446 MODULE_VERSION(t4nex, 1);
8447 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
8448
8449 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
8450 MODULE_VERSION(t5nex, 1);
8451 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
8452
8453 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
8454 MODULE_VERSION(cxgbe, 1);
8455
8456 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
8457 MODULE_VERSION(cxl, 1);