sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/conf.h>
  36 #include <sys/priv.h>
  37 #include <sys/kernel.h>
  38 #include <sys/bus.h>
  39 #include <sys/module.h>
  40 #include <sys/malloc.h>
  41 #include <sys/queue.h>
  42 #include <sys/taskqueue.h>
  43 #include <sys/pciio.h>
  44 #include <dev/pci/pcireg.h>
  45 #include <dev/pci/pcivar.h>
  46 #include <dev/pci/pci_private.h>
  47 #include <sys/firmware.h>
  48 #include <sys/sbuf.h>
  49 #include <sys/smp.h>
  50 #include <sys/socket.h>
  51 #include <sys/sockio.h>
  52 #include <sys/sysctl.h>
  53 #include <net/ethernet.h>
  54 #include <net/if.h>
  55 #include <net/if_types.h>
  56 #include <net/if_dl.h>
  57 #include <net/if_vlan_var.h>
  58 #if defined(__i386__) || defined(__amd64__)
  59 #include <vm/vm.h>
  60 #include <vm/pmap.h>
  61 #endif
  62
  63 #include "common/common.h"
  64 #include "common/t4_msg.h"
  65 #include "common/t4_regs.h"
  66 #include "common/t4_regs_values.h"
  67 #include "t4_ioctl.h"
  68 #include "t4_l2t.h"
  69
  70 /* T4 bus driver interface */
  71 static int t4_probe(device_t);
  72 static int t4_attach(device_t);
  73 static int t4_detach(device_t);
  74 static device_method_t t4_methods[] = {
  75         DEVMETHOD(device_probe,         t4_probe),
  76         DEVMETHOD(device_attach,        t4_attach),
  77         DEVMETHOD(device_detach,        t4_detach),
  78
  79         DEVMETHOD_END
  80 };
  81 static driver_t t4_driver = {
  82         "t4nex",
  83         t4_methods,
  84         sizeof(struct adapter)
  85 };
  86
  87
  88 /* T4 port (cxgbe) interface */
  89 static int cxgbe_probe(device_t);
  90 static int cxgbe_attach(device_t);
  91 static int cxgbe_detach(device_t);
  92 static device_method_t cxgbe_methods[] = {
  93         DEVMETHOD(device_probe,         cxgbe_probe),
  94         DEVMETHOD(device_attach,        cxgbe_attach),
  95         DEVMETHOD(device_detach,        cxgbe_detach),
  96         { 0, 0 }
  97 };
  98 static driver_t cxgbe_driver = {
  99         "cxgbe",
 100         cxgbe_methods,
 101         sizeof(struct port_info)
 102 };
 103
 104 static d_ioctl_t t4_ioctl;
 105 static d_open_t t4_open;
 106 static d_close_t t4_close;
 107
 108 static struct cdevsw t4_cdevsw = {
 109        .d_version = D_VERSION,
 110        .d_flags = 0,
 111        .d_open = t4_open,
 112        .d_close = t4_close,
 113        .d_ioctl = t4_ioctl,
 114        .d_name = "t4nex",
 115 };
 116
 117 /* T5 bus driver interface */
 118 static int t5_probe(device_t);
 119 static device_method_t t5_methods[] = {
 120         DEVMETHOD(device_probe,         t5_probe),
 121         DEVMETHOD(device_attach,        t4_attach),
 122         DEVMETHOD(device_detach,        t4_detach),
 123
 124         DEVMETHOD_END
 125 };
 126 static driver_t t5_driver = {
 127         "t5nex",
 128         t5_methods,
 129         sizeof(struct adapter)
 130 };
 131
 132
 133 /* T5 port (cxl) interface */
 134 static driver_t cxl_driver = {
 135         "cxl",
 136         cxgbe_methods,
 137         sizeof(struct port_info)
 138 };
 139
 140 static struct cdevsw t5_cdevsw = {
 141        .d_version = D_VERSION,
 142        .d_flags = 0,
 143        .d_open = t4_open,
 144        .d_close = t4_close,
 145        .d_ioctl = t4_ioctl,
 146        .d_name = "t5nex",
 147 };
 148
 149 /* ifnet + media interface */
 150 static void cxgbe_init(void *);
 151 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 152 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 153 static void cxgbe_qflush(struct ifnet *);
 154 static int cxgbe_media_change(struct ifnet *);
 155 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 156
 157 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
 158
 159 /*
 160  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 161  * then ADAPTER_LOCK, then t4_uld_list_lock.
 162  */
 163 static struct sx t4_list_lock;
 164 SLIST_HEAD(, adapter) t4_list;
 165 #ifdef TCP_OFFLOAD
 166 static struct sx t4_uld_list_lock;
 167 SLIST_HEAD(, uld_info) t4_uld_list;
 168 #endif
 169
 170 /*
 171  * Tunables.  See tweak_tunables() too.
 172  *
 173  * Each tunable is set to a default value here if it's known at compile-time.
 174  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
 175  * provide a reasonable default when the driver is loaded.
 176  *
 177  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
 178  * T5 are under hw.cxl.
 179  */
 180
 181 /*
 182  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 183  */
 184 #define NTXQ_10G 16
 185 static int t4_ntxq10g = -1;
 186 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 187
 188 #define NRXQ_10G 8
 189 static int t4_nrxq10g = -1;
 190 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 191
 192 #define NTXQ_1G 4
 193 static int t4_ntxq1g = -1;
 194 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 195
 196 #define NRXQ_1G 2
 197 static int t4_nrxq1g = -1;
 198 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 199
 200 static int t4_rsrv_noflowq = 0;
 201 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
 202
 203 #ifdef TCP_OFFLOAD
 204 #define NOFLDTXQ_10G 8
 205 static int t4_nofldtxq10g = -1;
 206 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 207
 208 #define NOFLDRXQ_10G 2
 209 static int t4_nofldrxq10g = -1;
 210 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 211
 212 #define NOFLDTXQ_1G 2
 213 static int t4_nofldtxq1g = -1;
 214 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 215
 216 #define NOFLDRXQ_1G 1
 217 static int t4_nofldrxq1g = -1;
 218 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 219 #endif
 220
 221 #ifdef DEV_NETMAP
 222 #define NNMTXQ_10G 2
 223 static int t4_nnmtxq10g = -1;
 224 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g);
 225
 226 #define NNMRXQ_10G 2
 227 static int t4_nnmrxq10g = -1;
 228 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g);
 229
 230 #define NNMTXQ_1G 1
 231 static int t4_nnmtxq1g = -1;
 232 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g);
 233
 234 #define NNMRXQ_1G 1
 235 static int t4_nnmrxq1g = -1;
 236 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g);
 237 #endif
 238
 239 /*
 240  * Holdoff parameters for 10G and 1G ports.
 241  */
 242 #define TMR_IDX_10G 1
 243 static int t4_tmr_idx_10g = TMR_IDX_10G;
 244 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 245
 246 #define PKTC_IDX_10G (-1)
 247 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 248 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 249
 250 #define TMR_IDX_1G 1
 251 static int t4_tmr_idx_1g = TMR_IDX_1G;
 252 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 253
 254 #define PKTC_IDX_1G (-1)
 255 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 256 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 257
 258 /*
 259  * Size (# of entries) of each tx and rx queue.
 260  */
 261 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 262 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 263
 264 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 265 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 266
 267 /*
 268  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 269  */
 270 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 271 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 272
 273 /*
 274  * Configuration file.
 275  */
 276 #define DEFAULT_CF      "default"
 277 #define FLASH_CF        "flash"
 278 #define UWIRE_CF        "uwire"
 279 #define FPGA_CF         "fpga"
 280 static char t4_cfg_file[32] = DEFAULT_CF;
 281 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 282
 283 /*
 284  * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively).
 285  * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
 286  * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
 287  *            mark or when signalled to do so, 0 to never emit PAUSE.
 288  */
 289 static int t4_pause_settings = PAUSE_TX | PAUSE_RX;
 290 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings);
 291
 292 /*
 293  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
 294  * encouraged respectively).
 295  */
 296 static unsigned int t4_fw_install = 1;
 297 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 298
 299 /*
 300  * ASIC features that will be used.  Disable the ones you don't want so that the
 301  * chip resources aren't wasted on features that will not be used.
 302  */
 303 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 304 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 305
 306 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 307 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 308
 309 static int t4_toecaps_allowed = -1;
 310 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 311
 312 static int t4_rdmacaps_allowed = 0;
 313 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 314
 315 static int t4_iscsicaps_allowed = 0;
 316 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 317
 318 static int t4_fcoecaps_allowed = 0;
 319 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 320
 321 static int t5_write_combine = 0;
 322 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 323
 324 struct intrs_and_queues {
 325         uint16_t intr_type;     /* INTx, MSI, or MSI-X */
 326         uint16_t nirq;          /* Total # of vectors */
 327         uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
 328         uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */
 329         uint16_t ntxq10g;       /* # of NIC txq's for each 10G port */
 330         uint16_t nrxq10g;       /* # of NIC rxq's for each 10G port */
 331         uint16_t ntxq1g;        /* # of NIC txq's for each 1G port */
 332         uint16_t nrxq1g;        /* # of NIC rxq's for each 1G port */
 333         uint16_t rsrv_noflowq;  /* Flag whether to reserve queue 0 */
 334 #ifdef TCP_OFFLOAD
 335         uint16_t nofldtxq10g;   /* # of TOE txq's for each 10G port */
 336         uint16_t nofldrxq10g;   /* # of TOE rxq's for each 10G port */
 337         uint16_t nofldtxq1g;    /* # of TOE txq's for each 1G port */
 338         uint16_t nofldrxq1g;    /* # of TOE rxq's for each 1G port */
 339 #endif
 340 #ifdef DEV_NETMAP
 341         uint16_t nnmtxq10g;     /* # of netmap txq's for each 10G port */
 342         uint16_t nnmrxq10g;     /* # of netmap rxq's for each 10G port */
 343         uint16_t nnmtxq1g;      /* # of netmap txq's for each 1G port */
 344         uint16_t nnmrxq1g;      /* # of netmap rxq's for each 1G port */
 345 #endif
 346 };
 347
 348 struct filter_entry {
 349         uint32_t valid:1;       /* filter allocated and valid */
 350         uint32_t locked:1;      /* filter is administratively locked */
 351         uint32_t pending:1;     /* filter action is pending firmware reply */
 352         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 353         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 354
 355         struct t4_filter_specification fs;
 356 };
 357
 358 static int map_bars_0_and_4(struct adapter *);
 359 static int map_bar_2(struct adapter *);
 360 static void setup_memwin(struct adapter *);
 361 static int validate_mem_range(struct adapter *, uint32_t, int);
 362 static int fwmtype_to_hwmtype(int);
 363 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
 364     uint32_t *);
 365 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
 366 static uint32_t position_memwin(struct adapter *, int, uint32_t);
 367 static int cfg_itype_and_nqueues(struct adapter *, int, int,
 368     struct intrs_and_queues *);
 369 static int prep_firmware(struct adapter *);
 370 static int partition_resources(struct adapter *, const struct firmware *,
 371     const char *);
 372 static int get_params__pre_init(struct adapter *);
 373 static int get_params__post_init(struct adapter *);
 374 static int set_params__post_init(struct adapter *);
 375 static void t4_set_desc(struct adapter *);
 376 static void build_medialist(struct port_info *, struct ifmedia *);
 377 static int cxgbe_init_synchronized(struct port_info *);
 378 static int cxgbe_uninit_synchronized(struct port_info *);
 379 static int setup_intr_handlers(struct adapter *);
 380 static void quiesce_eq(struct adapter *, struct sge_eq *);
 381 static void quiesce_iq(struct adapter *, struct sge_iq *);
 382 static void quiesce_fl(struct adapter *, struct sge_fl *);
 383 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 384     driver_intr_t *, void *, char *);
 385 static int t4_free_irq(struct adapter *, struct irq *);
 386 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
 387     unsigned int);
 388 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 389 static void cxgbe_tick(void *);
 390 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 391 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
 392     struct mbuf *);
 393 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 394 static int fw_msg_not_handled(struct adapter *, const __be64 *);
 395 static int t4_sysctls(struct adapter *);
 396 static int cxgbe_sysctls(struct port_info *);
 397 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 398 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 399 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
 400 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 401 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 402 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 403 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 404 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 405 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
 406 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 407 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 408 #ifdef SBUF_DRAIN
 409 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 410 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 411 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 412 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
 413 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
 414 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 415 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 416 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 417 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 418 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 419 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 420 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 421 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
 422 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 423 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
 424 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 425 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 426 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 427 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 428 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 429 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 430 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
 431 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 432 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 433 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 434 #endif
 435 static inline void txq_start(struct ifnet *, struct sge_txq *);
 436 static uint32_t fconf_to_mode(uint32_t);
 437 static uint32_t mode_to_fconf(uint32_t);
 438 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 439 static int get_filter_mode(struct adapter *, uint32_t *);
 440 static int set_filter_mode(struct adapter *, uint32_t);
 441 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 442 static int get_filter(struct adapter *, struct t4_filter *);
 443 static int set_filter(struct adapter *, struct t4_filter *);
 444 static int del_filter(struct adapter *, struct t4_filter *);
 445 static void clear_filter(struct filter_entry *);
 446 static int set_filter_wr(struct adapter *, int);
 447 static int del_filter_wr(struct adapter *, int);
 448 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 449 static int load_fw(struct adapter *, struct t4_data *);
 450 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 451 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 452 static int set_sched_class(struct adapter *, struct t4_sched_params *);
 453 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
 454 #ifdef TCP_OFFLOAD
 455 static int toe_capability(struct port_info *, int);
 456 #endif
 457 static int mod_event(module_t, int, void *);
 458
 459 struct {
 460         uint16_t device;
 461         char *desc;
 462 } t4_pciids[] = {
 463         {0xa000, "Chelsio Terminator 4 FPGA"},
 464         {0x4400, "Chelsio T440-dbg"},
 465         {0x4401, "Chelsio T420-CR"},
 466         {0x4402, "Chelsio T422-CR"},
 467         {0x4403, "Chelsio T440-CR"},
 468         {0x4404, "Chelsio T420-BCH"},
 469         {0x4405, "Chelsio T440-BCH"},
 470         {0x4406, "Chelsio T440-CH"},
 471         {0x4407, "Chelsio T420-SO"},
 472         {0x4408, "Chelsio T420-CX"},
 473         {0x4409, "Chelsio T420-BT"},
 474         {0x440a, "Chelsio T404-BT"},
 475         {0x440e, "Chelsio T440-LP-CR"},
 476 }, t5_pciids[] = {
 477         {0xb000, "Chelsio Terminator 5 FPGA"},
 478         {0x5400, "Chelsio T580-dbg"},
 479         {0x5401,  "Chelsio T520-CR"},           /* 2 x 10G */
 480         {0x5402,  "Chelsio T522-CR"},           /* 2 x 10G, 2 X 1G */
 481         {0x5403,  "Chelsio T540-CR"},           /* 4 x 10G */
 482         {0x5407,  "Chelsio T520-SO"},           /* 2 x 10G, nomem */
 483         {0x5409,  "Chelsio T520-BT"},           /* 2 x 10GBaseT */
 484         {0x540a,  "Chelsio T504-BT"},           /* 4 x 1G */
 485         {0x540d,  "Chelsio T580-CR"},           /* 2 x 40G */
 486         {0x540e,  "Chelsio T540-LP-CR"},        /* 4 x 10G */
 487         {0x5410,  "Chelsio T580-LP-CR"},        /* 2 x 40G */
 488         {0x5411,  "Chelsio T520-LL-CR"},        /* 2 x 10G */
 489         {0x5412,  "Chelsio T560-CR"},           /* 1 x 40G, 2 x 10G */
 490         {0x5414,  "Chelsio T580-LP-SO-CR"},     /* 2 x 40G, nomem */
 491         {0x5415,  "Chelsio T502-BT"},           /* 2 x 1G */
 492 #ifdef notyet
 493         {0x5404,  "Chelsio T520-BCH"},
 494         {0x5405,  "Chelsio T540-BCH"},
 495         {0x5406,  "Chelsio T540-CH"},
 496         {0x5408,  "Chelsio T520-CX"},
 497         {0x540b,  "Chelsio B520-SR"},
 498         {0x540c,  "Chelsio B504-BT"},
 499         {0x540f,  "Chelsio Amsterdam"},
 500         {0x5413,  "Chelsio T580-CHR"},
 501 #endif
 502 };
 503
 504 #ifdef TCP_OFFLOAD
 505 /*
 506  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 507  * exactly the same for both rxq and ofld_rxq.
 508  */
 509 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 510 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 511 #endif
 512
 513 /* No easy way to include t4_msg.h before adapter.h so we check this way */
 514 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
 515 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
 516
 517 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
 518
 519 static int
 520 t4_probe(device_t dev)
 521 {
 522         int i;
 523         uint16_t v = pci_get_vendor(dev);
 524         uint16_t d = pci_get_device(dev);
 525         uint8_t f = pci_get_function(dev);
 526
 527         if (v != PCI_VENDOR_ID_CHELSIO)
 528                 return (ENXIO);
 529
 530         /* Attach only to PF0 of the FPGA */
 531         if (d == 0xa000 && f != 0)
 532                 return (ENXIO);
 533
 534         for (i = 0; i < nitems(t4_pciids); i++) {
 535                 if (d == t4_pciids[i].device) {
 536                         device_set_desc(dev, t4_pciids[i].desc);
 537                         return (BUS_PROBE_DEFAULT);
 538                 }
 539         }
 540
 541         return (ENXIO);
 542 }
 543
 544 static int
 545 t5_probe(device_t dev)
 546 {
 547         int i;
 548         uint16_t v = pci_get_vendor(dev);
 549         uint16_t d = pci_get_device(dev);
 550         uint8_t f = pci_get_function(dev);
 551
 552         if (v != PCI_VENDOR_ID_CHELSIO)
 553                 return (ENXIO);
 554
 555         /* Attach only to PF0 of the FPGA */
 556         if (d == 0xb000 && f != 0)
 557                 return (ENXIO);
 558
 559         for (i = 0; i < nitems(t5_pciids); i++) {
 560                 if (d == t5_pciids[i].device) {
 561                         device_set_desc(dev, t5_pciids[i].desc);
 562                         return (BUS_PROBE_DEFAULT);
 563                 }
 564         }
 565
 566         return (ENXIO);
 567 }
 568
 569 static int
 570 t4_attach(device_t dev)
 571 {
 572         struct adapter *sc;
 573         int rc = 0, i, n10g, n1g, rqidx, tqidx;
 574         struct intrs_and_queues iaq;
 575         struct sge *s;
 576 #ifdef TCP_OFFLOAD
 577         int ofld_rqidx, ofld_tqidx;
 578 #endif
 579 #ifdef DEV_NETMAP
 580         int nm_rqidx, nm_tqidx;
 581 #endif
 582
 583         sc = device_get_softc(dev);
 584         sc->dev = dev;
 585
 586         pci_enable_busmaster(dev);
 587         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 588                 uint32_t v;
 589
 590                 pci_set_max_read_req(dev, 4096);
 591                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 592                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 593                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 594
 595                 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
 596         }
 597
 598         sc->traceq = -1;
 599         mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
 600         snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
 601             device_get_nameunit(dev));
 602
 603         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 604             device_get_nameunit(dev));
 605         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 606         sx_xlock(&t4_list_lock);
 607         SLIST_INSERT_HEAD(&t4_list, sc, link);
 608         sx_xunlock(&t4_list_lock);
 609
 610         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 611         TAILQ_INIT(&sc->sfl);
 612         callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
 613
 614         rc = map_bars_0_and_4(sc);
 615         if (rc != 0)
 616                 goto done; /* error message displayed already */
 617
 618         /*
 619          * This is the real PF# to which we're attaching.  Works from within PCI
 620          * passthrough environments too, where pci_get_function() could return a
 621          * different PF# depending on the passthrough configuration.  We need to
 622          * use the real PF# in all our communication with the firmware.
 623          */
 624         sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 625         sc->mbox = sc->pf;
 626
 627         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 628         sc->an_handler = an_not_handled;
 629         for (i = 0; i < nitems(sc->cpl_handler); i++)
 630                 sc->cpl_handler[i] = cpl_not_handled;
 631         for (i = 0; i < nitems(sc->fw_msg_handler); i++)
 632                 sc->fw_msg_handler[i] = fw_msg_not_handled;
 633         t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 634         t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
 635         t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt);
 636         t4_init_sge_cpl_handlers(sc);
 637
 638         /* Prepare the adapter for operation */
 639         rc = -t4_prep_adapter(sc);
 640         if (rc != 0) {
 641                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 642                 goto done;
 643         }
 644
 645         /*
 646          * Do this really early, with the memory windows set up even before the
 647          * character device.  The userland tool's register i/o and mem read
 648          * will work even in "recovery mode".
 649          */
 650         setup_memwin(sc);
 651         sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
 652             device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
 653             device_get_nameunit(dev));
 654         if (sc->cdev == NULL)
 655                 device_printf(dev, "failed to create nexus char device.\n");
 656         else
 657                 sc->cdev->si_drv1 = sc;
 658
 659         /* Go no further if recovery mode has been requested. */
 660         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 661                 device_printf(dev, "recovery mode.\n");
 662                 goto done;
 663         }
 664
 665         /* Prepare the firmware for operation */
 666         rc = prep_firmware(sc);
 667         if (rc != 0)
 668                 goto done; /* error message displayed already */
 669
 670         rc = get_params__post_init(sc);
 671         if (rc != 0)
 672                 goto done; /* error message displayed already */
 673
 674         rc = set_params__post_init(sc);
 675         if (rc != 0)
 676                 goto done; /* error message displayed already */
 677
 678         rc = map_bar_2(sc);
 679         if (rc != 0)
 680                 goto done; /* error message displayed already */
 681
 682         rc = t4_create_dma_tag(sc);
 683         if (rc != 0)
 684                 goto done; /* error message displayed already */
 685
 686         /*
 687          * First pass over all the ports - allocate VIs and initialize some
 688          * basic parameters like mac address, port type, etc.  We also figure
 689          * out whether a port is 10G or 1G and use that information when
 690          * calculating how many interrupts to attempt to allocate.
 691          */
 692         n10g = n1g = 0;
 693         for_each_port(sc, i) {
 694                 struct port_info *pi;
 695
 696                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 697                 sc->port[i] = pi;
 698
 699                 /* These must be set before t4_port_init */
 700                 pi->adapter = sc;
 701                 pi->port_id = i;
 702
 703                 /* Allocate the vi and initialize parameters like mac addr */
 704                 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 705                 if (rc != 0) {
 706                         device_printf(dev, "unable to initialize port %d: %d\n",
 707                             i, rc);
 708                         free(pi, M_CXGBE);
 709                         sc->port[i] = NULL;
 710                         goto done;
 711                 }
 712
 713                 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX);
 714                 pi->link_cfg.requested_fc |= t4_pause_settings;
 715                 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX);
 716                 pi->link_cfg.fc |= t4_pause_settings;
 717
 718                 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
 719                 if (rc != 0) {
 720                         device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
 721                         free(pi, M_CXGBE);
 722                         sc->port[i] = NULL;
 723                         goto done;
 724                 }
 725
 726                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 727                     device_get_nameunit(dev), i);
 728                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 729                 sc->chan_map[pi->tx_chan] = i;
 730
 731                 if (is_10G_port(pi) || is_40G_port(pi)) {
 732                         n10g++;
 733                         pi->tmr_idx = t4_tmr_idx_10g;
 734                         pi->pktc_idx = t4_pktc_idx_10g;
 735                 } else {
 736                         n1g++;
 737                         pi->tmr_idx = t4_tmr_idx_1g;
 738                         pi->pktc_idx = t4_pktc_idx_1g;
 739                 }
 740
 741                 pi->xact_addr_filt = -1;
 742                 pi->linkdnrc = -1;
 743
 744                 pi->qsize_rxq = t4_qsize_rxq;
 745                 pi->qsize_txq = t4_qsize_txq;
 746
 747                 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
 748                 if (pi->dev == NULL) {
 749                         device_printf(dev,
 750                             "failed to add device for port %d.\n", i);
 751                         rc = ENXIO;
 752                         goto done;
 753                 }
 754                 device_set_softc(pi->dev, pi);
 755         }
 756
 757         /*
 758          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 759          */
 760         rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
 761         if (rc != 0)
 762                 goto done; /* error message displayed already */
 763
 764         sc->intr_type = iaq.intr_type;
 765         sc->intr_count = iaq.nirq;
 766
 767         s = &sc->sge;
 768         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 769         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 770         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
 771         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 772         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
 773 #ifdef TCP_OFFLOAD
 774         if (is_offload(sc)) {
 775                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 776                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 777                 s->neq += s->nofldtxq + s->nofldrxq;
 778                 s->niq += s->nofldrxq;
 779
 780                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 781                     M_CXGBE, M_ZERO | M_WAITOK);
 782                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 783                     M_CXGBE, M_ZERO | M_WAITOK);
 784         }
 785 #endif
 786 #ifdef DEV_NETMAP
 787         s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g;
 788         s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g;
 789         s->neq += s->nnmtxq + s->nnmrxq;
 790         s->niq += s->nnmrxq;
 791
 792         s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
 793             M_CXGBE, M_ZERO | M_WAITOK);
 794         s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
 795             M_CXGBE, M_ZERO | M_WAITOK);
 796 #endif
 797
 798         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 799             M_ZERO | M_WAITOK);
 800         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 801             M_ZERO | M_WAITOK);
 802         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 803             M_ZERO | M_WAITOK);
 804         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 805             M_ZERO | M_WAITOK);
 806         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 807             M_ZERO | M_WAITOK);
 808
 809         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 810             M_ZERO | M_WAITOK);
 811
 812         t4_init_l2t(sc, M_WAITOK);
 813
 814         /*
 815          * Second pass over the ports.  This time we know the number of rx and
 816          * tx queues that each port should get.
 817          */
 818         rqidx = tqidx = 0;
 819 #ifdef TCP_OFFLOAD
 820         ofld_rqidx = ofld_tqidx = 0;
 821 #endif
 822 #ifdef DEV_NETMAP
 823         nm_rqidx = nm_tqidx = 0;
 824 #endif
 825         for_each_port(sc, i) {
 826                 struct port_info *pi = sc->port[i];
 827
 828                 if (pi == NULL)
 829                         continue;
 830
 831                 pi->first_rxq = rqidx;
 832                 pi->first_txq = tqidx;
 833                 if (is_10G_port(pi) || is_40G_port(pi)) {
 834                         pi->flags |= iaq.intr_flags_10g;
 835                         pi->nrxq = iaq.nrxq10g;
 836                         pi->ntxq = iaq.ntxq10g;
 837                 } else {
 838                         pi->flags |= iaq.intr_flags_1g;
 839                         pi->nrxq = iaq.nrxq1g;
 840                         pi->ntxq = iaq.ntxq1g;
 841                 }
 842
 843                 if (pi->ntxq > 1)
 844                         pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
 845                 else
 846                         pi->rsrv_noflowq = 0;
 847
 848                 rqidx += pi->nrxq;
 849                 tqidx += pi->ntxq;
 850 #ifdef TCP_OFFLOAD
 851                 if (is_offload(sc)) {
 852                         pi->first_ofld_rxq = ofld_rqidx;
 853                         pi->first_ofld_txq = ofld_tqidx;
 854                         if (is_10G_port(pi) || is_40G_port(pi)) {
 855                                 pi->nofldrxq = iaq.nofldrxq10g;
 856                                 pi->nofldtxq = iaq.nofldtxq10g;
 857                         } else {
 858                                 pi->nofldrxq = iaq.nofldrxq1g;
 859                                 pi->nofldtxq = iaq.nofldtxq1g;
 860                         }
 861                         ofld_rqidx += pi->nofldrxq;
 862                         ofld_tqidx += pi->nofldtxq;
 863                 }
 864 #endif
 865 #ifdef DEV_NETMAP
 866                 pi->first_nm_rxq = nm_rqidx;
 867                 pi->first_nm_txq = nm_tqidx;
 868                 if (is_10G_port(pi) || is_40G_port(pi)) {
 869                         pi->nnmrxq = iaq.nnmrxq10g;
 870                         pi->nnmtxq = iaq.nnmtxq10g;
 871                 } else {
 872                         pi->nnmrxq = iaq.nnmrxq1g;
 873                         pi->nnmtxq = iaq.nnmtxq1g;
 874                 }
 875                 nm_rqidx += pi->nnmrxq;
 876                 nm_tqidx += pi->nnmtxq;
 877 #endif
 878         }
 879
 880         rc = setup_intr_handlers(sc);
 881         if (rc != 0) {
 882                 device_printf(dev,
 883                     "failed to setup interrupt handlers: %d\n", rc);
 884                 goto done;
 885         }
 886
 887         rc = bus_generic_attach(dev);
 888         if (rc != 0) {
 889                 device_printf(dev,
 890                     "failed to attach all child ports: %d\n", rc);
 891                 goto done;
 892         }
 893
 894         device_printf(dev,
 895             "PCIe x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
 896             sc->params.pci.width, sc->params.nports, sc->intr_count,
 897             sc->intr_type == INTR_MSIX ? "MSI-X" :
 898             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
 899             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
 900
 901         t4_set_desc(sc);
 902
 903 done:
 904         if (rc != 0 && sc->cdev) {
 905                 /* cdev was created and so cxgbetool works; recover that way. */
 906                 device_printf(dev,
 907                     "error during attach, adapter is now in recovery mode.\n");
 908                 rc = 0;
 909         }
 910
 911         if (rc != 0)
 912                 t4_detach(dev);
 913         else
 914                 t4_sysctls(sc);
 915
 916         return (rc);
 917 }
 918
 919 /*
 920  * Idempotent
 921  */
 922 static int
 923 t4_detach(device_t dev)
 924 {
 925         struct adapter *sc;
 926         struct port_info *pi;
 927         int i, rc;
 928
 929         sc = device_get_softc(dev);
 930
 931         if (sc->flags & FULL_INIT_DONE)
 932                 t4_intr_disable(sc);
 933
 934         if (sc->cdev) {
 935                 destroy_dev(sc->cdev);
 936                 sc->cdev = NULL;
 937         }
 938
 939         rc = bus_generic_detach(dev);
 940         if (rc) {
 941                 device_printf(dev,
 942                     "failed to detach child devices: %d\n", rc);
 943                 return (rc);
 944         }
 945
 946         for (i = 0; i < sc->intr_count; i++)
 947                 t4_free_irq(sc, &sc->irq[i]);
 948
 949         for (i = 0; i < MAX_NPORTS; i++) {
 950                 pi = sc->port[i];
 951                 if (pi) {
 952                         t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid);
 953                         if (pi->dev)
 954                                 device_delete_child(dev, pi->dev);
 955
 956                         mtx_destroy(&pi->pi_lock);
 957                         free(pi, M_CXGBE);
 958                 }
 959         }
 960
 961         if (sc->flags & FULL_INIT_DONE)
 962                 adapter_full_uninit(sc);
 963
 964         if (sc->flags & FW_OK)
 965                 t4_fw_bye(sc, sc->mbox);
 966
 967         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
 968                 pci_release_msi(dev);
 969
 970         if (sc->regs_res)
 971                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
 972                     sc->regs_res);
 973
 974         if (sc->udbs_res)
 975                 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
 976                     sc->udbs_res);
 977
 978         if (sc->msix_res)
 979                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
 980                     sc->msix_res);
 981
 982         if (sc->l2t)
 983                 t4_free_l2t(sc->l2t);
 984
 985 #ifdef TCP_OFFLOAD
 986         free(sc->sge.ofld_rxq, M_CXGBE);
 987         free(sc->sge.ofld_txq, M_CXGBE);
 988 #endif
 989 #ifdef DEV_NETMAP
 990         free(sc->sge.nm_rxq, M_CXGBE);
 991         free(sc->sge.nm_txq, M_CXGBE);
 992 #endif
 993         free(sc->irq, M_CXGBE);
 994         free(sc->sge.rxq, M_CXGBE);
 995         free(sc->sge.txq, M_CXGBE);
 996         free(sc->sge.ctrlq, M_CXGBE);
 997         free(sc->sge.iqmap, M_CXGBE);
 998         free(sc->sge.eqmap, M_CXGBE);
 999         free(sc->tids.ftid_tab, M_CXGBE);
1000         t4_destroy_dma_tag(sc);
1001         if (mtx_initialized(&sc->sc_lock)) {
1002                 sx_xlock(&t4_list_lock);
1003                 SLIST_REMOVE(&t4_list, sc, adapter, link);
1004                 sx_xunlock(&t4_list_lock);
1005                 mtx_destroy(&sc->sc_lock);
1006         }
1007
1008         if (mtx_initialized(&sc->tids.ftid_lock))
1009                 mtx_destroy(&sc->tids.ftid_lock);
1010         if (mtx_initialized(&sc->sfl_lock))
1011                 mtx_destroy(&sc->sfl_lock);
1012         if (mtx_initialized(&sc->ifp_lock))
1013                 mtx_destroy(&sc->ifp_lock);
1014
1015         bzero(sc, sizeof(*sc));
1016
1017         return (0);
1018 }
1019
1020 static int
1021 cxgbe_probe(device_t dev)
1022 {
1023         char buf[128];
1024         struct port_info *pi = device_get_softc(dev);
1025
1026         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
1027         device_set_desc_copy(dev, buf);
1028
1029         return (BUS_PROBE_DEFAULT);
1030 }
1031
1032 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
1033     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1034     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
1035 #define T4_CAP_ENABLE (T4_CAP)
1036
1037 static int
1038 cxgbe_attach(device_t dev)
1039 {
1040         struct port_info *pi = device_get_softc(dev);
1041         struct ifnet *ifp;
1042         char *s;
1043         int n, o;
1044
1045         /* Allocate an ifnet and set it up */
1046         ifp = if_alloc(IFT_ETHER);
1047         if (ifp == NULL) {
1048                 device_printf(dev, "Cannot allocate ifnet\n");
1049                 return (ENOMEM);
1050         }
1051         pi->ifp = ifp;
1052         ifp->if_softc = pi;
1053
1054         callout_init(&pi->tick, CALLOUT_MPSAFE);
1055
1056         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1057         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1058
1059         ifp->if_init = cxgbe_init;
1060         ifp->if_ioctl = cxgbe_ioctl;
1061         ifp->if_transmit = cxgbe_transmit;
1062         ifp->if_qflush = cxgbe_qflush;
1063
1064         ifp->if_capabilities = T4_CAP;
1065 #ifdef TCP_OFFLOAD
1066         if (is_offload(pi->adapter))
1067                 ifp->if_capabilities |= IFCAP_TOE;
1068 #endif
1069         ifp->if_capenable = T4_CAP_ENABLE;
1070         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1071             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1072
1073         ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1074         ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS;
1075         ifp->if_hw_tsomaxsegsize = 65536;
1076
1077         /* Initialize ifmedia for this port */
1078         ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1079             cxgbe_media_status);
1080         build_medialist(pi, &pi->media);
1081
1082         pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1083             EVENTHANDLER_PRI_ANY);
1084
1085         ether_ifattach(ifp, pi->hw_addr);
1086
1087         n = 128;
1088         s = malloc(n, M_CXGBE, M_WAITOK);
1089         o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq);
1090         MPASS(n > o);
1091 #ifdef TCP_OFFLOAD
1092         if (is_offload(pi->adapter)) {
1093                 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)",
1094                     pi->nofldtxq, pi->nofldrxq);
1095                 MPASS(n > o);
1096         }
1097 #endif
1098 #ifdef DEV_NETMAP
1099         o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq,
1100             pi->nnmrxq);
1101         MPASS(n > o);
1102 #endif
1103         device_printf(dev, "%s\n", s);
1104         free(s, M_CXGBE);
1105
1106 #ifdef DEV_NETMAP
1107         /* nm_media handled here to keep implementation private to this file */
1108         ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change,
1109             cxgbe_media_status);
1110         build_medialist(pi, &pi->nm_media);
1111         create_netmap_ifnet(pi);        /* logs errors it something fails */
1112 #endif
1113         cxgbe_sysctls(pi);
1114
1115         return (0);
1116 }
1117
1118 static int
1119 cxgbe_detach(device_t dev)
1120 {
1121         struct port_info *pi = device_get_softc(dev);
1122         struct adapter *sc = pi->adapter;
1123         struct ifnet *ifp = pi->ifp;
1124
1125         /* Tell if_ioctl and if_init that the port is going away */
1126         ADAPTER_LOCK(sc);
1127         SET_DOOMED(pi);
1128         wakeup(&sc->flags);
1129         while (IS_BUSY(sc))
1130                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
1131         SET_BUSY(sc);
1132 #ifdef INVARIANTS
1133         sc->last_op = "t4detach";
1134         sc->last_op_thr = curthread;
1135 #endif
1136         ADAPTER_UNLOCK(sc);
1137
1138         if (pi->flags & HAS_TRACEQ) {
1139                 sc->traceq = -1;        /* cloner should not create ifnet */
1140                 t4_tracer_port_detach(sc);
1141         }
1142
1143         if (pi->vlan_c)
1144                 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
1145
1146         PORT_LOCK(pi);
1147         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1148         callout_stop(&pi->tick);
1149         PORT_UNLOCK(pi);
1150         callout_drain(&pi->tick);
1151
1152         /* Let detach proceed even if these fail. */
1153         cxgbe_uninit_synchronized(pi);
1154         port_full_uninit(pi);
1155
1156         ifmedia_removeall(&pi->media);
1157         ether_ifdetach(pi->ifp);
1158         if_free(pi->ifp);
1159
1160 #ifdef DEV_NETMAP
1161         /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */
1162         destroy_netmap_ifnet(pi);
1163 #endif
1164
1165         ADAPTER_LOCK(sc);
1166         CLR_BUSY(sc);
1167         wakeup(&sc->flags);
1168         ADAPTER_UNLOCK(sc);
1169
1170         return (0);
1171 }
1172
1173 static void
1174 cxgbe_init(void *arg)
1175 {
1176         struct port_info *pi = arg;
1177         struct adapter *sc = pi->adapter;
1178
1179         if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
1180                 return;
1181         cxgbe_init_synchronized(pi);
1182         end_synchronized_op(sc, 0);
1183 }
1184
1185 static int
1186 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1187 {
1188         int rc = 0, mtu, flags, can_sleep;
1189         struct port_info *pi = ifp->if_softc;
1190         struct adapter *sc = pi->adapter;
1191         struct ifreq *ifr = (struct ifreq *)data;
1192         uint32_t mask;
1193
1194         switch (cmd) {
1195         case SIOCSIFMTU:
1196                 mtu = ifr->ifr_mtu;
1197                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1198                         return (EINVAL);
1199
1200                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
1201                 if (rc)
1202                         return (rc);
1203                 ifp->if_mtu = mtu;
1204                 if (pi->flags & PORT_INIT_DONE) {
1205                         t4_update_fl_bufsize(ifp);
1206                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1207                                 rc = update_mac_settings(ifp, XGMAC_MTU);
1208                 }
1209                 end_synchronized_op(sc, 0);
1210                 break;
1211
1212         case SIOCSIFFLAGS:
1213                 can_sleep = 0;
1214 redo_sifflags:
1215                 rc = begin_synchronized_op(sc, pi,
1216                     can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg");
1217                 if (rc)
1218                         return (rc);
1219
1220                 if (ifp->if_flags & IFF_UP) {
1221                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1222                                 flags = pi->if_flags;
1223                                 if ((ifp->if_flags ^ flags) &
1224                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1225                                         if (can_sleep == 1) {
1226                                                 end_synchronized_op(sc, 0);
1227                                                 can_sleep = 0;
1228                                                 goto redo_sifflags;
1229                                         }
1230                                         rc = update_mac_settings(ifp,
1231                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1232                                 }
1233                         } else {
1234                                 if (can_sleep == 0) {
1235                                         end_synchronized_op(sc, LOCK_HELD);
1236                                         can_sleep = 1;
1237                                         goto redo_sifflags;
1238                                 }
1239                                 rc = cxgbe_init_synchronized(pi);
1240                         }
1241                         pi->if_flags = ifp->if_flags;
1242                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1243                         if (can_sleep == 0) {
1244                                 end_synchronized_op(sc, LOCK_HELD);
1245                                 can_sleep = 1;
1246                                 goto redo_sifflags;
1247                         }
1248                         rc = cxgbe_uninit_synchronized(pi);
1249                 }
1250                 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD);
1251                 break;
1252
1253         case SIOCADDMULTI:
1254         case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1255                 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
1256                 if (rc)
1257                         return (rc);
1258                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1259                         rc = update_mac_settings(ifp, XGMAC_MCADDRS);
1260                 end_synchronized_op(sc, LOCK_HELD);
1261                 break;
1262
1263         case SIOCSIFCAP:
1264                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
1265                 if (rc)
1266                         return (rc);
1267
1268                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1269                 if (mask & IFCAP_TXCSUM) {
1270                         ifp->if_capenable ^= IFCAP_TXCSUM;
1271                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1272
1273                         if (IFCAP_TSO4 & ifp->if_capenable &&
1274                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1275                                 ifp->if_capenable &= ~IFCAP_TSO4;
1276                                 if_printf(ifp,
1277                                     "tso4 disabled due to -txcsum.\n");
1278                         }
1279                 }
1280                 if (mask & IFCAP_TXCSUM_IPV6) {
1281                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1282                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1283
1284                         if (IFCAP_TSO6 & ifp->if_capenable &&
1285                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1286                                 ifp->if_capenable &= ~IFCAP_TSO6;
1287                                 if_printf(ifp,
1288                                     "tso6 disabled due to -txcsum6.\n");
1289                         }
1290                 }
1291                 if (mask & IFCAP_RXCSUM)
1292                         ifp->if_capenable ^= IFCAP_RXCSUM;
1293                 if (mask & IFCAP_RXCSUM_IPV6)
1294                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1295
1296                 /*
1297                  * Note that we leave CSUM_TSO alone (it is always set).  The
1298                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1299                  * sending a TSO request our way, so it's sufficient to toggle
1300                  * IFCAP_TSOx only.
1301                  */
1302                 if (mask & IFCAP_TSO4) {
1303                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1304                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1305                                 if_printf(ifp, "enable txcsum first.\n");
1306                                 rc = EAGAIN;
1307                                 goto fail;
1308                         }
1309                         ifp->if_capenable ^= IFCAP_TSO4;
1310                 }
1311                 if (mask & IFCAP_TSO6) {
1312                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1313                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1314                                 if_printf(ifp, "enable txcsum6 first.\n");
1315                                 rc = EAGAIN;
1316                                 goto fail;
1317                         }
1318                         ifp->if_capenable ^= IFCAP_TSO6;
1319                 }
1320                 if (mask & IFCAP_LRO) {
1321 #if defined(INET) || defined(INET6)
1322                         int i;
1323                         struct sge_rxq *rxq;
1324
1325                         ifp->if_capenable ^= IFCAP_LRO;
1326                         for_each_rxq(pi, i, rxq) {
1327                                 if (ifp->if_capenable & IFCAP_LRO)
1328                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1329                                 else
1330                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1331                         }
1332 #endif
1333                 }
1334 #ifdef TCP_OFFLOAD
1335                 if (mask & IFCAP_TOE) {
1336                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1337
1338                         rc = toe_capability(pi, enable);
1339                         if (rc != 0)
1340                                 goto fail;
1341
1342                         ifp->if_capenable ^= mask;
1343                 }
1344 #endif
1345                 if (mask & IFCAP_VLAN_HWTAGGING) {
1346                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1347                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1348                                 rc = update_mac_settings(ifp, XGMAC_VLANEX);
1349                 }
1350                 if (mask & IFCAP_VLAN_MTU) {
1351                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1352
1353                         /* Need to find out how to disable auto-mtu-inflation */
1354                 }
1355                 if (mask & IFCAP_VLAN_HWTSO)
1356                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1357                 if (mask & IFCAP_VLAN_HWCSUM)
1358                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1359
1360 #ifdef VLAN_CAPABILITIES
1361                 VLAN_CAPABILITIES(ifp);
1362 #endif
1363 fail:
1364                 end_synchronized_op(sc, 0);
1365                 break;
1366
1367         case SIOCSIFMEDIA:
1368         case SIOCGIFMEDIA:
1369                 ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1370                 break;
1371
1372         default:
1373                 rc = ether_ioctl(ifp, cmd, data);
1374         }
1375
1376         return (rc);
1377 }
1378
1379 static int
1380 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1381 {
1382         struct port_info *pi = ifp->if_softc;
1383         struct adapter *sc = pi->adapter;
1384         struct sge_txq *txq = &sc->sge.txq[pi->first_txq];
1385         struct buf_ring *br;
1386         int rc;
1387
1388         M_ASSERTPKTHDR(m);
1389
1390         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1391                 m_freem(m);
1392                 return (ENETDOWN);
1393         }
1394
1395         if (m->m_flags & M_FLOWID)
1396                 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq))
1397                     + pi->rsrv_noflowq);
1398         br = txq->br;
1399
1400         if (TXQ_TRYLOCK(txq) == 0) {
1401                 struct sge_eq *eq = &txq->eq;
1402
1403                 /*
1404                  * It is possible that t4_eth_tx finishes up and releases the
1405                  * lock between the TRYLOCK above and the drbr_enqueue here.  We
1406                  * need to make sure that this mbuf doesn't just sit there in
1407                  * the drbr.
1408                  */
1409
1410                 rc = drbr_enqueue(ifp, br, m);
1411                 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 &&
1412                     !(eq->flags & EQ_DOOMED))
1413                         callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq);
1414                 return (rc);
1415         }
1416
1417         /*
1418          * txq->m is the mbuf that is held up due to a temporary shortage of
1419          * resources and it should be put on the wire first.  Then what's in
1420          * drbr and finally the mbuf that was just passed in to us.
1421          *
1422          * Return code should indicate the fate of the mbuf that was passed in
1423          * this time.
1424          */
1425
1426         TXQ_LOCK_ASSERT_OWNED(txq);
1427         if (drbr_needs_enqueue(ifp, br) || txq->m) {
1428
1429                 /* Queued for transmission. */
1430
1431                 rc = drbr_enqueue(ifp, br, m);
1432                 m = txq->m ? txq->m : drbr_dequeue(ifp, br);
1433                 (void) t4_eth_tx(ifp, txq, m);
1434                 TXQ_UNLOCK(txq);
1435                 return (rc);
1436         }
1437
1438         /* Direct transmission. */
1439         rc = t4_eth_tx(ifp, txq, m);
1440         if (rc != 0 && txq->m)
1441                 rc = 0; /* held, will be transmitted soon (hopefully) */
1442
1443         TXQ_UNLOCK(txq);
1444         return (rc);
1445 }
1446
1447 static void
1448 cxgbe_qflush(struct ifnet *ifp)
1449 {
1450         struct port_info *pi = ifp->if_softc;
1451         struct sge_txq *txq;
1452         int i;
1453         struct mbuf *m;
1454
1455         /* queues do not exist if !PORT_INIT_DONE. */
1456         if (pi->flags & PORT_INIT_DONE) {
1457                 for_each_txq(pi, i, txq) {
1458                         TXQ_LOCK(txq);
1459                         m_freem(txq->m);
1460                         txq->m = NULL;
1461                         while ((m = buf_ring_dequeue_sc(txq->br)) != NULL)
1462                                 m_freem(m);
1463                         TXQ_UNLOCK(txq);
1464                 }
1465         }
1466         if_qflush(ifp);
1467 }
1468
1469 static int
1470 cxgbe_media_change(struct ifnet *ifp)
1471 {
1472         struct port_info *pi = ifp->if_softc;
1473
1474         device_printf(pi->dev, "%s unimplemented.\n", __func__);
1475
1476         return (EOPNOTSUPP);
1477 }
1478
1479 static void
1480 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1481 {
1482         struct port_info *pi = ifp->if_softc;
1483         struct ifmedia *media = NULL;
1484         struct ifmedia_entry *cur;
1485         int speed = pi->link_cfg.speed;
1486         int data = (pi->port_type << 8) | pi->mod_type;
1487
1488         if (ifp == pi->ifp)
1489                 media = &pi->media;
1490 #ifdef DEV_NETMAP
1491         else if (ifp == pi->nm_ifp)
1492                 media = &pi->nm_media;
1493 #endif
1494         MPASS(media != NULL);
1495
1496         cur = media->ifm_cur;
1497         if (cur->ifm_data != data) {
1498                 build_medialist(pi, media);
1499                 cur = media->ifm_cur;
1500         }
1501
1502         ifmr->ifm_status = IFM_AVALID;
1503         if (!pi->link_cfg.link_ok)
1504                 return;
1505
1506         ifmr->ifm_status |= IFM_ACTIVE;
1507
1508         /* active and current will differ iff current media is autoselect. */
1509         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1510                 return;
1511
1512         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1513         if (speed == SPEED_10000)
1514                 ifmr->ifm_active |= IFM_10G_T;
1515         else if (speed == SPEED_1000)
1516                 ifmr->ifm_active |= IFM_1000_T;
1517         else if (speed == SPEED_100)
1518                 ifmr->ifm_active |= IFM_100_TX;
1519         else if (speed == SPEED_10)
1520                 ifmr->ifm_active |= IFM_10_T;
1521         else
1522                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1523                             speed));
1524 }
1525
1526 void
1527 t4_fatal_err(struct adapter *sc)
1528 {
1529         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1530         t4_intr_disable(sc);
1531         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1532             device_get_nameunit(sc->dev));
1533 }
1534
1535 static int
1536 map_bars_0_and_4(struct adapter *sc)
1537 {
1538         sc->regs_rid = PCIR_BAR(0);
1539         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1540             &sc->regs_rid, RF_ACTIVE);
1541         if (sc->regs_res == NULL) {
1542                 device_printf(sc->dev, "cannot map registers.\n");
1543                 return (ENXIO);
1544         }
1545         sc->bt = rman_get_bustag(sc->regs_res);
1546         sc->bh = rman_get_bushandle(sc->regs_res);
1547         sc->mmio_len = rman_get_size(sc->regs_res);
1548         setbit(&sc->doorbells, DOORBELL_KDB);
1549
1550         sc->msix_rid = PCIR_BAR(4);
1551         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1552             &sc->msix_rid, RF_ACTIVE);
1553         if (sc->msix_res == NULL) {
1554                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1555                 return (ENXIO);
1556         }
1557
1558         return (0);
1559 }
1560
1561 static int
1562 map_bar_2(struct adapter *sc)
1563 {
1564
1565         /*
1566          * T4: only iWARP driver uses the userspace doorbells.  There is no need
1567          * to map it if RDMA is disabled.
1568          */
1569         if (is_t4(sc) && sc->rdmacaps == 0)
1570                 return (0);
1571
1572         sc->udbs_rid = PCIR_BAR(2);
1573         sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1574             &sc->udbs_rid, RF_ACTIVE);
1575         if (sc->udbs_res == NULL) {
1576                 device_printf(sc->dev, "cannot map doorbell BAR.\n");
1577                 return (ENXIO);
1578         }
1579         sc->udbs_base = rman_get_virtual(sc->udbs_res);
1580
1581         if (is_t5(sc)) {
1582                 setbit(&sc->doorbells, DOORBELL_UDB);
1583 #if defined(__i386__) || defined(__amd64__)
1584                 if (t5_write_combine) {
1585                         int rc;
1586
1587                         /*
1588                          * Enable write combining on BAR2.  This is the
1589                          * userspace doorbell BAR and is split into 128B
1590                          * (UDBS_SEG_SIZE) doorbell regions, each associated
1591                          * with an egress queue.  The first 64B has the doorbell
1592                          * and the second 64B can be used to submit a tx work
1593                          * request with an implicit doorbell.
1594                          */
1595
1596                         rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
1597                             rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
1598                         if (rc == 0) {
1599                                 clrbit(&sc->doorbells, DOORBELL_UDB);
1600                                 setbit(&sc->doorbells, DOORBELL_WCWR);
1601                                 setbit(&sc->doorbells, DOORBELL_UDBWC);
1602                         } else {
1603                                 device_printf(sc->dev,
1604                                     "couldn't enable write combining: %d\n",
1605                                     rc);
1606                         }
1607
1608                         t4_write_reg(sc, A_SGE_STAT_CFG,
1609                             V_STATSOURCE_T5(7) | V_STATMODE(0));
1610                 }
1611 #endif
1612         }
1613
1614         return (0);
1615 }
1616
1617 static const struct memwin t4_memwin[] = {
1618         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1619         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1620         { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
1621 };
1622
1623 static const struct memwin t5_memwin[] = {
1624         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1625         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1626         { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1627 };
1628
1629 static void
1630 setup_memwin(struct adapter *sc)
1631 {
1632         const struct memwin *mw;
1633         int i, n;
1634         uint32_t bar0;
1635
1636         if (is_t4(sc)) {
1637                 /*
1638                  * Read low 32b of bar0 indirectly via the hardware backdoor
1639                  * mechanism.  Works from within PCI passthrough environments
1640                  * too, where rman_get_start() can return a different value.  We
1641                  * need to program the T4 memory window decoders with the actual
1642                  * addresses that will be coming across the PCIe link.
1643                  */
1644                 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1645                 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1646
1647                 mw = &t4_memwin[0];
1648                 n = nitems(t4_memwin);
1649         } else {
1650                 /* T5 uses the relative offset inside the PCIe BAR */
1651                 bar0 = 0;
1652
1653                 mw = &t5_memwin[0];
1654                 n = nitems(t5_memwin);
1655         }
1656
1657         for (i = 0; i < n; i++, mw++) {
1658                 t4_write_reg(sc,
1659                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
1660                     (mw->base + bar0) | V_BIR(0) |
1661                     V_WINDOW(ilog2(mw->aperture) - 10));
1662         }
1663
1664         /* flush */
1665         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1666 }
1667
1668 /*
1669  * Verify that the memory range specified by the addr/len pair is valid and lies
1670  * entirely within a single region (EDCx or MCx).
1671  */
1672 static int
1673 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
1674 {
1675         uint32_t em, addr_len, maddr, mlen;
1676
1677         /* Memory can only be accessed in naturally aligned 4 byte units */
1678         if (addr & 3 || len & 3 || len == 0)
1679                 return (EINVAL);
1680
1681         /* Enabled memories */
1682         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1683         if (em & F_EDRAM0_ENABLE) {
1684                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1685                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1686                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1687                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1688                     addr + len <= maddr + mlen)
1689                         return (0);
1690         }
1691         if (em & F_EDRAM1_ENABLE) {
1692                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1693                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1694                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1695                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1696                     addr + len <= maddr + mlen)
1697                         return (0);
1698         }
1699         if (em & F_EXT_MEM_ENABLE) {
1700                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1701                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1702                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1703                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1704                     addr + len <= maddr + mlen)
1705                         return (0);
1706         }
1707         if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
1708                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1709                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1710                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1711                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1712                     addr + len <= maddr + mlen)
1713                         return (0);
1714         }
1715
1716         return (EFAULT);
1717 }
1718
1719 static int
1720 fwmtype_to_hwmtype(int mtype)
1721 {
1722
1723         switch (mtype) {
1724         case FW_MEMTYPE_EDC0:
1725                 return (MEM_EDC0);
1726         case FW_MEMTYPE_EDC1:
1727                 return (MEM_EDC1);
1728         case FW_MEMTYPE_EXTMEM:
1729                 return (MEM_MC0);
1730         case FW_MEMTYPE_EXTMEM1:
1731                 return (MEM_MC1);
1732         default:
1733                 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
1734         }
1735 }
1736
1737 /*
1738  * Verify that the memory range specified by the memtype/offset/len pair is
1739  * valid and lies entirely within the memtype specified.  The global address of
1740  * the start of the range is returned in addr.
1741  */
1742 static int
1743 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1744     uint32_t *addr)
1745 {
1746         uint32_t em, addr_len, maddr, mlen;
1747
1748         /* Memory can only be accessed in naturally aligned 4 byte units */
1749         if (off & 3 || len & 3 || len == 0)
1750                 return (EINVAL);
1751
1752         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1753         switch (fwmtype_to_hwmtype(mtype)) {
1754         case MEM_EDC0:
1755                 if (!(em & F_EDRAM0_ENABLE))
1756                         return (EINVAL);
1757                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1758                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1759                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1760                 break;
1761         case MEM_EDC1:
1762                 if (!(em & F_EDRAM1_ENABLE))
1763                         return (EINVAL);
1764                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1765                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1766                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1767                 break;
1768         case MEM_MC:
1769                 if (!(em & F_EXT_MEM_ENABLE))
1770                         return (EINVAL);
1771                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1772                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1773                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1774                 break;
1775         case MEM_MC1:
1776                 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
1777                         return (EINVAL);
1778                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1779                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1780                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1781                 break;
1782         default:
1783                 return (EINVAL);
1784         }
1785
1786         if (mlen > 0 && off < mlen && off + len <= mlen) {
1787                 *addr = maddr + off;    /* global address */
1788                 return (0);
1789         }
1790
1791         return (EFAULT);
1792 }
1793
1794 static void
1795 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1796 {
1797         const struct memwin *mw;
1798
1799         if (is_t4(sc)) {
1800                 KASSERT(win >= 0 && win < nitems(t4_memwin),
1801                     ("%s: incorrect memwin# (%d)", __func__, win));
1802                 mw = &t4_memwin[win];
1803         } else {
1804                 KASSERT(win >= 0 && win < nitems(t5_memwin),
1805                     ("%s: incorrect memwin# (%d)", __func__, win));
1806                 mw = &t5_memwin[win];
1807         }
1808
1809         if (base != NULL)
1810                 *base = mw->base;
1811         if (aperture != NULL)
1812                 *aperture = mw->aperture;
1813 }
1814
1815 /*
1816  * Positions the memory window such that it can be used to access the specified
1817  * address in the chip's address space.  The return value is the offset of addr
1818  * from the start of the window.
1819  */
1820 static uint32_t
1821 position_memwin(struct adapter *sc, int n, uint32_t addr)
1822 {
1823         uint32_t start, pf;
1824         uint32_t reg;
1825
1826         KASSERT(n >= 0 && n <= 3,
1827             ("%s: invalid window %d.", __func__, n));
1828         KASSERT((addr & 3) == 0,
1829             ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
1830
1831         if (is_t4(sc)) {
1832                 pf = 0;
1833                 start = addr & ~0xf;    /* start must be 16B aligned */
1834         } else {
1835                 pf = V_PFNUM(sc->pf);
1836                 start = addr & ~0x7f;   /* start must be 128B aligned */
1837         }
1838         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1839
1840         t4_write_reg(sc, reg, start | pf);
1841         t4_read_reg(sc, reg);
1842
1843         return (addr - start);
1844 }
1845
1846 static int
1847 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1848     struct intrs_and_queues *iaq)
1849 {
1850         int rc, itype, navail, nrxq10g, nrxq1g, n;
1851         int nofldrxq10g = 0, nofldrxq1g = 0;
1852         int nnmrxq10g = 0, nnmrxq1g = 0;
1853
1854         bzero(iaq, sizeof(*iaq));
1855
1856         iaq->ntxq10g = t4_ntxq10g;
1857         iaq->ntxq1g = t4_ntxq1g;
1858         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1859         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1860         iaq->rsrv_noflowq = t4_rsrv_noflowq;
1861 #ifdef TCP_OFFLOAD
1862         if (is_offload(sc)) {
1863                 iaq->nofldtxq10g = t4_nofldtxq10g;
1864                 iaq->nofldtxq1g = t4_nofldtxq1g;
1865                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1866                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1867         }
1868 #endif
1869 #ifdef DEV_NETMAP
1870         iaq->nnmtxq10g = t4_nnmtxq10g;
1871         iaq->nnmtxq1g = t4_nnmtxq1g;
1872         iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g;
1873         iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g;
1874 #endif
1875
1876         for (itype = INTR_MSIX; itype; itype >>= 1) {
1877
1878                 if ((itype & t4_intr_types) == 0)
1879                         continue;       /* not allowed */
1880
1881                 if (itype == INTR_MSIX)
1882                         navail = pci_msix_count(sc->dev);
1883                 else if (itype == INTR_MSI)
1884                         navail = pci_msi_count(sc->dev);
1885                 else
1886                         navail = 1;
1887 restart:
1888                 if (navail == 0)
1889                         continue;
1890
1891                 iaq->intr_type = itype;
1892                 iaq->intr_flags_10g = 0;
1893                 iaq->intr_flags_1g = 0;
1894
1895                 /*
1896                  * Best option: an interrupt vector for errors, one for the
1897                  * firmware event queue, and one for every rxq (NIC, TOE, and
1898                  * netmap).
1899                  */
1900                 iaq->nirq = T4_EXTRA_INTR;
1901                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g);
1902                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g);
1903                 if (iaq->nirq <= navail &&
1904                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
1905                         iaq->intr_flags_10g = INTR_ALL;
1906                         iaq->intr_flags_1g = INTR_ALL;
1907                         goto allocate;
1908                 }
1909
1910                 /*
1911                  * Second best option: a vector for errors, one for the firmware
1912                  * event queue, and vectors for either all the NIC rx queues or
1913                  * all the TOE rx queues.  The queues that don't get vectors
1914                  * will forward their interrupts to those that do.
1915                  *
1916                  * Note: netmap rx queues cannot be created early and so they
1917                  * can't be setup to receive forwarded interrupts for others.
1918                  */
1919                 iaq->nirq = T4_EXTRA_INTR;
1920                 if (nrxq10g >= nofldrxq10g) {
1921                         iaq->intr_flags_10g = INTR_RXQ;
1922                         iaq->nirq += n10g * nrxq10g;
1923 #ifdef DEV_NETMAP
1924                         iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g);
1925 #endif
1926                 } else {
1927                         iaq->intr_flags_10g = INTR_OFLD_RXQ;
1928                         iaq->nirq += n10g * nofldrxq10g;
1929 #ifdef DEV_NETMAP
1930                         iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g);
1931 #endif
1932                 }
1933                 if (nrxq1g >= nofldrxq1g) {
1934                         iaq->intr_flags_1g = INTR_RXQ;
1935                         iaq->nirq += n1g * nrxq1g;
1936 #ifdef DEV_NETMAP
1937                         iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g);
1938 #endif
1939                 } else {
1940                         iaq->intr_flags_1g = INTR_OFLD_RXQ;
1941                         iaq->nirq += n1g * nofldrxq1g;
1942 #ifdef DEV_NETMAP
1943                         iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g);
1944 #endif
1945                 }
1946                 if (iaq->nirq <= navail &&
1947                     (itype != INTR_MSI || powerof2(iaq->nirq)))
1948                         goto allocate;
1949
1950                 /*
1951                  * Next best option: an interrupt vector for errors, one for the
1952                  * firmware event queue, and at least one per port.  At this
1953                  * point we know we'll have to downsize nrxq and/or nofldrxq
1954                  * and/or nnmrxq to fit what's available to us.
1955                  */
1956                 iaq->nirq = T4_EXTRA_INTR;
1957                 iaq->nirq += n10g + n1g;
1958                 if (iaq->nirq <= navail) {
1959                         int leftover = navail - iaq->nirq;
1960
1961                         if (n10g > 0) {
1962                                 int target = max(nrxq10g, nofldrxq10g);
1963
1964                                 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
1965                                     INTR_RXQ : INTR_OFLD_RXQ;
1966
1967                                 n = 1;
1968                                 while (n < target && leftover >= n10g) {
1969                                         leftover -= n10g;
1970                                         iaq->nirq += n10g;
1971                                         n++;
1972                                 }
1973                                 iaq->nrxq10g = min(n, nrxq10g);
1974 #ifdef TCP_OFFLOAD
1975                                 iaq->nofldrxq10g = min(n, nofldrxq10g);
1976 #endif
1977 #ifdef DEV_NETMAP
1978                                 iaq->nnmrxq10g = min(n, nnmrxq10g);
1979 #endif
1980                         }
1981
1982                         if (n1g > 0) {
1983                                 int target = max(nrxq1g, nofldrxq1g);
1984
1985                                 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
1986                                     INTR_RXQ : INTR_OFLD_RXQ;
1987
1988                                 n = 1;
1989                                 while (n < target && leftover >= n1g) {
1990                                         leftover -= n1g;
1991                                         iaq->nirq += n1g;
1992                                         n++;
1993                                 }
1994                                 iaq->nrxq1g = min(n, nrxq1g);
1995 #ifdef TCP_OFFLOAD
1996                                 iaq->nofldrxq1g = min(n, nofldrxq1g);
1997 #endif
1998 #ifdef DEV_NETMAP
1999                                 iaq->nnmrxq1g = min(n, nnmrxq1g);
2000 #endif
2001                         }
2002
2003                         if (itype != INTR_MSI || powerof2(iaq->nirq))
2004                                 goto allocate;
2005                 }
2006
2007                 /*
2008                  * Least desirable option: one interrupt vector for everything.
2009                  */
2010                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
2011                 iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
2012 #ifdef TCP_OFFLOAD
2013                 if (is_offload(sc))
2014                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2015 #endif
2016 #ifdef DEV_NETMAP
2017                 iaq->nnmrxq10g = iaq->nnmrxq1g = 1;
2018 #endif
2019
2020 allocate:
2021                 navail = iaq->nirq;
2022                 rc = 0;
2023                 if (itype == INTR_MSIX)
2024                         rc = pci_alloc_msix(sc->dev, &navail);
2025                 else if (itype == INTR_MSI)
2026                         rc = pci_alloc_msi(sc->dev, &navail);
2027
2028                 if (rc == 0) {
2029                         if (navail == iaq->nirq)
2030                                 return (0);
2031
2032                         /*
2033                          * Didn't get the number requested.  Use whatever number
2034                          * the kernel is willing to allocate (it's in navail).
2035                          */
2036                         device_printf(sc->dev, "fewer vectors than requested, "
2037                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
2038                             itype, iaq->nirq, navail);
2039                         pci_release_msi(sc->dev);
2040                         goto restart;
2041                 }
2042
2043                 device_printf(sc->dev,
2044                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
2045                     itype, rc, iaq->nirq, navail);
2046         }
2047
2048         device_printf(sc->dev,
2049             "failed to find a usable interrupt type.  "
2050             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
2051             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
2052
2053         return (ENXIO);
2054 }
2055
2056 #define FW_VERSION(chip) ( \
2057     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
2058     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
2059     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
2060     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
2061 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
2062
2063 struct fw_info {
2064         uint8_t chip;
2065         char *kld_name;
2066         char *fw_mod_name;
2067         struct fw_hdr fw_hdr;   /* XXX: waste of space, need a sparse struct */
2068 } fw_info[] = {
2069         {
2070                 .chip = CHELSIO_T4,
2071                 .kld_name = "t4fw_cfg",
2072                 .fw_mod_name = "t4fw",
2073                 .fw_hdr = {
2074                         .chip = FW_HDR_CHIP_T4,
2075                         .fw_ver = htobe32_const(FW_VERSION(T4)),
2076                         .intfver_nic = FW_INTFVER(T4, NIC),
2077                         .intfver_vnic = FW_INTFVER(T4, VNIC),
2078                         .intfver_ofld = FW_INTFVER(T4, OFLD),
2079                         .intfver_ri = FW_INTFVER(T4, RI),
2080                         .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
2081                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
2082                         .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
2083                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
2084                 },
2085         }, {
2086                 .chip = CHELSIO_T5,
2087                 .kld_name = "t5fw_cfg",
2088                 .fw_mod_name = "t5fw",
2089                 .fw_hdr = {
2090                         .chip = FW_HDR_CHIP_T5,
2091                         .fw_ver = htobe32_const(FW_VERSION(T5)),
2092                         .intfver_nic = FW_INTFVER(T5, NIC),
2093                         .intfver_vnic = FW_INTFVER(T5, VNIC),
2094                         .intfver_ofld = FW_INTFVER(T5, OFLD),
2095                         .intfver_ri = FW_INTFVER(T5, RI),
2096                         .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
2097                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
2098                         .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
2099                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
2100                 },
2101         }
2102 };
2103
2104 static struct fw_info *
2105 find_fw_info(int chip)
2106 {
2107         int i;
2108
2109         for (i = 0; i < nitems(fw_info); i++) {
2110                 if (fw_info[i].chip == chip)
2111                         return (&fw_info[i]);
2112         }
2113         return (NULL);
2114 }
2115
2116 /*
2117  * Is the given firmware API compatible with the one the driver was compiled
2118  * with?
2119  */
2120 static int
2121 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
2122 {
2123
2124         /* short circuit if it's the exact same firmware version */
2125         if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
2126                 return (1);
2127
2128         /*
2129          * XXX: Is this too conservative?  Perhaps I should limit this to the
2130          * features that are supported in the driver.
2131          */
2132 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
2133         if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
2134             SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
2135             SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
2136                 return (1);
2137 #undef SAME_INTF
2138
2139         return (0);
2140 }
2141
2142 /*
2143  * The firmware in the KLD is usable, but should it be installed?  This routine
2144  * explains itself in detail if it indicates the KLD firmware should be
2145  * installed.
2146  */
2147 static int
2148 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
2149 {
2150         const char *reason;
2151
2152         if (!card_fw_usable) {
2153                 reason = "incompatible or unusable";
2154                 goto install;
2155         }
2156
2157         if (k > c) {
2158                 reason = "older than the version bundled with this driver";
2159                 goto install;
2160         }
2161
2162         if (t4_fw_install == 2 && k != c) {
2163                 reason = "different than the version bundled with this driver";
2164                 goto install;
2165         }
2166
2167         return (0);
2168
2169 install:
2170         if (t4_fw_install == 0) {
2171                 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2172                     "but the driver is prohibited from installing a different "
2173                     "firmware on the card.\n",
2174                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2175                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2176
2177                 return (0);
2178         }
2179
2180         device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2181             "installing firmware %u.%u.%u.%u on card.\n",
2182             G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2183             G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2184             G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2185             G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2186
2187         return (1);
2188 }
2189 /*
2190  * Establish contact with the firmware and determine if we are the master driver
2191  * or not, and whether we are responsible for chip initialization.
2192  */
2193 static int
2194 prep_firmware(struct adapter *sc)
2195 {
2196         const struct firmware *fw = NULL, *default_cfg;
2197         int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2198         enum dev_state state;
2199         struct fw_info *fw_info;
2200         struct fw_hdr *card_fw;         /* fw on the card */
2201         const struct fw_hdr *kld_fw;    /* fw in the KLD */
2202         const struct fw_hdr *drv_fw;    /* fw header the driver was compiled
2203                                            against */
2204
2205         /* Contact firmware. */
2206         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2207         if (rc < 0 || state == DEV_STATE_ERR) {
2208                 rc = -rc;
2209                 device_printf(sc->dev,
2210                     "failed to connect to the firmware: %d, %d.\n", rc, state);
2211                 return (rc);
2212         }
2213         pf = rc;
2214         if (pf == sc->mbox)
2215                 sc->flags |= MASTER_PF;
2216         else if (state == DEV_STATE_UNINIT) {
2217                 /*
2218                  * We didn't get to be the master so we definitely won't be
2219                  * configuring the chip.  It's a bug if someone else hasn't
2220                  * configured it already.
2221                  */
2222                 device_printf(sc->dev, "couldn't be master(%d), "
2223                     "device not already initialized either(%d).\n", rc, state);
2224                 return (EDOOFUS);
2225         }
2226
2227         /* This is the firmware whose headers the driver was compiled against */
2228         fw_info = find_fw_info(chip_id(sc));
2229         if (fw_info == NULL) {
2230                 device_printf(sc->dev,
2231                     "unable to look up firmware information for chip %d.\n",
2232                     chip_id(sc));
2233                 return (EINVAL);
2234         }
2235         drv_fw = &fw_info->fw_hdr;
2236
2237         /*
2238          * The firmware KLD contains many modules.  The KLD name is also the
2239          * name of the module that contains the default config file.
2240          */
2241         default_cfg = firmware_get(fw_info->kld_name);
2242
2243         /* Read the header of the firmware on the card */
2244         card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2245         rc = -t4_read_flash(sc, FLASH_FW_START,
2246             sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2247         if (rc == 0)
2248                 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2249         else {
2250                 device_printf(sc->dev,
2251                     "Unable to read card's firmware header: %d\n", rc);
2252                 card_fw_usable = 0;
2253         }
2254
2255         /* This is the firmware in the KLD */
2256         fw = firmware_get(fw_info->fw_mod_name);
2257         if (fw != NULL) {
2258                 kld_fw = (const void *)fw->data;
2259                 kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2260         } else {
2261                 kld_fw = NULL;
2262                 kld_fw_usable = 0;
2263         }
2264
2265         if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2266             (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2267                 /*
2268                  * Common case: the firmware on the card is an exact match and
2269                  * the KLD is an exact match too, or the KLD is
2270                  * absent/incompatible.  Note that t4_fw_install = 2 is ignored
2271                  * here -- use cxgbetool loadfw if you want to reinstall the
2272                  * same firmware as the one on the card.
2273                  */
2274         } else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2275             should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2276             be32toh(card_fw->fw_ver))) {
2277
2278                 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
2279                 if (rc != 0) {
2280                         device_printf(sc->dev,
2281                             "failed to install firmware: %d\n", rc);
2282                         goto done;
2283                 }
2284
2285                 /* Installed successfully, update the cached header too. */
2286                 memcpy(card_fw, kld_fw, sizeof(*card_fw));
2287                 card_fw_usable = 1;
2288                 need_fw_reset = 0;      /* already reset as part of load_fw */
2289         }
2290
2291         if (!card_fw_usable) {
2292                 uint32_t d, c, k;
2293
2294                 d = ntohl(drv_fw->fw_ver);
2295                 c = ntohl(card_fw->fw_ver);
2296                 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
2297
2298                 device_printf(sc->dev, "Cannot find a usable firmware: "
2299                     "fw_install %d, chip state %d, "
2300                     "driver compiled with %d.%d.%d.%d, "
2301                     "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
2302                     t4_fw_install, state,
2303                     G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
2304                     G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
2305                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2306                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
2307                     G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2308                     G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2309                 rc = EINVAL;
2310                 goto done;
2311         }
2312
2313         /* We're using whatever's on the card and it's known to be good. */
2314         sc->params.fw_vers = ntohl(card_fw->fw_ver);
2315         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
2316             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2317             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2318             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2319             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2320         t4_get_tp_version(sc, &sc->params.tp_vers);
2321
2322         /* Reset device */
2323         if (need_fw_reset &&
2324             (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
2325                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
2326                 if (rc != ETIMEDOUT && rc != EIO)
2327                         t4_fw_bye(sc, sc->mbox);
2328                 goto done;
2329         }
2330         sc->flags |= FW_OK;
2331
2332         rc = get_params__pre_init(sc);
2333         if (rc != 0)
2334                 goto done; /* error message displayed already */
2335
2336         /* Partition adapter resources as specified in the config file. */
2337         if (state == DEV_STATE_UNINIT) {
2338
2339                 KASSERT(sc->flags & MASTER_PF,
2340                     ("%s: trying to change chip settings when not master.",
2341                     __func__));
2342
2343                 rc = partition_resources(sc, default_cfg, fw_info->kld_name);
2344                 if (rc != 0)
2345                         goto done;      /* error message displayed already */
2346
2347                 t4_tweak_chip_settings(sc);
2348
2349                 /* get basic stuff going */
2350                 rc = -t4_fw_initialize(sc, sc->mbox);
2351                 if (rc != 0) {
2352                         device_printf(sc->dev, "fw init failed: %d.\n", rc);
2353                         goto done;
2354                 }
2355         } else {
2356                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
2357                 sc->cfcsum = 0;
2358         }
2359
2360 done:
2361         free(card_fw, M_CXGBE);
2362         if (fw != NULL)
2363                 firmware_put(fw, FIRMWARE_UNLOAD);
2364         if (default_cfg != NULL)
2365                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
2366
2367         return (rc);
2368 }
2369
2370 #define FW_PARAM_DEV(param) \
2371         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2372          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2373 #define FW_PARAM_PFVF(param) \
2374         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2375          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2376
2377 /*
2378  * Partition chip resources for use between various PFs, VFs, etc.
2379  */
2380 static int
2381 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
2382     const char *name_prefix)
2383 {
2384         const struct firmware *cfg = NULL;
2385         int rc = 0;
2386         struct fw_caps_config_cmd caps;
2387         uint32_t mtype, moff, finicsum, cfcsum;
2388
2389         /*
2390          * Figure out what configuration file to use.  Pick the default config
2391          * file for the card if the user hasn't specified one explicitly.
2392          */
2393         snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
2394         if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
2395                 /* Card specific overrides go here. */
2396                 if (pci_get_device(sc->dev) == 0x440a)
2397                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
2398                 if (is_fpga(sc))
2399                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
2400         }
2401
2402         /*
2403          * We need to load another module if the profile is anything except
2404          * "default" or "flash".
2405          */
2406         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
2407             strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2408                 char s[32];
2409
2410                 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
2411                 cfg = firmware_get(s);
2412                 if (cfg == NULL) {
2413                         if (default_cfg != NULL) {
2414                                 device_printf(sc->dev,
2415                                     "unable to load module \"%s\" for "
2416                                     "configuration profile \"%s\", will use "
2417                                     "the default config file instead.\n",
2418                                     s, sc->cfg_file);
2419                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2420                                     "%s", DEFAULT_CF);
2421                         } else {
2422                                 device_printf(sc->dev,
2423                                     "unable to load module \"%s\" for "
2424                                     "configuration profile \"%s\", will use "
2425                                     "the config file on the card's flash "
2426                                     "instead.\n", s, sc->cfg_file);
2427                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2428                                     "%s", FLASH_CF);
2429                         }
2430                 }
2431         }
2432
2433         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
2434             default_cfg == NULL) {
2435                 device_printf(sc->dev,
2436                     "default config file not available, will use the config "
2437                     "file on the card's flash instead.\n");
2438                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
2439         }
2440
2441         if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2442                 u_int cflen, i, n;
2443                 const uint32_t *cfdata;
2444                 uint32_t param, val, addr, off, mw_base, mw_aperture;
2445
2446                 KASSERT(cfg != NULL || default_cfg != NULL,
2447                     ("%s: no config to upload", __func__));
2448
2449                 /*
2450                  * Ask the firmware where it wants us to upload the config file.
2451                  */
2452                 param = FW_PARAM_DEV(CF);
2453                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2454                 if (rc != 0) {
2455                         /* No support for config file?  Shouldn't happen. */
2456                         device_printf(sc->dev,
2457                             "failed to query config file location: %d.\n", rc);
2458                         goto done;
2459                 }
2460                 mtype = G_FW_PARAMS_PARAM_Y(val);
2461                 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
2462
2463                 /*
2464                  * XXX: sheer laziness.  We deliberately added 4 bytes of
2465                  * useless stuffing/comments at the end of the config file so
2466                  * it's ok to simply throw away the last remaining bytes when
2467                  * the config file is not an exact multiple of 4.  This also
2468                  * helps with the validate_mt_off_len check.
2469                  */
2470                 if (cfg != NULL) {
2471                         cflen = cfg->datasize & ~3;
2472                         cfdata = cfg->data;
2473                 } else {
2474                         cflen = default_cfg->datasize & ~3;
2475                         cfdata = default_cfg->data;
2476                 }
2477
2478                 if (cflen > FLASH_CFG_MAX_SIZE) {
2479                         device_printf(sc->dev,
2480                             "config file too long (%d, max allowed is %d).  "
2481                             "Will try to use the config on the card, if any.\n",
2482                             cflen, FLASH_CFG_MAX_SIZE);
2483                         goto use_config_on_flash;
2484                 }
2485
2486                 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
2487                 if (rc != 0) {
2488                         device_printf(sc->dev,
2489                             "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
2490                             "Will try to use the config on the card, if any.\n",
2491                             __func__, mtype, moff, cflen, rc);
2492                         goto use_config_on_flash;
2493                 }
2494
2495                 memwin_info(sc, 2, &mw_base, &mw_aperture);
2496                 while (cflen) {
2497                         off = position_memwin(sc, 2, addr);
2498                         n = min(cflen, mw_aperture - off);
2499                         for (i = 0; i < n; i += 4)
2500                                 t4_write_reg(sc, mw_base + off + i, *cfdata++);
2501                         cflen -= n;
2502                         addr += n;
2503                 }
2504         } else {
2505 use_config_on_flash:
2506                 mtype = FW_MEMTYPE_FLASH;
2507                 moff = t4_flash_cfg_addr(sc);
2508         }
2509
2510         bzero(&caps, sizeof(caps));
2511         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2512             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2513         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
2514             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
2515             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
2516         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2517         if (rc != 0) {
2518                 device_printf(sc->dev,
2519                     "failed to pre-process config file: %d "
2520                     "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
2521                 goto done;
2522         }
2523
2524         finicsum = be32toh(caps.finicsum);
2525         cfcsum = be32toh(caps.cfcsum);
2526         if (finicsum != cfcsum) {
2527                 device_printf(sc->dev,
2528                     "WARNING: config file checksum mismatch: %08x %08x\n",
2529                     finicsum, cfcsum);
2530         }
2531         sc->cfcsum = cfcsum;
2532
2533 #define LIMIT_CAPS(x) do { \
2534         caps.x &= htobe16(t4_##x##_allowed); \
2535 } while (0)
2536
2537         /*
2538          * Let the firmware know what features will (not) be used so it can tune
2539          * things accordingly.
2540          */
2541         LIMIT_CAPS(linkcaps);
2542         LIMIT_CAPS(niccaps);
2543         LIMIT_CAPS(toecaps);
2544         LIMIT_CAPS(rdmacaps);
2545         LIMIT_CAPS(iscsicaps);
2546         LIMIT_CAPS(fcoecaps);
2547 #undef LIMIT_CAPS
2548
2549         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2550             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
2551         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2552         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
2553         if (rc != 0) {
2554                 device_printf(sc->dev,
2555                     "failed to process config file: %d.\n", rc);
2556         }
2557 done:
2558         if (cfg != NULL)
2559                 firmware_put(cfg, FIRMWARE_UNLOAD);
2560         return (rc);
2561 }
2562
2563 /*
2564  * Retrieve parameters that are needed (or nice to have) very early.
2565  */
2566 static int
2567 get_params__pre_init(struct adapter *sc)
2568 {
2569         int rc;
2570         uint32_t param[2], val[2];
2571         struct fw_devlog_cmd cmd;
2572         struct devlog_params *dlog = &sc->params.devlog;
2573
2574         param[0] = FW_PARAM_DEV(PORTVEC);
2575         param[1] = FW_PARAM_DEV(CCLK);
2576         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2577         if (rc != 0) {
2578                 device_printf(sc->dev,
2579                     "failed to query parameters (pre_init): %d.\n", rc);
2580                 return (rc);
2581         }
2582
2583         sc->params.portvec = val[0];
2584         sc->params.nports = bitcount32(val[0]);
2585         sc->params.vpd.cclk = val[1];
2586
2587         /* Read device log parameters. */
2588         bzero(&cmd, sizeof(cmd));
2589         cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
2590             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2591         cmd.retval_len16 = htobe32(FW_LEN16(cmd));
2592         rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
2593         if (rc != 0) {
2594                 device_printf(sc->dev,
2595                     "failed to get devlog parameters: %d.\n", rc);
2596                 bzero(dlog, sizeof (*dlog));
2597                 rc = 0; /* devlog isn't critical for device operation */
2598         } else {
2599                 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
2600                 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
2601                 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
2602                 dlog->size = be32toh(cmd.memsize_devlog);
2603         }
2604
2605         return (rc);
2606 }
2607
2608 /*
2609  * Retrieve various parameters that are of interest to the driver.  The device
2610  * has been initialized by the firmware at this point.
2611  */
2612 static int
2613 get_params__post_init(struct adapter *sc)
2614 {
2615         int rc;
2616         uint32_t param[7], val[7];
2617         struct fw_caps_config_cmd caps;
2618
2619         param[0] = FW_PARAM_PFVF(IQFLINT_START);
2620         param[1] = FW_PARAM_PFVF(EQ_START);
2621         param[2] = FW_PARAM_PFVF(FILTER_START);
2622         param[3] = FW_PARAM_PFVF(FILTER_END);
2623         param[4] = FW_PARAM_PFVF(L2T_START);
2624         param[5] = FW_PARAM_PFVF(L2T_END);
2625         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2626         if (rc != 0) {
2627                 device_printf(sc->dev,
2628                     "failed to query parameters (post_init): %d.\n", rc);
2629                 return (rc);
2630         }
2631
2632         sc->sge.iq_start = val[0];
2633         sc->sge.eq_start = val[1];
2634         sc->tids.ftid_base = val[2];
2635         sc->tids.nftids = val[3] - val[2] + 1;
2636         sc->params.ftid_min = val[2];
2637         sc->params.ftid_max = val[3];
2638         sc->vres.l2t.start = val[4];
2639         sc->vres.l2t.size = val[5] - val[4] + 1;
2640         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
2641             ("%s: L2 table size (%u) larger than expected (%u)",
2642             __func__, sc->vres.l2t.size, L2T_SIZE));
2643
2644         /* get capabilites */
2645         bzero(&caps, sizeof(caps));
2646         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2647             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2648         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2649         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2650         if (rc != 0) {
2651                 device_printf(sc->dev,
2652                     "failed to get card capabilities: %d.\n", rc);
2653                 return (rc);
2654         }
2655
2656 #define READ_CAPS(x) do { \
2657         sc->x = htobe16(caps.x); \
2658 } while (0)
2659         READ_CAPS(linkcaps);
2660         READ_CAPS(niccaps);
2661         READ_CAPS(toecaps);
2662         READ_CAPS(rdmacaps);
2663         READ_CAPS(iscsicaps);
2664         READ_CAPS(fcoecaps);
2665
2666         if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
2667                 param[0] = FW_PARAM_PFVF(ETHOFLD_START);
2668                 param[1] = FW_PARAM_PFVF(ETHOFLD_END);
2669                 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2670                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
2671                 if (rc != 0) {
2672                         device_printf(sc->dev,
2673                             "failed to query NIC parameters: %d.\n", rc);
2674                         return (rc);
2675                 }
2676                 sc->tids.etid_base = val[0];
2677                 sc->params.etid_min = val[0];
2678                 sc->tids.netids = val[1] - val[0] + 1;
2679                 sc->params.netids = sc->tids.netids;
2680                 sc->params.eo_wr_cred = val[2];
2681                 sc->params.ethoffload = 1;
2682         }
2683
2684         if (sc->toecaps) {
2685                 /* query offload-related parameters */
2686                 param[0] = FW_PARAM_DEV(NTID);
2687                 param[1] = FW_PARAM_PFVF(SERVER_START);
2688                 param[2] = FW_PARAM_PFVF(SERVER_END);
2689                 param[3] = FW_PARAM_PFVF(TDDP_START);
2690                 param[4] = FW_PARAM_PFVF(TDDP_END);
2691                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2692                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2693                 if (rc != 0) {
2694                         device_printf(sc->dev,
2695                             "failed to query TOE parameters: %d.\n", rc);
2696                         return (rc);
2697                 }
2698                 sc->tids.ntids = val[0];
2699                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
2700                 sc->tids.stid_base = val[1];
2701                 sc->tids.nstids = val[2] - val[1] + 1;
2702                 sc->vres.ddp.start = val[3];
2703                 sc->vres.ddp.size = val[4] - val[3] + 1;
2704                 sc->params.ofldq_wr_cred = val[5];
2705                 sc->params.offload = 1;
2706         }
2707         if (sc->rdmacaps) {
2708                 param[0] = FW_PARAM_PFVF(STAG_START);
2709                 param[1] = FW_PARAM_PFVF(STAG_END);
2710                 param[2] = FW_PARAM_PFVF(RQ_START);
2711                 param[3] = FW_PARAM_PFVF(RQ_END);
2712                 param[4] = FW_PARAM_PFVF(PBL_START);
2713                 param[5] = FW_PARAM_PFVF(PBL_END);
2714                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2715                 if (rc != 0) {
2716                         device_printf(sc->dev,
2717                             "failed to query RDMA parameters(1): %d.\n", rc);
2718                         return (rc);
2719                 }
2720                 sc->vres.stag.start = val[0];
2721                 sc->vres.stag.size = val[1] - val[0] + 1;
2722                 sc->vres.rq.start = val[2];
2723                 sc->vres.rq.size = val[3] - val[2] + 1;
2724                 sc->vres.pbl.start = val[4];
2725                 sc->vres.pbl.size = val[5] - val[4] + 1;
2726
2727                 param[0] = FW_PARAM_PFVF(SQRQ_START);
2728                 param[1] = FW_PARAM_PFVF(SQRQ_END);
2729                 param[2] = FW_PARAM_PFVF(CQ_START);
2730                 param[3] = FW_PARAM_PFVF(CQ_END);
2731                 param[4] = FW_PARAM_PFVF(OCQ_START);
2732                 param[5] = FW_PARAM_PFVF(OCQ_END);
2733                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2734                 if (rc != 0) {
2735                         device_printf(sc->dev,
2736                             "failed to query RDMA parameters(2): %d.\n", rc);
2737                         return (rc);
2738                 }
2739                 sc->vres.qp.start = val[0];
2740                 sc->vres.qp.size = val[1] - val[0] + 1;
2741                 sc->vres.cq.start = val[2];
2742                 sc->vres.cq.size = val[3] - val[2] + 1;
2743                 sc->vres.ocq.start = val[4];
2744                 sc->vres.ocq.size = val[5] - val[4] + 1;
2745         }
2746         if (sc->iscsicaps) {
2747                 param[0] = FW_PARAM_PFVF(ISCSI_START);
2748                 param[1] = FW_PARAM_PFVF(ISCSI_END);
2749                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2750                 if (rc != 0) {
2751                         device_printf(sc->dev,
2752                             "failed to query iSCSI parameters: %d.\n", rc);
2753                         return (rc);
2754                 }
2755                 sc->vres.iscsi.start = val[0];
2756                 sc->vres.iscsi.size = val[1] - val[0] + 1;
2757         }
2758
2759         /*
2760          * We've got the params we wanted to query via the firmware.  Now grab
2761          * some others directly from the chip.
2762          */
2763         rc = t4_read_chip_settings(sc);
2764
2765         return (rc);
2766 }
2767
2768 static int
2769 set_params__post_init(struct adapter *sc)
2770 {
2771         uint32_t param, val;
2772
2773         /* ask for encapsulated CPLs */
2774         param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
2775         val = 1;
2776         (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2777
2778         return (0);
2779 }
2780
2781 #undef FW_PARAM_PFVF
2782 #undef FW_PARAM_DEV
2783
2784 static void
2785 t4_set_desc(struct adapter *sc)
2786 {
2787         char buf[128];
2788         struct adapter_params *p = &sc->params;
2789
2790         snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
2791             "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
2792             chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
2793
2794         device_set_desc_copy(sc->dev, buf);
2795 }
2796
2797 static void
2798 build_medialist(struct port_info *pi, struct ifmedia *media)
2799 {
2800         int data, m;
2801
2802         PORT_LOCK(pi);
2803
2804         ifmedia_removeall(media);
2805
2806         m = IFM_ETHER | IFM_FDX;
2807         data = (pi->port_type << 8) | pi->mod_type;
2808
2809         switch(pi->port_type) {
2810         case FW_PORT_TYPE_BT_XFI:
2811                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2812                 break;
2813
2814         case FW_PORT_TYPE_BT_XAUI:
2815                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2816                 /* fall through */
2817
2818         case FW_PORT_TYPE_BT_SGMII:
2819                 ifmedia_add(media, m | IFM_1000_T, data, NULL);
2820                 ifmedia_add(media, m | IFM_100_TX, data, NULL);
2821                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL);
2822                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2823                 break;
2824
2825         case FW_PORT_TYPE_CX4:
2826                 ifmedia_add(media, m | IFM_10G_CX4, data, NULL);
2827                 ifmedia_set(media, m | IFM_10G_CX4);
2828                 break;
2829
2830         case FW_PORT_TYPE_QSFP_10G:
2831         case FW_PORT_TYPE_SFP:
2832         case FW_PORT_TYPE_FIBER_XFI:
2833         case FW_PORT_TYPE_FIBER_XAUI:
2834                 switch (pi->mod_type) {
2835
2836                 case FW_PORT_MOD_TYPE_LR:
2837                         ifmedia_add(media, m | IFM_10G_LR, data, NULL);
2838                         ifmedia_set(media, m | IFM_10G_LR);
2839                         break;
2840
2841                 case FW_PORT_MOD_TYPE_SR:
2842                         ifmedia_add(media, m | IFM_10G_SR, data, NULL);
2843                         ifmedia_set(media, m | IFM_10G_SR);
2844                         break;
2845
2846                 case FW_PORT_MOD_TYPE_LRM:
2847                         ifmedia_add(media, m | IFM_10G_LRM, data, NULL);
2848                         ifmedia_set(media, m | IFM_10G_LRM);
2849                         break;
2850
2851                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2852                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2853                         ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL);
2854                         ifmedia_set(media, m | IFM_10G_TWINAX);
2855                         break;
2856
2857                 case FW_PORT_MOD_TYPE_NONE:
2858                         m &= ~IFM_FDX;
2859                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2860                         ifmedia_set(media, m | IFM_NONE);
2861                         break;
2862
2863                 case FW_PORT_MOD_TYPE_NA:
2864                 case FW_PORT_MOD_TYPE_ER:
2865                 default:
2866                         device_printf(pi->dev,
2867                             "unknown port_type (%d), mod_type (%d)\n",
2868                             pi->port_type, pi->mod_type);
2869                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2870                         ifmedia_set(media, m | IFM_UNKNOWN);
2871                         break;
2872                 }
2873                 break;
2874
2875         case FW_PORT_TYPE_QSFP:
2876                 switch (pi->mod_type) {
2877
2878                 case FW_PORT_MOD_TYPE_LR:
2879                         ifmedia_add(media, m | IFM_40G_LR4, data, NULL);
2880                         ifmedia_set(media, m | IFM_40G_LR4);
2881                         break;
2882
2883                 case FW_PORT_MOD_TYPE_SR:
2884                         ifmedia_add(media, m | IFM_40G_SR4, data, NULL);
2885                         ifmedia_set(media, m | IFM_40G_SR4);
2886                         break;
2887
2888                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2889                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2890                         ifmedia_add(media, m | IFM_40G_CR4, data, NULL);
2891                         ifmedia_set(media, m | IFM_40G_CR4);
2892                         break;
2893
2894                 case FW_PORT_MOD_TYPE_NONE:
2895                         m &= ~IFM_FDX;
2896                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2897                         ifmedia_set(media, m | IFM_NONE);
2898                         break;
2899
2900                 default:
2901                         device_printf(pi->dev,
2902                             "unknown port_type (%d), mod_type (%d)\n",
2903                             pi->port_type, pi->mod_type);
2904                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2905                         ifmedia_set(media, m | IFM_UNKNOWN);
2906                         break;
2907                 }
2908                 break;
2909
2910         default:
2911                 device_printf(pi->dev,
2912                     "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
2913                     pi->mod_type);
2914                 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2915                 ifmedia_set(media, m | IFM_UNKNOWN);
2916                 break;
2917         }
2918
2919         PORT_UNLOCK(pi);
2920 }
2921
2922 #define FW_MAC_EXACT_CHUNK      7
2923
2924 /*
2925  * Program the port's XGMAC based on parameters in ifnet.  The caller also
2926  * indicates which parameters should be programmed (the rest are left alone).
2927  */
2928 int
2929 update_mac_settings(struct ifnet *ifp, int flags)
2930 {
2931         int rc = 0;
2932         struct port_info *pi = ifp->if_softc;
2933         struct adapter *sc = pi->adapter;
2934         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
2935         uint16_t viid = 0xffff;
2936         int16_t *xact_addr_filt = NULL;
2937
2938         ASSERT_SYNCHRONIZED_OP(sc);
2939         KASSERT(flags, ("%s: not told what to update.", __func__));
2940
2941         if (ifp == pi->ifp) {
2942                 viid = pi->viid;
2943                 xact_addr_filt = &pi->xact_addr_filt;
2944         }
2945 #ifdef DEV_NETMAP
2946         else if (ifp == pi->nm_ifp) {
2947                 viid = pi->nm_viid;
2948                 xact_addr_filt = &pi->nm_xact_addr_filt;
2949         }
2950 #endif
2951         if (flags & XGMAC_MTU)
2952                 mtu = ifp->if_mtu;
2953
2954         if (flags & XGMAC_PROMISC)
2955                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
2956
2957         if (flags & XGMAC_ALLMULTI)
2958                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
2959
2960         if (flags & XGMAC_VLANEX)
2961                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
2962
2963         if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
2964                 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti,
2965                     1, vlanex, false);
2966                 if (rc) {
2967                         if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
2968                             rc);
2969                         return (rc);
2970                 }
2971         }
2972
2973         if (flags & XGMAC_UCADDR) {
2974                 uint8_t ucaddr[ETHER_ADDR_LEN];
2975
2976                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
2977                 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr,
2978                     true, true);
2979                 if (rc < 0) {
2980                         rc = -rc;
2981                         if_printf(ifp, "change_mac failed: %d\n", rc);
2982                         return (rc);
2983                 } else {
2984                         *xact_addr_filt = rc;
2985                         rc = 0;
2986                 }
2987         }
2988
2989         if (flags & XGMAC_MCADDRS) {
2990                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
2991                 int del = 1;
2992                 uint64_t hash = 0;
2993                 struct ifmultiaddr *ifma;
2994                 int i = 0, j;
2995
2996                 if_maddr_rlock(ifp);
2997                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2998                         if (ifma->ifma_addr->sa_family != AF_LINK)
2999                                 continue;
3000                         mcaddr[i++] =
3001                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
3002
3003                         if (i == FW_MAC_EXACT_CHUNK) {
3004                                 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del,
3005                                     i, mcaddr, NULL, &hash, 0);
3006                                 if (rc < 0) {
3007                                         rc = -rc;
3008                                         for (j = 0; j < i; j++) {
3009                                                 if_printf(ifp,
3010                                                     "failed to add mc address"
3011                                                     " %02x:%02x:%02x:"
3012                                                     "%02x:%02x:%02x rc=%d\n",
3013                                                     mcaddr[j][0], mcaddr[j][1],
3014                                                     mcaddr[j][2], mcaddr[j][3],
3015                                                     mcaddr[j][4], mcaddr[j][5],
3016                                                     rc);
3017                                         }
3018                                         goto mcfail;
3019                                 }
3020                                 del = 0;
3021                                 i = 0;
3022                         }
3023                 }
3024                 if (i > 0) {
3025                         rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i,
3026                             mcaddr, NULL, &hash, 0);
3027                         if (rc < 0) {
3028                                 rc = -rc;
3029                                 for (j = 0; j < i; j++) {
3030                                         if_printf(ifp,
3031                                             "failed to add mc address"
3032                                             " %02x:%02x:%02x:"
3033                                             "%02x:%02x:%02x rc=%d\n",
3034                                             mcaddr[j][0], mcaddr[j][1],
3035                                             mcaddr[j][2], mcaddr[j][3],
3036                                             mcaddr[j][4], mcaddr[j][5],
3037                                             rc);
3038                                 }
3039                                 goto mcfail;
3040                         }
3041                 }
3042
3043                 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0);
3044                 if (rc != 0)
3045                         if_printf(ifp, "failed to set mc address hash: %d", rc);
3046 mcfail:
3047                 if_maddr_runlock(ifp);
3048         }
3049
3050         return (rc);
3051 }
3052
3053 int
3054 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
3055     char *wmesg)
3056 {
3057         int rc, pri;
3058
3059 #ifdef WITNESS
3060         /* the caller thinks it's ok to sleep, but is it really? */
3061         if (flags & SLEEP_OK)
3062                 pause("t4slptst", 1);
3063 #endif
3064
3065         if (INTR_OK)
3066                 pri = PCATCH;
3067         else
3068                 pri = 0;
3069
3070         ADAPTER_LOCK(sc);
3071         for (;;) {
3072
3073                 if (pi && IS_DOOMED(pi)) {
3074                         rc = ENXIO;
3075                         goto done;
3076                 }
3077
3078                 if (!IS_BUSY(sc)) {
3079                         rc = 0;
3080                         break;
3081                 }
3082
3083                 if (!(flags & SLEEP_OK)) {
3084                         rc = EBUSY;
3085                         goto done;
3086                 }
3087
3088                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
3089                         rc = EINTR;
3090                         goto done;
3091                 }
3092         }
3093
3094         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
3095         SET_BUSY(sc);
3096 #ifdef INVARIANTS
3097         sc->last_op = wmesg;
3098         sc->last_op_thr = curthread;
3099 #endif
3100
3101 done:
3102         if (!(flags & HOLD_LOCK) || rc)
3103                 ADAPTER_UNLOCK(sc);
3104
3105         return (rc);
3106 }
3107
3108 void
3109 end_synchronized_op(struct adapter *sc, int flags)
3110 {
3111
3112         if (flags & LOCK_HELD)
3113                 ADAPTER_LOCK_ASSERT_OWNED(sc);
3114         else
3115                 ADAPTER_LOCK(sc);
3116
3117         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
3118         CLR_BUSY(sc);
3119         wakeup(&sc->flags);
3120         ADAPTER_UNLOCK(sc);
3121 }
3122
3123 static int
3124 cxgbe_init_synchronized(struct port_info *pi)
3125 {
3126         struct adapter *sc = pi->adapter;
3127         struct ifnet *ifp = pi->ifp;
3128         int rc = 0;
3129
3130         ASSERT_SYNCHRONIZED_OP(sc);
3131
3132         if (isset(&sc->open_device_map, pi->port_id)) {
3133                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
3134                     ("mismatch between open_device_map and if_drv_flags"));
3135                 return (0);     /* already running */
3136         }
3137
3138         if (!(sc->flags & FULL_INIT_DONE) &&
3139             ((rc = adapter_full_init(sc)) != 0))
3140                 return (rc);    /* error message displayed already */
3141
3142         if (!(pi->flags & PORT_INIT_DONE) &&
3143             ((rc = port_full_init(pi)) != 0))
3144                 return (rc); /* error message displayed already */
3145
3146         rc = update_mac_settings(ifp, XGMAC_ALL);
3147         if (rc)
3148                 goto done;      /* error message displayed already */
3149
3150         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
3151         if (rc != 0) {
3152                 if_printf(ifp, "enable_vi failed: %d\n", rc);
3153                 goto done;
3154         }
3155
3156         /*
3157          * The first iq of the first port to come up is used for tracing.
3158          */
3159         if (sc->traceq < 0) {
3160                 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
3161                 t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
3162                     A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
3163                     V_QUEUENUMBER(sc->traceq));
3164                 pi->flags |= HAS_TRACEQ;
3165         }
3166
3167         /* all ok */
3168         setbit(&sc->open_device_map, pi->port_id);
3169         PORT_LOCK(pi);
3170         ifp->if_drv_flags |= IFF_DRV_RUNNING;
3171         PORT_UNLOCK(pi);
3172
3173         callout_reset(&pi->tick, hz, cxgbe_tick, pi);
3174 done:
3175         if (rc != 0)
3176                 cxgbe_uninit_synchronized(pi);
3177
3178         return (rc);
3179 }
3180
3181 /*
3182  * Idempotent.
3183  */
3184 static int
3185 cxgbe_uninit_synchronized(struct port_info *pi)
3186 {
3187         struct adapter *sc = pi->adapter;
3188         struct ifnet *ifp = pi->ifp;
3189         int rc;
3190
3191         ASSERT_SYNCHRONIZED_OP(sc);
3192
3193         /*
3194          * Disable the VI so that all its data in either direction is discarded
3195          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
3196          * tick) intact as the TP can deliver negative advice or data that it's
3197          * holding in its RAM (for an offloaded connection) even after the VI is
3198          * disabled.
3199          */
3200         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
3201         if (rc) {
3202                 if_printf(ifp, "disable_vi failed: %d\n", rc);
3203                 return (rc);
3204         }
3205
3206         clrbit(&sc->open_device_map, pi->port_id);
3207         PORT_LOCK(pi);
3208         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3209         PORT_UNLOCK(pi);
3210
3211         pi->link_cfg.link_ok = 0;
3212         pi->link_cfg.speed = 0;
3213         pi->linkdnrc = -1;
3214         t4_os_link_changed(sc, pi->port_id, 0, -1);
3215
3216         return (0);
3217 }
3218
3219 /*
3220  * It is ok for this function to fail midway and return right away.  t4_detach
3221  * will walk the entire sc->irq list and clean up whatever is valid.
3222  */
3223 static int
3224 setup_intr_handlers(struct adapter *sc)
3225 {
3226         int rc, rid, p, q;
3227         char s[8];
3228         struct irq *irq;
3229         struct port_info *pi;
3230         struct sge_rxq *rxq;
3231 #ifdef TCP_OFFLOAD
3232         struct sge_ofld_rxq *ofld_rxq;
3233 #endif
3234 #ifdef DEV_NETMAP
3235         struct sge_nm_rxq *nm_rxq;
3236 #endif
3237
3238         /*
3239          * Setup interrupts.
3240          */
3241         irq = &sc->irq[0];
3242         rid = sc->intr_type == INTR_INTX ? 0 : 1;
3243         if (sc->intr_count == 1)
3244                 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
3245
3246         /* Multiple interrupts. */
3247         KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
3248             ("%s: too few intr.", __func__));
3249
3250         /* The first one is always error intr */
3251         rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
3252         if (rc != 0)
3253                 return (rc);
3254         irq++;
3255         rid++;
3256
3257         /* The second one is always the firmware event queue */
3258         rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
3259         if (rc != 0)
3260                 return (rc);
3261         irq++;
3262         rid++;
3263
3264         for_each_port(sc, p) {
3265                 pi = sc->port[p];
3266
3267                 if (pi->flags & INTR_RXQ) {
3268                         for_each_rxq(pi, q, rxq) {
3269                                 snprintf(s, sizeof(s), "%d.%d", p, q);
3270                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
3271                                     s);
3272                                 if (rc != 0)
3273                                         return (rc);
3274                                 irq++;
3275                                 rid++;
3276                         }
3277                 }
3278 #ifdef TCP_OFFLOAD
3279                 if (pi->flags & INTR_OFLD_RXQ) {
3280                         for_each_ofld_rxq(pi, q, ofld_rxq) {
3281                                 snprintf(s, sizeof(s), "%d,%d", p, q);
3282                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr,
3283                                     ofld_rxq, s);
3284                                 if (rc != 0)
3285                                         return (rc);
3286                                 irq++;
3287                                 rid++;
3288                         }
3289                 }
3290 #endif
3291 #ifdef DEV_NETMAP
3292                 if (pi->flags & INTR_NM_RXQ) {
3293                         for_each_nm_rxq(pi, q, nm_rxq) {
3294                                 snprintf(s, sizeof(s), "%d-%d", p, q);
3295                                 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr,
3296                                     nm_rxq, s);
3297                                 if (rc != 0)
3298                                         return (rc);
3299                                 irq++;
3300                                 rid++;
3301                         }
3302                 }
3303 #endif
3304         }
3305         MPASS(irq == &sc->irq[sc->intr_count]);
3306
3307         return (0);
3308 }
3309
3310 int
3311 adapter_full_init(struct adapter *sc)
3312 {
3313         int rc, i;
3314
3315         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3316         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
3317             ("%s: FULL_INIT_DONE already", __func__));
3318
3319         /*
3320          * queues that belong to the adapter (not any particular port).
3321          */
3322         rc = t4_setup_adapter_queues(sc);
3323         if (rc != 0)
3324                 goto done;
3325
3326         for (i = 0; i < nitems(sc->tq); i++) {
3327                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
3328                     taskqueue_thread_enqueue, &sc->tq[i]);
3329                 if (sc->tq[i] == NULL) {
3330                         device_printf(sc->dev,
3331                             "failed to allocate task queue %d\n", i);
3332                         rc = ENOMEM;
3333                         goto done;
3334                 }
3335                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
3336                     device_get_nameunit(sc->dev), i);
3337         }
3338
3339         t4_intr_enable(sc);
3340         sc->flags |= FULL_INIT_DONE;
3341 done:
3342         if (rc != 0)
3343                 adapter_full_uninit(sc);
3344
3345         return (rc);
3346 }
3347
3348 int
3349 adapter_full_uninit(struct adapter *sc)
3350 {
3351         int i;
3352
3353         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3354
3355         t4_teardown_adapter_queues(sc);
3356
3357         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
3358                 taskqueue_free(sc->tq[i]);
3359                 sc->tq[i] = NULL;
3360         }
3361
3362         sc->flags &= ~FULL_INIT_DONE;
3363
3364         return (0);
3365 }
3366
3367 int
3368 port_full_init(struct port_info *pi)
3369 {
3370         struct adapter *sc = pi->adapter;
3371         struct ifnet *ifp = pi->ifp;
3372         uint16_t *rss;
3373         struct sge_rxq *rxq;
3374         int rc, i, j;
3375
3376         ASSERT_SYNCHRONIZED_OP(sc);
3377         KASSERT((pi->flags & PORT_INIT_DONE) == 0,
3378             ("%s: PORT_INIT_DONE already", __func__));
3379
3380         sysctl_ctx_init(&pi->ctx);
3381         pi->flags |= PORT_SYSCTL_CTX;
3382
3383         /*
3384          * Allocate tx/rx/fl queues for this port.
3385          */
3386         rc = t4_setup_port_queues(pi);
3387         if (rc != 0)
3388                 goto done;      /* error message displayed already */
3389
3390         /*
3391          * Setup RSS for this port.  Save a copy of the RSS table for later use.
3392          */
3393         rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
3394         for (i = 0; i < pi->rss_size;) {
3395                 for_each_rxq(pi, j, rxq) {
3396                         rss[i++] = rxq->iq.abs_id;
3397                         if (i == pi->rss_size)
3398                                 break;
3399                 }
3400         }
3401
3402         rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
3403             pi->rss_size);
3404         if (rc != 0) {
3405                 if_printf(ifp, "rss_config failed: %d\n", rc);
3406                 goto done;
3407         }
3408
3409         pi->rss = rss;
3410         pi->flags |= PORT_INIT_DONE;
3411 done:
3412         if (rc != 0)
3413                 port_full_uninit(pi);
3414
3415         return (rc);
3416 }
3417
3418 /*
3419  * Idempotent.
3420  */
3421 int
3422 port_full_uninit(struct port_info *pi)
3423 {
3424         struct adapter *sc = pi->adapter;
3425         int i;
3426         struct sge_rxq *rxq;
3427         struct sge_txq *txq;
3428 #ifdef TCP_OFFLOAD
3429         struct sge_ofld_rxq *ofld_rxq;
3430         struct sge_wrq *ofld_txq;
3431 #endif
3432
3433         if (pi->flags & PORT_INIT_DONE) {
3434
3435                 /* Need to quiesce queues.  XXX: ctrl queues? */
3436
3437                 for_each_txq(pi, i, txq) {
3438                         quiesce_eq(sc, &txq->eq);
3439                 }
3440
3441 #ifdef TCP_OFFLOAD
3442                 for_each_ofld_txq(pi, i, ofld_txq) {
3443                         quiesce_eq(sc, &ofld_txq->eq);
3444                 }
3445 #endif
3446
3447                 for_each_rxq(pi, i, rxq) {
3448                         quiesce_iq(sc, &rxq->iq);
3449                         quiesce_fl(sc, &rxq->fl);
3450                 }
3451
3452 #ifdef TCP_OFFLOAD
3453                 for_each_ofld_rxq(pi, i, ofld_rxq) {
3454                         quiesce_iq(sc, &ofld_rxq->iq);
3455                         quiesce_fl(sc, &ofld_rxq->fl);
3456                 }
3457 #endif
3458                 free(pi->rss, M_CXGBE);
3459         }
3460
3461         t4_teardown_port_queues(pi);
3462         pi->flags &= ~PORT_INIT_DONE;
3463
3464         return (0);
3465 }
3466
3467 static void
3468 quiesce_eq(struct adapter *sc, struct sge_eq *eq)
3469 {
3470         EQ_LOCK(eq);
3471         eq->flags |= EQ_DOOMED;
3472
3473         /*
3474          * Wait for the response to a credit flush if one's
3475          * pending.
3476          */
3477         while (eq->flags & EQ_CRFLUSHED)
3478                 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0);
3479         EQ_UNLOCK(eq);
3480
3481         callout_drain(&eq->tx_callout); /* XXX: iffy */
3482         pause("callout", 10);           /* Still iffy */
3483
3484         taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task);
3485 }
3486
3487 static void
3488 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
3489 {
3490         (void) sc;      /* unused */
3491
3492         /* Synchronize with the interrupt handler */
3493         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
3494                 pause("iqfree", 1);
3495 }
3496
3497 static void
3498 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
3499 {
3500         mtx_lock(&sc->sfl_lock);
3501         FL_LOCK(fl);
3502         fl->flags |= FL_DOOMED;
3503         FL_UNLOCK(fl);
3504         mtx_unlock(&sc->sfl_lock);
3505
3506         callout_drain(&sc->sfl_callout);
3507         KASSERT((fl->flags & FL_STARVING) == 0,
3508             ("%s: still starving", __func__));
3509 }
3510
3511 static int
3512 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
3513     driver_intr_t *handler, void *arg, char *name)
3514 {
3515         int rc;
3516
3517         irq->rid = rid;
3518         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
3519             RF_SHAREABLE | RF_ACTIVE);
3520         if (irq->res == NULL) {
3521                 device_printf(sc->dev,
3522                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
3523                 return (ENOMEM);
3524         }
3525
3526         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
3527             NULL, handler, arg, &irq->tag);
3528         if (rc != 0) {
3529                 device_printf(sc->dev,
3530                     "failed to setup interrupt for rid %d, name %s: %d\n",
3531                     rid, name, rc);
3532         } else if (name)
3533                 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
3534
3535         return (rc);
3536 }
3537
3538 static int
3539 t4_free_irq(struct adapter *sc, struct irq *irq)
3540 {
3541         if (irq->tag)
3542                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
3543         if (irq->res)
3544                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
3545
3546         bzero(irq, sizeof(*irq));
3547
3548         return (0);
3549 }
3550
3551 static void
3552 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
3553     unsigned int end)
3554 {
3555         uint32_t *p = (uint32_t *)(buf + start);
3556
3557         for ( ; start <= end; start += sizeof(uint32_t))
3558                 *p++ = t4_read_reg(sc, start);
3559 }
3560
3561 static void
3562 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
3563 {
3564         int i, n;
3565         const unsigned int *reg_ranges;
3566         static const unsigned int t4_reg_ranges[] = {
3567                 0x1008, 0x1108,
3568                 0x1180, 0x11b4,
3569                 0x11fc, 0x123c,
3570                 0x1300, 0x173c,
3571                 0x1800, 0x18fc,
3572                 0x3000, 0x30d8,
3573                 0x30e0, 0x5924,
3574                 0x5960, 0x59d4,
3575                 0x5a00, 0x5af8,
3576                 0x6000, 0x6098,
3577                 0x6100, 0x6150,
3578                 0x6200, 0x6208,
3579                 0x6240, 0x6248,
3580                 0x6280, 0x6338,
3581                 0x6370, 0x638c,
3582                 0x6400, 0x643c,
3583                 0x6500, 0x6524,
3584                 0x6a00, 0x6a38,
3585                 0x6a60, 0x6a78,
3586                 0x6b00, 0x6b84,
3587                 0x6bf0, 0x6c84,
3588                 0x6cf0, 0x6d84,
3589                 0x6df0, 0x6e84,
3590                 0x6ef0, 0x6f84,
3591                 0x6ff0, 0x7084,
3592                 0x70f0, 0x7184,
3593                 0x71f0, 0x7284,
3594                 0x72f0, 0x7384,
3595                 0x73f0, 0x7450,
3596                 0x7500, 0x7530,
3597                 0x7600, 0x761c,
3598                 0x7680, 0x76cc,
3599                 0x7700, 0x7798,
3600                 0x77c0, 0x77fc,
3601                 0x7900, 0x79fc,
3602                 0x7b00, 0x7c38,
3603                 0x7d00, 0x7efc,
3604                 0x8dc0, 0x8e1c,
3605                 0x8e30, 0x8e78,
3606                 0x8ea0, 0x8f6c,
3607                 0x8fc0, 0x9074,
3608                 0x90fc, 0x90fc,
3609                 0x9400, 0x9458,
3610                 0x9600, 0x96bc,
3611                 0x9800, 0x9808,
3612                 0x9820, 0x983c,
3613                 0x9850, 0x9864,
3614                 0x9c00, 0x9c6c,
3615                 0x9c80, 0x9cec,
3616                 0x9d00, 0x9d6c,
3617                 0x9d80, 0x9dec,
3618                 0x9e00, 0x9e6c,
3619                 0x9e80, 0x9eec,
3620                 0x9f00, 0x9f6c,
3621                 0x9f80, 0x9fec,
3622                 0xd004, 0xd03c,
3623                 0xdfc0, 0xdfe0,
3624                 0xe000, 0xea7c,
3625                 0xf000, 0x11110,
3626                 0x11118, 0x11190,
3627                 0x19040, 0x1906c,
3628                 0x19078, 0x19080,
3629                 0x1908c, 0x19124,
3630                 0x19150, 0x191b0,
3631                 0x191d0, 0x191e8,
3632                 0x19238, 0x1924c,
3633                 0x193f8, 0x19474,
3634                 0x19490, 0x194f8,
3635                 0x19800, 0x19f30,
3636                 0x1a000, 0x1a06c,
3637                 0x1a0b0, 0x1a120,
3638                 0x1a128, 0x1a138,
3639                 0x1a190, 0x1a1c4,
3640                 0x1a1fc, 0x1a1fc,
3641                 0x1e040, 0x1e04c,
3642                 0x1e284, 0x1e28c,
3643                 0x1e2c0, 0x1e2c0,
3644                 0x1e2e0, 0x1e2e0,
3645                 0x1e300, 0x1e384,
3646                 0x1e3c0, 0x1e3c8,
3647                 0x1e440, 0x1e44c,
3648                 0x1e684, 0x1e68c,
3649                 0x1e6c0, 0x1e6c0,
3650                 0x1e6e0, 0x1e6e0,
3651                 0x1e700, 0x1e784,
3652                 0x1e7c0, 0x1e7c8,
3653                 0x1e840, 0x1e84c,
3654                 0x1ea84, 0x1ea8c,
3655                 0x1eac0, 0x1eac0,
3656                 0x1eae0, 0x1eae0,
3657                 0x1eb00, 0x1eb84,
3658                 0x1ebc0, 0x1ebc8,
3659                 0x1ec40, 0x1ec4c,
3660                 0x1ee84, 0x1ee8c,
3661                 0x1eec0, 0x1eec0,
3662                 0x1eee0, 0x1eee0,
3663                 0x1ef00, 0x1ef84,
3664                 0x1efc0, 0x1efc8,
3665                 0x1f040, 0x1f04c,
3666                 0x1f284, 0x1f28c,
3667                 0x1f2c0, 0x1f2c0,
3668                 0x1f2e0, 0x1f2e0,
3669                 0x1f300, 0x1f384,
3670                 0x1f3c0, 0x1f3c8,
3671                 0x1f440, 0x1f44c,
3672                 0x1f684, 0x1f68c,
3673                 0x1f6c0, 0x1f6c0,
3674                 0x1f6e0, 0x1f6e0,
3675                 0x1f700, 0x1f784,
3676                 0x1f7c0, 0x1f7c8,
3677                 0x1f840, 0x1f84c,
3678                 0x1fa84, 0x1fa8c,
3679                 0x1fac0, 0x1fac0,
3680                 0x1fae0, 0x1fae0,
3681                 0x1fb00, 0x1fb84,
3682                 0x1fbc0, 0x1fbc8,
3683                 0x1fc40, 0x1fc4c,
3684                 0x1fe84, 0x1fe8c,
3685                 0x1fec0, 0x1fec0,
3686                 0x1fee0, 0x1fee0,
3687                 0x1ff00, 0x1ff84,
3688                 0x1ffc0, 0x1ffc8,
3689                 0x20000, 0x2002c,
3690                 0x20100, 0x2013c,
3691                 0x20190, 0x201c8,
3692                 0x20200, 0x20318,
3693                 0x20400, 0x20528,
3694                 0x20540, 0x20614,
3695                 0x21000, 0x21040,
3696                 0x2104c, 0x21060,
3697                 0x210c0, 0x210ec,
3698                 0x21200, 0x21268,
3699                 0x21270, 0x21284,
3700                 0x212fc, 0x21388,
3701                 0x21400, 0x21404,
3702                 0x21500, 0x21518,
3703                 0x2152c, 0x2153c,
3704                 0x21550, 0x21554,
3705                 0x21600, 0x21600,
3706                 0x21608, 0x21628,
3707                 0x21630, 0x2163c,
3708                 0x21700, 0x2171c,
3709                 0x21780, 0x2178c,
3710                 0x21800, 0x21c38,
3711                 0x21c80, 0x21d7c,
3712                 0x21e00, 0x21e04,
3713                 0x22000, 0x2202c,
3714                 0x22100, 0x2213c,
3715                 0x22190, 0x221c8,
3716                 0x22200, 0x22318,
3717                 0x22400, 0x22528,
3718                 0x22540, 0x22614,
3719                 0x23000, 0x23040,
3720                 0x2304c, 0x23060,
3721                 0x230c0, 0x230ec,
3722                 0x23200, 0x23268,
3723                 0x23270, 0x23284,
3724                 0x232fc, 0x23388,
3725                 0x23400, 0x23404,
3726                 0x23500, 0x23518,
3727                 0x2352c, 0x2353c,
3728                 0x23550, 0x23554,
3729                 0x23600, 0x23600,
3730                 0x23608, 0x23628,
3731                 0x23630, 0x2363c,
3732                 0x23700, 0x2371c,
3733                 0x23780, 0x2378c,
3734                 0x23800, 0x23c38,
3735                 0x23c80, 0x23d7c,
3736                 0x23e00, 0x23e04,
3737                 0x24000, 0x2402c,
3738                 0x24100, 0x2413c,
3739                 0x24190, 0x241c8,
3740                 0x24200, 0x24318,
3741                 0x24400, 0x24528,
3742                 0x24540, 0x24614,
3743                 0x25000, 0x25040,
3744                 0x2504c, 0x25060,
3745                 0x250c0, 0x250ec,
3746                 0x25200, 0x25268,
3747                 0x25270, 0x25284,
3748                 0x252fc, 0x25388,
3749                 0x25400, 0x25404,
3750                 0x25500, 0x25518,
3751                 0x2552c, 0x2553c,
3752                 0x25550, 0x25554,
3753                 0x25600, 0x25600,
3754                 0x25608, 0x25628,
3755                 0x25630, 0x2563c,
3756                 0x25700, 0x2571c,
3757                 0x25780, 0x2578c,
3758                 0x25800, 0x25c38,
3759                 0x25c80, 0x25d7c,
3760                 0x25e00, 0x25e04,
3761                 0x26000, 0x2602c,
3762                 0x26100, 0x2613c,
3763                 0x26190, 0x261c8,
3764                 0x26200, 0x26318,
3765                 0x26400, 0x26528,
3766                 0x26540, 0x26614,
3767                 0x27000, 0x27040,
3768                 0x2704c, 0x27060,
3769                 0x270c0, 0x270ec,
3770                 0x27200, 0x27268,
3771                 0x27270, 0x27284,
3772                 0x272fc, 0x27388,
3773                 0x27400, 0x27404,
3774                 0x27500, 0x27518,
3775                 0x2752c, 0x2753c,
3776                 0x27550, 0x27554,
3777                 0x27600, 0x27600,
3778                 0x27608, 0x27628,
3779                 0x27630, 0x2763c,
3780                 0x27700, 0x2771c,
3781                 0x27780, 0x2778c,
3782                 0x27800, 0x27c38,
3783                 0x27c80, 0x27d7c,
3784                 0x27e00, 0x27e04
3785         };
3786         static const unsigned int t5_reg_ranges[] = {
3787                 0x1008, 0x1148,
3788                 0x1180, 0x11b4,
3789                 0x11fc, 0x123c,
3790                 0x1280, 0x173c,
3791                 0x1800, 0x18fc,
3792                 0x3000, 0x3028,
3793                 0x3060, 0x30d8,
3794                 0x30e0, 0x30fc,
3795                 0x3140, 0x357c,
3796                 0x35a8, 0x35cc,
3797                 0x35ec, 0x35ec,
3798                 0x3600, 0x5624,
3799                 0x56cc, 0x575c,
3800                 0x580c, 0x5814,
3801                 0x5890, 0x58bc,
3802                 0x5940, 0x59dc,
3803                 0x59fc, 0x5a18,
3804                 0x5a60, 0x5a9c,
3805                 0x5b94, 0x5bfc,
3806                 0x6000, 0x6040,
3807                 0x6058, 0x614c,
3808                 0x7700, 0x7798,
3809                 0x77c0, 0x78fc,
3810                 0x7b00, 0x7c54,
3811                 0x7d00, 0x7efc,
3812                 0x8dc0, 0x8de0,
3813                 0x8df8, 0x8e84,
3814                 0x8ea0, 0x8f84,
3815                 0x8fc0, 0x90f8,
3816                 0x9400, 0x9470,
3817                 0x9600, 0x96f4,
3818                 0x9800, 0x9808,
3819                 0x9820, 0x983c,
3820                 0x9850, 0x9864,
3821                 0x9c00, 0x9c6c,
3822                 0x9c80, 0x9cec,
3823                 0x9d00, 0x9d6c,
3824                 0x9d80, 0x9dec,
3825                 0x9e00, 0x9e6c,
3826                 0x9e80, 0x9eec,
3827                 0x9f00, 0x9f6c,
3828                 0x9f80, 0xa020,
3829                 0xd004, 0xd03c,
3830                 0xdfc0, 0xdfe0,
3831                 0xe000, 0x11088,
3832                 0x1109c, 0x11110,
3833                 0x11118, 0x1117c,
3834                 0x11190, 0x11204,
3835                 0x19040, 0x1906c,
3836                 0x19078, 0x19080,
3837                 0x1908c, 0x19124,
3838                 0x19150, 0x191b0,
3839                 0x191d0, 0x191e8,
3840                 0x19238, 0x19290,
3841                 0x193f8, 0x19474,
3842                 0x19490, 0x194cc,
3843                 0x194f0, 0x194f8,
3844                 0x19c00, 0x19c60,
3845                 0x19c94, 0x19e10,
3846                 0x19e50, 0x19f34,
3847                 0x19f40, 0x19f50,
3848                 0x19f90, 0x19fe4,
3849                 0x1a000, 0x1a06c,
3850                 0x1a0b0, 0x1a120,
3851                 0x1a128, 0x1a138,
3852                 0x1a190, 0x1a1c4,
3853                 0x1a1fc, 0x1a1fc,
3854                 0x1e008, 0x1e00c,
3855                 0x1e040, 0x1e04c,
3856                 0x1e284, 0x1e290,
3857                 0x1e2c0, 0x1e2c0,
3858                 0x1e2e0, 0x1e2e0,
3859                 0x1e300, 0x1e384,
3860                 0x1e3c0, 0x1e3c8,
3861                 0x1e408, 0x1e40c,
3862                 0x1e440, 0x1e44c,
3863                 0x1e684, 0x1e690,
3864                 0x1e6c0, 0x1e6c0,
3865                 0x1e6e0, 0x1e6e0,
3866                 0x1e700, 0x1e784,
3867                 0x1e7c0, 0x1e7c8,
3868                 0x1e808, 0x1e80c,
3869                 0x1e840, 0x1e84c,
3870                 0x1ea84, 0x1ea90,
3871                 0x1eac0, 0x1eac0,
3872                 0x1eae0, 0x1eae0,
3873                 0x1eb00, 0x1eb84,
3874                 0x1ebc0, 0x1ebc8,
3875                 0x1ec08, 0x1ec0c,
3876                 0x1ec40, 0x1ec4c,
3877                 0x1ee84, 0x1ee90,
3878                 0x1eec0, 0x1eec0,
3879                 0x1eee0, 0x1eee0,
3880                 0x1ef00, 0x1ef84,
3881                 0x1efc0, 0x1efc8,
3882                 0x1f008, 0x1f00c,
3883                 0x1f040, 0x1f04c,
3884                 0x1f284, 0x1f290,
3885                 0x1f2c0, 0x1f2c0,
3886                 0x1f2e0, 0x1f2e0,
3887                 0x1f300, 0x1f384,
3888                 0x1f3c0, 0x1f3c8,
3889                 0x1f408, 0x1f40c,
3890                 0x1f440, 0x1f44c,
3891                 0x1f684, 0x1f690,
3892                 0x1f6c0, 0x1f6c0,
3893                 0x1f6e0, 0x1f6e0,
3894                 0x1f700, 0x1f784,
3895                 0x1f7c0, 0x1f7c8,
3896                 0x1f808, 0x1f80c,
3897                 0x1f840, 0x1f84c,
3898                 0x1fa84, 0x1fa90,
3899                 0x1fac0, 0x1fac0,
3900                 0x1fae0, 0x1fae0,
3901                 0x1fb00, 0x1fb84,
3902                 0x1fbc0, 0x1fbc8,
3903                 0x1fc08, 0x1fc0c,
3904                 0x1fc40, 0x1fc4c,
3905                 0x1fe84, 0x1fe90,
3906                 0x1fec0, 0x1fec0,
3907                 0x1fee0, 0x1fee0,
3908                 0x1ff00, 0x1ff84,
3909                 0x1ffc0, 0x1ffc8,
3910                 0x30000, 0x30030,
3911                 0x30100, 0x30144,
3912                 0x30190, 0x301d0,
3913                 0x30200, 0x30318,
3914                 0x30400, 0x3052c,
3915                 0x30540, 0x3061c,
3916                 0x30800, 0x30834,
3917                 0x308c0, 0x30908,
3918                 0x30910, 0x309ac,
3919                 0x30a00, 0x30a2c,
3920                 0x30a44, 0x30a50,
3921                 0x30a74, 0x30c24,
3922                 0x30d00, 0x30d00,
3923                 0x30d08, 0x30d14,
3924                 0x30d1c, 0x30d20,
3925                 0x30d3c, 0x30d50,
3926                 0x31200, 0x3120c,
3927                 0x31220, 0x31220,
3928                 0x31240, 0x31240,
3929                 0x31600, 0x3160c,
3930                 0x31a00, 0x31a1c,
3931                 0x31e00, 0x31e20,
3932                 0x31e38, 0x31e3c,
3933                 0x31e80, 0x31e80,
3934                 0x31e88, 0x31ea8,
3935                 0x31eb0, 0x31eb4,
3936                 0x31ec8, 0x31ed4,
3937                 0x31fb8, 0x32004,
3938                 0x32200, 0x32200,
3939                 0x32208, 0x32240,
3940                 0x32248, 0x32280,
3941                 0x32288, 0x322c0,
3942                 0x322c8, 0x322fc,
3943                 0x32600, 0x32630,
3944                 0x32a00, 0x32abc,
3945                 0x32b00, 0x32b70,
3946                 0x33000, 0x33048,
3947                 0x33060, 0x3309c,
3948                 0x330f0, 0x33148,
3949                 0x33160, 0x3319c,
3950                 0x331f0, 0x332e4,
3951                 0x332f8, 0x333e4,
3952                 0x333f8, 0x33448,
3953                 0x33460, 0x3349c,
3954                 0x334f0, 0x33548,
3955                 0x33560, 0x3359c,
3956                 0x335f0, 0x336e4,
3957                 0x336f8, 0x337e4,
3958                 0x337f8, 0x337fc,
3959                 0x33814, 0x33814,
3960                 0x3382c, 0x3382c,
3961                 0x33880, 0x3388c,
3962                 0x338e8, 0x338ec,
3963                 0x33900, 0x33948,
3964                 0x33960, 0x3399c,
3965                 0x339f0, 0x33ae4,
3966                 0x33af8, 0x33b10,
3967                 0x33b28, 0x33b28,
3968                 0x33b3c, 0x33b50,
3969                 0x33bf0, 0x33c10,
3970                 0x33c28, 0x33c28,
3971                 0x33c3c, 0x33c50,
3972                 0x33cf0, 0x33cfc,
3973                 0x34000, 0x34030,
3974                 0x34100, 0x34144,
3975                 0x34190, 0x341d0,
3976                 0x34200, 0x34318,
3977                 0x34400, 0x3452c,
3978                 0x34540, 0x3461c,
3979                 0x34800, 0x34834,
3980                 0x348c0, 0x34908,
3981                 0x34910, 0x349ac,
3982                 0x34a00, 0x34a2c,
3983                 0x34a44, 0x34a50,
3984                 0x34a74, 0x34c24,
3985                 0x34d00, 0x34d00,
3986                 0x34d08, 0x34d14,
3987                 0x34d1c, 0x34d20,
3988                 0x34d3c, 0x34d50,
3989                 0x35200, 0x3520c,
3990                 0x35220, 0x35220,
3991                 0x35240, 0x35240,
3992                 0x35600, 0x3560c,
3993                 0x35a00, 0x35a1c,
3994                 0x35e00, 0x35e20,
3995                 0x35e38, 0x35e3c,
3996                 0x35e80, 0x35e80,
3997                 0x35e88, 0x35ea8,
3998                 0x35eb0, 0x35eb4,
3999                 0x35ec8, 0x35ed4,
4000                 0x35fb8, 0x36004,
4001                 0x36200, 0x36200,
4002                 0x36208, 0x36240,
4003                 0x36248, 0x36280,
4004                 0x36288, 0x362c0,
4005                 0x362c8, 0x362fc,
4006                 0x36600, 0x36630,
4007                 0x36a00, 0x36abc,
4008                 0x36b00, 0x36b70,
4009                 0x37000, 0x37048,
4010                 0x37060, 0x3709c,
4011                 0x370f0, 0x37148,
4012                 0x37160, 0x3719c,
4013                 0x371f0, 0x372e4,
4014                 0x372f8, 0x373e4,
4015                 0x373f8, 0x37448,
4016                 0x37460, 0x3749c,
4017                 0x374f0, 0x37548,
4018                 0x37560, 0x3759c,
4019                 0x375f0, 0x376e4,
4020                 0x376f8, 0x377e4,
4021                 0x377f8, 0x377fc,
4022                 0x37814, 0x37814,
4023                 0x3782c, 0x3782c,
4024                 0x37880, 0x3788c,
4025                 0x378e8, 0x378ec,
4026                 0x37900, 0x37948,
4027                 0x37960, 0x3799c,
4028                 0x379f0, 0x37ae4,
4029                 0x37af8, 0x37b10,
4030                 0x37b28, 0x37b28,
4031                 0x37b3c, 0x37b50,
4032                 0x37bf0, 0x37c10,
4033                 0x37c28, 0x37c28,
4034                 0x37c3c, 0x37c50,
4035                 0x37cf0, 0x37cfc,
4036                 0x38000, 0x38030,
4037                 0x38100, 0x38144,
4038                 0x38190, 0x381d0,
4039                 0x38200, 0x38318,
4040                 0x38400, 0x3852c,
4041                 0x38540, 0x3861c,
4042                 0x38800, 0x38834,
4043                 0x388c0, 0x38908,
4044                 0x38910, 0x389ac,
4045                 0x38a00, 0x38a2c,
4046                 0x38a44, 0x38a50,
4047                 0x38a74, 0x38c24,
4048                 0x38d00, 0x38d00,
4049                 0x38d08, 0x38d14,
4050                 0x38d1c, 0x38d20,
4051                 0x38d3c, 0x38d50,
4052                 0x39200, 0x3920c,
4053                 0x39220, 0x39220,
4054                 0x39240, 0x39240,
4055                 0x39600, 0x3960c,
4056                 0x39a00, 0x39a1c,
4057                 0x39e00, 0x39e20,
4058                 0x39e38, 0x39e3c,
4059                 0x39e80, 0x39e80,
4060                 0x39e88, 0x39ea8,
4061                 0x39eb0, 0x39eb4,
4062                 0x39ec8, 0x39ed4,
4063                 0x39fb8, 0x3a004,
4064                 0x3a200, 0x3a200,
4065                 0x3a208, 0x3a240,
4066                 0x3a248, 0x3a280,
4067                 0x3a288, 0x3a2c0,
4068                 0x3a2c8, 0x3a2fc,
4069                 0x3a600, 0x3a630,
4070                 0x3aa00, 0x3aabc,
4071                 0x3ab00, 0x3ab70,
4072                 0x3b000, 0x3b048,
4073                 0x3b060, 0x3b09c,
4074                 0x3b0f0, 0x3b148,
4075                 0x3b160, 0x3b19c,
4076                 0x3b1f0, 0x3b2e4,
4077                 0x3b2f8, 0x3b3e4,
4078                 0x3b3f8, 0x3b448,
4079                 0x3b460, 0x3b49c,
4080                 0x3b4f0, 0x3b548,
4081                 0x3b560, 0x3b59c,
4082                 0x3b5f0, 0x3b6e4,
4083                 0x3b6f8, 0x3b7e4,
4084                 0x3b7f8, 0x3b7fc,
4085                 0x3b814, 0x3b814,
4086                 0x3b82c, 0x3b82c,
4087                 0x3b880, 0x3b88c,
4088                 0x3b8e8, 0x3b8ec,
4089                 0x3b900, 0x3b948,
4090                 0x3b960, 0x3b99c,
4091                 0x3b9f0, 0x3bae4,
4092                 0x3baf8, 0x3bb10,
4093                 0x3bb28, 0x3bb28,
4094                 0x3bb3c, 0x3bb50,
4095                 0x3bbf0, 0x3bc10,
4096                 0x3bc28, 0x3bc28,
4097                 0x3bc3c, 0x3bc50,
4098                 0x3bcf0, 0x3bcfc,
4099                 0x3c000, 0x3c030,
4100                 0x3c100, 0x3c144,
4101                 0x3c190, 0x3c1d0,
4102                 0x3c200, 0x3c318,
4103                 0x3c400, 0x3c52c,
4104                 0x3c540, 0x3c61c,
4105                 0x3c800, 0x3c834,
4106                 0x3c8c0, 0x3c908,
4107                 0x3c910, 0x3c9ac,
4108                 0x3ca00, 0x3ca2c,
4109                 0x3ca44, 0x3ca50,
4110                 0x3ca74, 0x3cc24,
4111                 0x3cd00, 0x3cd00,
4112                 0x3cd08, 0x3cd14,
4113                 0x3cd1c, 0x3cd20,
4114                 0x3cd3c, 0x3cd50,
4115                 0x3d200, 0x3d20c,
4116                 0x3d220, 0x3d220,
4117                 0x3d240, 0x3d240,
4118                 0x3d600, 0x3d60c,
4119                 0x3da00, 0x3da1c,
4120                 0x3de00, 0x3de20,
4121                 0x3de38, 0x3de3c,
4122                 0x3de80, 0x3de80,
4123                 0x3de88, 0x3dea8,
4124                 0x3deb0, 0x3deb4,
4125                 0x3dec8, 0x3ded4,
4126                 0x3dfb8, 0x3e004,
4127                 0x3e200, 0x3e200,
4128                 0x3e208, 0x3e240,
4129                 0x3e248, 0x3e280,
4130                 0x3e288, 0x3e2c0,
4131                 0x3e2c8, 0x3e2fc,
4132                 0x3e600, 0x3e630,
4133                 0x3ea00, 0x3eabc,
4134                 0x3eb00, 0x3eb70,
4135                 0x3f000, 0x3f048,
4136                 0x3f060, 0x3f09c,
4137                 0x3f0f0, 0x3f148,
4138                 0x3f160, 0x3f19c,
4139                 0x3f1f0, 0x3f2e4,
4140                 0x3f2f8, 0x3f3e4,
4141                 0x3f3f8, 0x3f448,
4142                 0x3f460, 0x3f49c,
4143                 0x3f4f0, 0x3f548,
4144                 0x3f560, 0x3f59c,
4145                 0x3f5f0, 0x3f6e4,
4146                 0x3f6f8, 0x3f7e4,
4147                 0x3f7f8, 0x3f7fc,
4148                 0x3f814, 0x3f814,
4149                 0x3f82c, 0x3f82c,
4150                 0x3f880, 0x3f88c,
4151                 0x3f8e8, 0x3f8ec,
4152                 0x3f900, 0x3f948,
4153                 0x3f960, 0x3f99c,
4154                 0x3f9f0, 0x3fae4,
4155                 0x3faf8, 0x3fb10,
4156                 0x3fb28, 0x3fb28,
4157                 0x3fb3c, 0x3fb50,
4158                 0x3fbf0, 0x3fc10,
4159                 0x3fc28, 0x3fc28,
4160                 0x3fc3c, 0x3fc50,
4161                 0x3fcf0, 0x3fcfc,
4162                 0x40000, 0x4000c,
4163                 0x40040, 0x40068,
4164                 0x4007c, 0x40144,
4165                 0x40180, 0x4018c,
4166                 0x40200, 0x40298,
4167                 0x402ac, 0x4033c,
4168                 0x403f8, 0x403fc,
4169                 0x41304, 0x413c4,
4170                 0x41400, 0x4141c,
4171                 0x41480, 0x414d0,
4172                 0x44000, 0x44078,
4173                 0x440c0, 0x44278,
4174                 0x442c0, 0x44478,
4175                 0x444c0, 0x44678,
4176                 0x446c0, 0x44878,
4177                 0x448c0, 0x449fc,
4178                 0x45000, 0x45068,
4179                 0x45080, 0x45084,
4180                 0x450a0, 0x450b0,
4181                 0x45200, 0x45268,
4182                 0x45280, 0x45284,
4183                 0x452a0, 0x452b0,
4184                 0x460c0, 0x460e4,
4185                 0x47000, 0x4708c,
4186                 0x47200, 0x47250,
4187                 0x47400, 0x47420,
4188                 0x47600, 0x47618,
4189                 0x47800, 0x47814,
4190                 0x48000, 0x4800c,
4191                 0x48040, 0x48068,
4192                 0x4807c, 0x48144,
4193                 0x48180, 0x4818c,
4194                 0x48200, 0x48298,
4195                 0x482ac, 0x4833c,
4196                 0x483f8, 0x483fc,
4197                 0x49304, 0x493c4,
4198                 0x49400, 0x4941c,
4199                 0x49480, 0x494d0,
4200                 0x4c000, 0x4c078,
4201                 0x4c0c0, 0x4c278,
4202                 0x4c2c0, 0x4c478,
4203                 0x4c4c0, 0x4c678,
4204                 0x4c6c0, 0x4c878,
4205                 0x4c8c0, 0x4c9fc,
4206                 0x4d000, 0x4d068,
4207                 0x4d080, 0x4d084,
4208                 0x4d0a0, 0x4d0b0,
4209                 0x4d200, 0x4d268,
4210                 0x4d280, 0x4d284,
4211                 0x4d2a0, 0x4d2b0,
4212                 0x4e0c0, 0x4e0e4,
4213                 0x4f000, 0x4f08c,
4214                 0x4f200, 0x4f250,
4215                 0x4f400, 0x4f420,
4216                 0x4f600, 0x4f618,
4217                 0x4f800, 0x4f814,
4218                 0x50000, 0x500cc,
4219                 0x50400, 0x50400,
4220                 0x50800, 0x508cc,
4221                 0x50c00, 0x50c00,
4222                 0x51000, 0x5101c,
4223                 0x51300, 0x51308,
4224         };
4225
4226         if (is_t4(sc)) {
4227                 reg_ranges = &t4_reg_ranges[0];
4228                 n = nitems(t4_reg_ranges);
4229         } else {
4230                 reg_ranges = &t5_reg_ranges[0];
4231                 n = nitems(t5_reg_ranges);
4232         }
4233
4234         regs->version = chip_id(sc) | chip_rev(sc) << 10;
4235         for (i = 0; i < n; i += 2)
4236                 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
4237 }
4238
4239 static void
4240 cxgbe_tick(void *arg)
4241 {
4242         struct port_info *pi = arg;
4243         struct adapter *sc = pi->adapter;
4244         struct ifnet *ifp = pi->ifp;
4245         struct sge_txq *txq;
4246         int i, drops;
4247         struct port_stats *s = &pi->stats;
4248
4249         PORT_LOCK(pi);
4250         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4251                 PORT_UNLOCK(pi);
4252                 return; /* without scheduling another callout */
4253         }
4254
4255         t4_get_port_stats(sc, pi->tx_chan, s);
4256
4257         ifp->if_opackets = s->tx_frames - s->tx_pause;
4258         ifp->if_ipackets = s->rx_frames - s->rx_pause;
4259         ifp->if_obytes = s->tx_octets - s->tx_pause * 64;
4260         ifp->if_ibytes = s->rx_octets - s->rx_pause * 64;
4261         ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause;
4262         ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause;
4263         ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
4264             s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
4265             s->rx_trunc3;
4266         for (i = 0; i < 4; i++) {
4267                 if (pi->rx_chan_map & (1 << i)) {
4268                         uint32_t v;
4269
4270                         /*
4271                          * XXX: indirect reads from the same ADDR/DATA pair can
4272                          * race with each other.
4273                          */
4274                         t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
4275                             1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4276                         ifp->if_iqdrops += v;
4277                 }
4278         }
4279
4280         drops = s->tx_drop;
4281         for_each_txq(pi, i, txq)
4282                 drops += txq->br->br_drops;
4283         ifp->if_snd.ifq_drops = drops;
4284
4285         ifp->if_oerrors = s->tx_error_frames;
4286         ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long +
4287             s->rx_fcs_err + s->rx_len_err;
4288
4289         callout_schedule(&pi->tick, hz);
4290         PORT_UNLOCK(pi);
4291 }
4292
4293 static void
4294 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4295 {
4296         struct ifnet *vlan;
4297
4298         if (arg != ifp || ifp->if_type != IFT_ETHER)
4299                 return;
4300
4301         vlan = VLAN_DEVAT(ifp, vid);
4302         VLAN_SETCOOKIE(vlan, ifp);
4303 }
4304
4305 static int
4306 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4307 {
4308
4309 #ifdef INVARIANTS
4310         panic("%s: opcode 0x%02x on iq %p with payload %p",
4311             __func__, rss->opcode, iq, m);
4312 #else
4313         log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
4314             __func__, rss->opcode, iq, m);
4315         m_freem(m);
4316 #endif
4317         return (EDOOFUS);
4318 }
4319
4320 int
4321 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
4322 {
4323         uintptr_t *loc, new;
4324
4325         if (opcode >= nitems(sc->cpl_handler))
4326                 return (EINVAL);
4327
4328         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
4329         loc = (uintptr_t *) &sc->cpl_handler[opcode];
4330         atomic_store_rel_ptr(loc, new);
4331
4332         return (0);
4333 }
4334
4335 static int
4336 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
4337 {
4338
4339 #ifdef INVARIANTS
4340         panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
4341 #else
4342         log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
4343             __func__, iq, ctrl);
4344 #endif
4345         return (EDOOFUS);
4346 }
4347
4348 int
4349 t4_register_an_handler(struct adapter *sc, an_handler_t h)
4350 {
4351         uintptr_t *loc, new;
4352
4353         new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
4354         loc = (uintptr_t *) &sc->an_handler;
4355         atomic_store_rel_ptr(loc, new);
4356
4357         return (0);
4358 }
4359
4360 static int
4361 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
4362 {
4363         const struct cpl_fw6_msg *cpl =
4364             __containerof(rpl, struct cpl_fw6_msg, data[0]);
4365
4366 #ifdef INVARIANTS
4367         panic("%s: fw_msg type %d", __func__, cpl->type);
4368 #else
4369         log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
4370 #endif
4371         return (EDOOFUS);
4372 }
4373
4374 int
4375 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
4376 {
4377         uintptr_t *loc, new;
4378
4379         if (type >= nitems(sc->fw_msg_handler))
4380                 return (EINVAL);
4381
4382         /*
4383          * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
4384          * handler dispatch table.  Reject any attempt to install a handler for
4385          * this subtype.
4386          */
4387         if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
4388                 return (EINVAL);
4389
4390         new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
4391         loc = (uintptr_t *) &sc->fw_msg_handler[type];
4392         atomic_store_rel_ptr(loc, new);
4393
4394         return (0);
4395 }
4396
4397 static int
4398 t4_sysctls(struct adapter *sc)
4399 {
4400         struct sysctl_ctx_list *ctx;
4401         struct sysctl_oid *oid;
4402         struct sysctl_oid_list *children, *c0;
4403         static char *caps[] = {
4404                 "\20\1PPP\2QFC\3DCBX",                  /* caps[0] linkcaps */
4405                 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL"        /* caps[1] niccaps */
4406                     "\6HASHFILTER\7ETHOFLD",
4407                 "\20\1TOE",                             /* caps[2] toecaps */
4408                 "\20\1RDDP\2RDMAC",                     /* caps[3] rdmacaps */
4409                 "\20\1INITIATOR_PDU\2TARGET_PDU"        /* caps[4] iscsicaps */
4410                     "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
4411                     "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
4412                 "\20\1INITIATOR\2TARGET\3CTRL_OFLD"     /* caps[5] fcoecaps */
4413                     "\4PO_INITIAOR\5PO_TARGET"
4414         };
4415         static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
4416
4417         ctx = device_get_sysctl_ctx(sc->dev);
4418
4419         /*
4420          * dev.t4nex.X.
4421          */
4422         oid = device_get_sysctl_tree(sc->dev);
4423         c0 = children = SYSCTL_CHILDREN(oid);
4424
4425         sc->sc_do_rxcopy = 1;
4426         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
4427             &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
4428
4429         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
4430             sc->params.nports, "# of ports");
4431
4432         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
4433             NULL, chip_rev(sc), "chip hardware revision");
4434
4435         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
4436             CTLFLAG_RD, sc->fw_version, 0, "firmware version");
4437
4438         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
4439             CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
4440
4441         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
4442             sc->cfcsum, "config file checksum");
4443
4444         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
4445             CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
4446             sysctl_bitfield, "A", "available doorbells");
4447
4448         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
4449             CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
4450             sysctl_bitfield, "A", "available link capabilities");
4451
4452         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
4453             CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
4454             sysctl_bitfield, "A", "available NIC capabilities");
4455
4456         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
4457             CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
4458             sysctl_bitfield, "A", "available TCP offload capabilities");
4459
4460         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
4461             CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
4462             sysctl_bitfield, "A", "available RDMA capabilities");
4463
4464         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
4465             CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
4466             sysctl_bitfield, "A", "available iSCSI capabilities");
4467
4468         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
4469             CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
4470             sysctl_bitfield, "A", "available FCoE capabilities");
4471
4472         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
4473             sc->params.vpd.cclk, "core clock frequency (in KHz)");
4474
4475         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
4476             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
4477             sizeof(sc->sge.timer_val), sysctl_int_array, "A",
4478             "interrupt holdoff timer values (us)");
4479
4480         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
4481             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
4482             sizeof(sc->sge.counter_val), sysctl_int_array, "A",
4483             "interrupt holdoff packet counter values");
4484
4485         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
4486             NULL, sc->tids.nftids, "number of filters");
4487
4488         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
4489             CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
4490             "chip temperature (in Celsius)");
4491
4492         t4_sge_sysctls(sc, ctx, children);
4493
4494         sc->lro_timeout = 100;
4495         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
4496             &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
4497
4498 #ifdef SBUF_DRAIN
4499         /*
4500          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
4501          */
4502         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
4503             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
4504             "logs and miscellaneous information");
4505         children = SYSCTL_CHILDREN(oid);
4506
4507         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
4508             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4509             sysctl_cctrl, "A", "congestion control");
4510
4511         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
4512             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4513             sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
4514
4515         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
4516             CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
4517             sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
4518
4519         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
4520             CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
4521             sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
4522
4523         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
4524             CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
4525             sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
4526
4527         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
4528             CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
4529             sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
4530
4531         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
4532             CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
4533             sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
4534
4535         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
4536             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4537             sysctl_cim_la, "A", "CIM logic analyzer");
4538
4539         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
4540             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4541             sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
4542
4543         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
4544             CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
4545             sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
4546
4547         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
4548             CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
4549             sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
4550
4551         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
4552             CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
4553             sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
4554
4555         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
4556             CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
4557             sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
4558
4559         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
4560             CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
4561             sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
4562
4563         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
4564             CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
4565             sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
4566
4567         if (is_t5(sc)) {
4568                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
4569                     CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
4570                     sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
4571
4572                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
4573                     CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
4574                     sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
4575         }
4576
4577         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
4578             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4579             sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
4580
4581         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
4582             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4583             sysctl_cim_qcfg, "A", "CIM queue configuration");
4584
4585         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
4586             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4587             sysctl_cpl_stats, "A", "CPL statistics");
4588
4589         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
4590             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4591             sysctl_ddp_stats, "A", "DDP statistics");
4592
4593         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
4594             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4595             sysctl_devlog, "A", "firmware's device log");
4596
4597         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
4598             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4599             sysctl_fcoe_stats, "A", "FCoE statistics");
4600
4601         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
4602             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4603             sysctl_hw_sched, "A", "hardware scheduler ");
4604
4605         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
4606             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4607             sysctl_l2t, "A", "hardware L2 table");
4608
4609         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
4610             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4611             sysctl_lb_stats, "A", "loopback statistics");
4612
4613         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
4614             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4615             sysctl_meminfo, "A", "memory regions");
4616
4617         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
4618             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4619             sysctl_mps_tcam, "A", "MPS TCAM entries");
4620
4621         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
4622             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4623             sysctl_path_mtus, "A", "path MTUs");
4624
4625         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
4626             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4627             sysctl_pm_stats, "A", "PM statistics");
4628
4629         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
4630             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4631             sysctl_rdma_stats, "A", "RDMA statistics");
4632
4633         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
4634             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4635             sysctl_tcp_stats, "A", "TCP statistics");
4636
4637         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
4638             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4639             sysctl_tids, "A", "TID information");
4640
4641         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
4642             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4643             sysctl_tp_err_stats, "A", "TP error statistics");
4644
4645         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
4646             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4647             sysctl_tp_la, "A", "TP logic analyzer");
4648
4649         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
4650             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4651             sysctl_tx_rate, "A", "Tx rate");
4652
4653         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
4654             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4655             sysctl_ulprx_la, "A", "ULPRX logic analyzer");
4656
4657         if (is_t5(sc)) {
4658                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
4659                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4660                     sysctl_wcwr_stats, "A", "write combined work requests");
4661         }
4662 #endif
4663
4664 #ifdef TCP_OFFLOAD
4665         if (is_offload(sc)) {
4666                 /*
4667                  * dev.t4nex.X.toe.
4668                  */
4669                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
4670                     NULL, "TOE parameters");
4671                 children = SYSCTL_CHILDREN(oid);
4672
4673                 sc->tt.sndbuf = 256 * 1024;
4674                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
4675                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
4676
4677                 sc->tt.ddp = 0;
4678                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
4679                     &sc->tt.ddp, 0, "DDP allowed");
4680
4681                 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
4682                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
4683                     &sc->tt.indsz, 0, "DDP max indicate size allowed");
4684
4685                 sc->tt.ddp_thres =
4686                     G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
4687                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
4688                     &sc->tt.ddp_thres, 0, "DDP threshold");
4689
4690                 sc->tt.rx_coalesce = 1;
4691                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
4692                     CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
4693         }
4694 #endif
4695
4696
4697         return (0);
4698 }
4699
4700 static int
4701 cxgbe_sysctls(struct port_info *pi)
4702 {
4703         struct sysctl_ctx_list *ctx;
4704         struct sysctl_oid *oid;
4705         struct sysctl_oid_list *children;
4706         struct adapter *sc = pi->adapter;
4707
4708         ctx = device_get_sysctl_ctx(pi->dev);
4709
4710         /*
4711          * dev.cxgbe.X.
4712          */
4713         oid = device_get_sysctl_tree(pi->dev);
4714         children = SYSCTL_CHILDREN(oid);
4715
4716         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
4717            CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
4718         if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
4719                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
4720                     CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
4721                     "PHY temperature (in Celsius)");
4722                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
4723                     CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
4724                     "PHY firmware version");
4725         }
4726         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
4727             &pi->nrxq, 0, "# of rx queues");
4728         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
4729             &pi->ntxq, 0, "# of tx queues");
4730         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
4731             &pi->first_rxq, 0, "index of first rx queue");
4732         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
4733             &pi->first_txq, 0, "index of first tx queue");
4734         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
4735             CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU",
4736             "Reserve queue 0 for non-flowid packets");
4737
4738 #ifdef TCP_OFFLOAD
4739         if (is_offload(sc)) {
4740                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
4741                     &pi->nofldrxq, 0,
4742                     "# of rx queues for offloaded TCP connections");
4743                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
4744                     &pi->nofldtxq, 0,
4745                     "# of tx queues for offloaded TCP connections");
4746                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
4747                     CTLFLAG_RD, &pi->first_ofld_rxq, 0,
4748                     "index of first TOE rx queue");
4749                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
4750                     CTLFLAG_RD, &pi->first_ofld_txq, 0,
4751                     "index of first TOE tx queue");
4752         }
4753 #endif
4754 #ifdef DEV_NETMAP
4755         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
4756             &pi->nnmrxq, 0, "# of rx queues for netmap");
4757         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
4758             &pi->nnmtxq, 0, "# of tx queues for netmap");
4759         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
4760             CTLFLAG_RD, &pi->first_nm_rxq, 0,
4761             "index of first netmap rx queue");
4762         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
4763             CTLFLAG_RD, &pi->first_nm_txq, 0,
4764             "index of first netmap tx queue");
4765 #endif
4766
4767         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
4768             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
4769             "holdoff timer index");
4770         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
4771             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
4772             "holdoff packet counter index");
4773
4774         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
4775             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
4776             "rx queue size");
4777         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
4778             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
4779             "tx queue size");
4780
4781         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
4782             CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings,
4783             "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
4784
4785         /*
4786          * dev.cxgbe.X.stats.
4787          */
4788         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
4789             NULL, "port statistics");
4790         children = SYSCTL_CHILDREN(oid);
4791
4792 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
4793         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
4794             CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
4795             sysctl_handle_t4_reg64, "QU", desc)
4796
4797         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
4798             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
4799         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
4800             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
4801         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
4802             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
4803         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
4804             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
4805         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
4806             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
4807         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
4808             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
4809         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
4810             "# of tx frames in this range",
4811             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
4812         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
4813             "# of tx frames in this range",
4814             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
4815         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
4816             "# of tx frames in this range",
4817             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
4818         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
4819             "# of tx frames in this range",
4820             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
4821         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
4822             "# of tx frames in this range",
4823             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
4824         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
4825             "# of tx frames in this range",
4826             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
4827         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
4828             "# of tx frames in this range",
4829             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
4830         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
4831             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
4832         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
4833             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
4834         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
4835             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
4836         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
4837             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
4838         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
4839             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
4840         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
4841             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
4842         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
4843             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
4844         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
4845             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
4846         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
4847             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
4848         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
4849             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
4850
4851         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
4852             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
4853         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
4854             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
4855         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
4856             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
4857         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
4858             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
4859         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
4860             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
4861         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
4862             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
4863         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
4864             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
4865         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
4866             "# of frames received with bad FCS",
4867             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
4868         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
4869             "# of frames received with length error",
4870             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
4871         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
4872             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
4873         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
4874             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
4875         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
4876             "# of rx frames in this range",
4877             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
4878         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
4879             "# of rx frames in this range",
4880             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
4881         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
4882             "# of rx frames in this range",
4883             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
4884         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
4885             "# of rx frames in this range",
4886             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
4887         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
4888             "# of rx frames in this range",
4889             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
4890         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
4891             "# of rx frames in this range",
4892             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
4893         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
4894             "# of rx frames in this range",
4895             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
4896         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
4897             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
4898         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
4899             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
4900         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
4901             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
4902         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
4903             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
4904         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
4905             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
4906         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
4907             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
4908         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
4909             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
4910         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
4911             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
4912         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
4913             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
4914
4915 #undef SYSCTL_ADD_T4_REG64
4916
4917 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
4918         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
4919             &pi->stats.name, desc)
4920
4921         /* We get these from port_stats and they may be stale by upto 1s */
4922         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
4923             "# drops due to buffer-group 0 overflows");
4924         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
4925             "# drops due to buffer-group 1 overflows");
4926         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
4927             "# drops due to buffer-group 2 overflows");
4928         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
4929             "# drops due to buffer-group 3 overflows");
4930         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
4931             "# of buffer-group 0 truncated packets");
4932         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
4933             "# of buffer-group 1 truncated packets");
4934         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
4935             "# of buffer-group 2 truncated packets");
4936         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
4937             "# of buffer-group 3 truncated packets");
4938
4939 #undef SYSCTL_ADD_T4_PORTSTAT
4940
4941         return (0);
4942 }
4943
4944 static int
4945 sysctl_int_array(SYSCTL_HANDLER_ARGS)
4946 {
4947         int rc, *i;
4948         struct sbuf sb;
4949
4950         sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
4951         for (i = arg1; arg2; arg2 -= sizeof(int), i++)
4952                 sbuf_printf(&sb, "%d ", *i);
4953         sbuf_trim(&sb);
4954         sbuf_finish(&sb);
4955         rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
4956         sbuf_delete(&sb);
4957         return (rc);
4958 }
4959
4960 static int
4961 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
4962 {
4963         int rc;
4964         struct sbuf *sb;
4965
4966         rc = sysctl_wire_old_buffer(req, 0);
4967         if (rc != 0)
4968                 return(rc);
4969
4970         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
4971         if (sb == NULL)
4972                 return (ENOMEM);
4973
4974         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
4975         rc = sbuf_finish(sb);
4976         sbuf_delete(sb);
4977
4978         return (rc);
4979 }
4980
4981 static int
4982 sysctl_btphy(SYSCTL_HANDLER_ARGS)
4983 {
4984         struct port_info *pi = arg1;
4985         int op = arg2;
4986         struct adapter *sc = pi->adapter;
4987         u_int v;
4988         int rc;
4989
4990         rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt");
4991         if (rc)
4992                 return (rc);
4993         /* XXX: magic numbers */
4994         rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
4995             &v);
4996         end_synchronized_op(sc, 0);
4997         if (rc)
4998                 return (rc);
4999         if (op == 0)
5000                 v /= 256;
5001
5002         rc = sysctl_handle_int(oidp, &v, 0, req);
5003         return (rc);
5004 }
5005
5006 static int
5007 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
5008 {
5009         struct port_info *pi = arg1;
5010         int rc, val;
5011
5012         val = pi->rsrv_noflowq;
5013         rc = sysctl_handle_int(oidp, &val, 0, req);
5014         if (rc != 0 || req->newptr == NULL)
5015                 return (rc);
5016
5017         if ((val >= 1) && (pi->ntxq > 1))
5018                 pi->rsrv_noflowq = 1;
5019         else
5020                 pi->rsrv_noflowq = 0;
5021
5022         return (rc);
5023 }
5024
5025 static int
5026 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
5027 {
5028         struct port_info *pi = arg1;
5029         struct adapter *sc = pi->adapter;
5030         int idx, rc, i;
5031         struct sge_rxq *rxq;
5032 #ifdef TCP_OFFLOAD
5033         struct sge_ofld_rxq *ofld_rxq;
5034 #endif
5035         uint8_t v;
5036
5037         idx = pi->tmr_idx;
5038
5039         rc = sysctl_handle_int(oidp, &idx, 0, req);
5040         if (rc != 0 || req->newptr == NULL)
5041                 return (rc);
5042
5043         if (idx < 0 || idx >= SGE_NTIMERS)
5044                 return (EINVAL);
5045
5046         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5047             "t4tmr");
5048         if (rc)
5049                 return (rc);
5050
5051         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
5052         for_each_rxq(pi, i, rxq) {
5053 #ifdef atomic_store_rel_8
5054                 atomic_store_rel_8(&rxq->iq.intr_params, v);
5055 #else
5056                 rxq->iq.intr_params = v;
5057 #endif
5058         }
5059 #ifdef TCP_OFFLOAD
5060         for_each_ofld_rxq(pi, i, ofld_rxq) {
5061 #ifdef atomic_store_rel_8
5062                 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
5063 #else
5064                 ofld_rxq->iq.intr_params = v;
5065 #endif
5066         }
5067 #endif
5068         pi->tmr_idx = idx;
5069
5070         end_synchronized_op(sc, LOCK_HELD);
5071         return (0);
5072 }
5073
5074 static int
5075 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
5076 {
5077         struct port_info *pi = arg1;
5078         struct adapter *sc = pi->adapter;
5079         int idx, rc;
5080
5081         idx = pi->pktc_idx;
5082
5083         rc = sysctl_handle_int(oidp, &idx, 0, req);
5084         if (rc != 0 || req->newptr == NULL)
5085                 return (rc);
5086
5087         if (idx < -1 || idx >= SGE_NCOUNTERS)
5088                 return (EINVAL);
5089
5090         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5091             "t4pktc");
5092         if (rc)
5093                 return (rc);
5094
5095         if (pi->flags & PORT_INIT_DONE)
5096                 rc = EBUSY; /* cannot be changed once the queues are created */
5097         else
5098                 pi->pktc_idx = idx;
5099
5100         end_synchronized_op(sc, LOCK_HELD);
5101         return (rc);
5102 }
5103
5104 static int
5105 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
5106 {
5107         struct port_info *pi = arg1;
5108         struct adapter *sc = pi->adapter;
5109         int qsize, rc;
5110
5111         qsize = pi->qsize_rxq;
5112
5113         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5114         if (rc != 0 || req->newptr == NULL)
5115                 return (rc);
5116
5117         if (qsize < 128 || (qsize & 7))
5118                 return (EINVAL);
5119
5120         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5121             "t4rxqs");
5122         if (rc)
5123                 return (rc);
5124
5125         if (pi->flags & PORT_INIT_DONE)
5126                 rc = EBUSY; /* cannot be changed once the queues are created */
5127         else
5128                 pi->qsize_rxq = qsize;
5129
5130         end_synchronized_op(sc, LOCK_HELD);
5131         return (rc);
5132 }
5133
5134 static int
5135 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
5136 {
5137         struct port_info *pi = arg1;
5138         struct adapter *sc = pi->adapter;
5139         int qsize, rc;
5140
5141         qsize = pi->qsize_txq;
5142
5143         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5144         if (rc != 0 || req->newptr == NULL)
5145                 return (rc);
5146
5147         /* bufring size must be powerof2 */
5148         if (qsize < 128 || !powerof2(qsize))
5149                 return (EINVAL);
5150
5151         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5152             "t4txqs");
5153         if (rc)
5154                 return (rc);
5155
5156         if (pi->flags & PORT_INIT_DONE)
5157                 rc = EBUSY; /* cannot be changed once the queues are created */
5158         else
5159                 pi->qsize_txq = qsize;
5160
5161         end_synchronized_op(sc, LOCK_HELD);
5162         return (rc);
5163 }
5164
5165 static int
5166 sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
5167 {
5168         struct port_info *pi = arg1;
5169         struct adapter *sc = pi->adapter;
5170         struct link_config *lc = &pi->link_cfg;
5171         int rc;
5172
5173         if (req->newptr == NULL) {
5174                 struct sbuf *sb;
5175                 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX";
5176
5177                 rc = sysctl_wire_old_buffer(req, 0);
5178                 if (rc != 0)
5179                         return(rc);
5180
5181                 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5182                 if (sb == NULL)
5183                         return (ENOMEM);
5184
5185                 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits);
5186                 rc = sbuf_finish(sb);
5187                 sbuf_delete(sb);
5188         } else {
5189                 char s[2];
5190                 int n;
5191
5192                 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX));
5193                 s[1] = 0;
5194
5195                 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
5196                 if (rc != 0)
5197                         return(rc);
5198
5199                 if (s[1] != 0)
5200                         return (EINVAL);
5201                 if (s[0] < '0' || s[0] > '9')
5202                         return (EINVAL);        /* not a number */
5203                 n = s[0] - '0';
5204                 if (n & ~(PAUSE_TX | PAUSE_RX))
5205                         return (EINVAL);        /* some other bit is set too */
5206
5207                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE");
5208                 if (rc)
5209                         return (rc);
5210                 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) {
5211                         int link_ok = lc->link_ok;
5212
5213                         lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX);
5214                         lc->requested_fc |= n;
5215                         rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc);
5216                         lc->link_ok = link_ok;  /* restore */
5217                 }
5218                 end_synchronized_op(sc, 0);
5219         }
5220
5221         return (rc);
5222 }
5223
5224 static int
5225 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
5226 {
5227         struct adapter *sc = arg1;
5228         int reg = arg2;
5229         uint64_t val;
5230
5231         val = t4_read_reg64(sc, reg);
5232
5233         return (sysctl_handle_64(oidp, &val, 0, req));
5234 }
5235
5236 static int
5237 sysctl_temperature(SYSCTL_HANDLER_ARGS)
5238 {
5239         struct adapter *sc = arg1;
5240         int rc, t;
5241         uint32_t param, val;
5242
5243         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
5244         if (rc)
5245                 return (rc);
5246         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5247             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5248             V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
5249         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
5250         end_synchronized_op(sc, 0);
5251         if (rc)
5252                 return (rc);
5253
5254         /* unknown is returned as 0 but we display -1 in that case */
5255         t = val == 0 ? -1 : val;
5256
5257         rc = sysctl_handle_int(oidp, &t, 0, req);
5258         return (rc);
5259 }
5260
5261 #ifdef SBUF_DRAIN
5262 static int
5263 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
5264 {
5265         struct adapter *sc = arg1;
5266         struct sbuf *sb;
5267         int rc, i;
5268         uint16_t incr[NMTUS][NCCTRL_WIN];
5269         static const char *dec_fac[] = {
5270                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
5271                 "0.9375"
5272         };
5273
5274         rc = sysctl_wire_old_buffer(req, 0);
5275         if (rc != 0)
5276                 return (rc);
5277
5278         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5279         if (sb == NULL)
5280                 return (ENOMEM);
5281
5282         t4_read_cong_tbl(sc, incr);
5283
5284         for (i = 0; i < NCCTRL_WIN; ++i) {
5285                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
5286                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
5287                     incr[5][i], incr[6][i], incr[7][i]);
5288                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
5289                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
5290                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
5291                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
5292         }
5293
5294         rc = sbuf_finish(sb);
5295         sbuf_delete(sb);
5296
5297         return (rc);
5298 }
5299
5300 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
5301         "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",   /* ibq's */
5302         "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
5303         "SGE0-RX", "SGE1-RX"    /* additional obq's (T5 onwards) */
5304 };
5305
5306 static int
5307 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
5308 {
5309         struct adapter *sc = arg1;
5310         struct sbuf *sb;
5311         int rc, i, n, qid = arg2;
5312         uint32_t *buf, *p;
5313         char *qtype;
5314         u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
5315
5316         KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
5317             ("%s: bad qid %d\n", __func__, qid));
5318
5319         if (qid < CIM_NUM_IBQ) {
5320                 /* inbound queue */
5321                 qtype = "IBQ";
5322                 n = 4 * CIM_IBQ_SIZE;
5323                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5324                 rc = t4_read_cim_ibq(sc, qid, buf, n);
5325         } else {
5326                 /* outbound queue */
5327                 qtype = "OBQ";
5328                 qid -= CIM_NUM_IBQ;
5329                 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
5330                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5331                 rc = t4_read_cim_obq(sc, qid, buf, n);
5332         }
5333
5334         if (rc < 0) {
5335                 rc = -rc;
5336                 goto done;
5337         }
5338         n = rc * sizeof(uint32_t);      /* rc has # of words actually read */
5339
5340         rc = sysctl_wire_old_buffer(req, 0);
5341         if (rc != 0)
5342                 goto done;
5343
5344         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5345         if (sb == NULL) {
5346                 rc = ENOMEM;
5347                 goto done;
5348         }
5349
5350         sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
5351         for (i = 0, p = buf; i < n; i += 16, p += 4)
5352                 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
5353                     p[2], p[3]);
5354
5355         rc = sbuf_finish(sb);
5356         sbuf_delete(sb);
5357 done:
5358         free(buf, M_CXGBE);
5359         return (rc);
5360 }
5361
5362 static int
5363 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
5364 {
5365         struct adapter *sc = arg1;
5366         u_int cfg;
5367         struct sbuf *sb;
5368         uint32_t *buf, *p;
5369         int rc;
5370
5371         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
5372         if (rc != 0)
5373                 return (rc);
5374
5375         rc = sysctl_wire_old_buffer(req, 0);
5376         if (rc != 0)
5377                 return (rc);
5378
5379         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5380         if (sb == NULL)
5381                 return (ENOMEM);
5382
5383         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
5384             M_ZERO | M_WAITOK);
5385
5386         rc = -t4_cim_read_la(sc, buf, NULL);
5387         if (rc != 0)
5388                 goto done;
5389
5390         sbuf_printf(sb, "Status   Data      PC%s",
5391             cfg & F_UPDBGLACAPTPCONLY ? "" :
5392             "     LS0Stat  LS0Addr             LS0Data");
5393
5394         KASSERT((sc->params.cim_la_size & 7) == 0,
5395             ("%s: p will walk off the end of buf", __func__));
5396
5397         for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
5398                 if (cfg & F_UPDBGLACAPTPCONLY) {
5399                         sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
5400                             p[6], p[7]);
5401                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
5402                             (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
5403                             p[4] & 0xff, p[5] >> 8);
5404                         sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
5405                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5406                             p[1] & 0xf, p[2] >> 4);
5407                 } else {
5408                         sbuf_printf(sb,
5409                             "\n  %02x   %x%07x %x%07x %08x %08x "
5410                             "%08x%08x%08x%08x",
5411                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5412                             p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
5413                             p[6], p[7]);
5414                 }
5415         }
5416
5417         rc = sbuf_finish(sb);
5418         sbuf_delete(sb);
5419 done:
5420         free(buf, M_CXGBE);
5421         return (rc);
5422 }
5423
5424 static int
5425 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
5426 {
5427         struct adapter *sc = arg1;
5428         u_int i;
5429         struct sbuf *sb;
5430         uint32_t *buf, *p;
5431         int rc;
5432
5433         rc = sysctl_wire_old_buffer(req, 0);
5434         if (rc != 0)
5435                 return (rc);
5436
5437         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5438         if (sb == NULL)
5439                 return (ENOMEM);
5440
5441         buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
5442             M_ZERO | M_WAITOK);
5443
5444         t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
5445         p = buf;
5446
5447         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5448                 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
5449                     p[1], p[0]);
5450         }
5451
5452         sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
5453         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5454                 sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
5455                     (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
5456                     (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
5457                     (p[1] >> 2) | ((p[2] & 3) << 30),
5458                     (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
5459                     p[0] & 1);
5460         }
5461
5462         rc = sbuf_finish(sb);
5463         sbuf_delete(sb);
5464         free(buf, M_CXGBE);
5465         return (rc);
5466 }
5467
5468 static int
5469 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
5470 {
5471         struct adapter *sc = arg1;
5472         u_int i;
5473         struct sbuf *sb;
5474         uint32_t *buf, *p;
5475         int rc;
5476
5477         rc = sysctl_wire_old_buffer(req, 0);
5478         if (rc != 0)
5479                 return (rc);
5480
5481         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5482         if (sb == NULL)
5483                 return (ENOMEM);
5484
5485         buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
5486             M_ZERO | M_WAITOK);
5487
5488         t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
5489         p = buf;
5490
5491         sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
5492         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5493                 sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
5494                     (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
5495                     p[4], p[3], p[2], p[1], p[0]);
5496         }
5497
5498         sbuf_printf(sb, "\n\nCntl ID               Data");
5499         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5500                 sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
5501                     (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
5502         }
5503
5504         rc = sbuf_finish(sb);
5505         sbuf_delete(sb);
5506         free(buf, M_CXGBE);
5507         return (rc);
5508 }
5509
5510 static int
5511 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
5512 {
5513         struct adapter *sc = arg1;
5514         struct sbuf *sb;
5515         int rc, i;
5516         uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5517         uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5518         uint16_t thres[CIM_NUM_IBQ];
5519         uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
5520         uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
5521         u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
5522
5523         if (is_t4(sc)) {
5524                 cim_num_obq = CIM_NUM_OBQ;
5525                 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
5526                 obq_rdaddr = A_UP_OBQ_0_REALADDR;
5527         } else {
5528                 cim_num_obq = CIM_NUM_OBQ_T5;
5529                 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
5530                 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
5531         }
5532         nq = CIM_NUM_IBQ + cim_num_obq;
5533
5534         rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
5535         if (rc == 0)
5536                 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
5537         if (rc != 0)
5538                 return (rc);
5539
5540         t4_read_cimq_cfg(sc, base, size, thres);
5541
5542         rc = sysctl_wire_old_buffer(req, 0);
5543         if (rc != 0)
5544                 return (rc);
5545
5546         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5547         if (sb == NULL)
5548                 return (ENOMEM);
5549
5550         sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
5551
5552         for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
5553                 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
5554                     qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
5555                     G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5556                     G_QUEREMFLITS(p[2]) * 16);
5557         for ( ; i < nq; i++, p += 4, wr += 2)
5558                 sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
5559                     base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
5560                     wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5561                     G_QUEREMFLITS(p[2]) * 16);
5562
5563         rc = sbuf_finish(sb);
5564         sbuf_delete(sb);
5565
5566         return (rc);
5567 }
5568
5569 static int
5570 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
5571 {
5572         struct adapter *sc = arg1;
5573         struct sbuf *sb;
5574         int rc;
5575         struct tp_cpl_stats stats;
5576
5577         rc = sysctl_wire_old_buffer(req, 0);
5578         if (rc != 0)
5579                 return (rc);
5580
5581         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5582         if (sb == NULL)
5583                 return (ENOMEM);
5584
5585         t4_tp_get_cpl_stats(sc, &stats);
5586
5587         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
5588             "channel 3\n");
5589         sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
5590                    stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
5591         sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
5592                    stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
5593
5594         rc = sbuf_finish(sb);
5595         sbuf_delete(sb);
5596
5597         return (rc);
5598 }
5599
5600 static int
5601 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
5602 {
5603         struct adapter *sc = arg1;
5604         struct sbuf *sb;
5605         int rc;
5606         struct tp_usm_stats stats;
5607
5608         rc = sysctl_wire_old_buffer(req, 0);
5609         if (rc != 0)
5610                 return(rc);
5611
5612         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5613         if (sb == NULL)
5614                 return (ENOMEM);
5615
5616         t4_get_usm_stats(sc, &stats);
5617
5618         sbuf_printf(sb, "Frames: %u\n", stats.frames);
5619         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
5620         sbuf_printf(sb, "Drops:  %u", stats.drops);
5621
5622         rc = sbuf_finish(sb);
5623         sbuf_delete(sb);
5624
5625         return (rc);
5626 }
5627
5628 const char *devlog_level_strings[] = {
5629         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
5630         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
5631         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
5632         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
5633         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
5634         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
5635 };
5636
5637 const char *devlog_facility_strings[] = {
5638         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
5639         [FW_DEVLOG_FACILITY_CF]         = "CF",
5640         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
5641         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
5642         [FW_DEVLOG_FACILITY_RES]        = "RES",
5643         [FW_DEVLOG_FACILITY_HW]         = "HW",
5644         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
5645         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
5646         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
5647         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
5648         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
5649         [FW_DEVLOG_FACILITY_VI]         = "VI",
5650         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
5651         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
5652         [FW_DEVLOG_FACILITY_TM]         = "TM",
5653         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
5654         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
5655         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
5656         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
5657         [FW_DEVLOG_FACILITY_RI]         = "RI",
5658         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
5659         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
5660         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
5661         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE"
5662 };
5663
5664 static int
5665 sysctl_devlog(SYSCTL_HANDLER_ARGS)
5666 {
5667         struct adapter *sc = arg1;
5668         struct devlog_params *dparams = &sc->params.devlog;
5669         struct fw_devlog_e *buf, *e;
5670         int i, j, rc, nentries, first = 0, m;
5671         struct sbuf *sb;
5672         uint64_t ftstamp = UINT64_MAX;
5673
5674         if (dparams->start == 0) {
5675                 dparams->memtype = FW_MEMTYPE_EDC0;
5676                 dparams->start = 0x84000;
5677                 dparams->size = 32768;
5678         }
5679
5680         nentries = dparams->size / sizeof(struct fw_devlog_e);
5681
5682         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
5683         if (buf == NULL)
5684                 return (ENOMEM);
5685
5686         m = fwmtype_to_hwmtype(dparams->memtype);
5687         rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
5688         if (rc != 0)
5689                 goto done;
5690
5691         for (i = 0; i < nentries; i++) {
5692                 e = &buf[i];
5693
5694                 if (e->timestamp == 0)
5695                         break;  /* end */
5696
5697                 e->timestamp = be64toh(e->timestamp);
5698                 e->seqno = be32toh(e->seqno);
5699                 for (j = 0; j < 8; j++)
5700                         e->params[j] = be32toh(e->params[j]);
5701
5702                 if (e->timestamp < ftstamp) {
5703                         ftstamp = e->timestamp;
5704                         first = i;
5705                 }
5706         }
5707
5708         if (buf[first].timestamp == 0)
5709                 goto done;      /* nothing in the log */
5710
5711         rc = sysctl_wire_old_buffer(req, 0);
5712         if (rc != 0)
5713                 goto done;
5714
5715         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5716         if (sb == NULL) {
5717                 rc = ENOMEM;
5718                 goto done;
5719         }
5720         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
5721             "Seq#", "Tstamp", "Level", "Facility", "Message");
5722
5723         i = first;
5724         do {
5725                 e = &buf[i];
5726                 if (e->timestamp == 0)
5727                         break;  /* end */
5728
5729                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
5730                     e->seqno, e->timestamp,
5731                     (e->level < nitems(devlog_level_strings) ?
5732                         devlog_level_strings[e->level] : "UNKNOWN"),
5733                     (e->facility < nitems(devlog_facility_strings) ?
5734                         devlog_facility_strings[e->facility] : "UNKNOWN"));
5735                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
5736                     e->params[2], e->params[3], e->params[4],
5737                     e->params[5], e->params[6], e->params[7]);
5738
5739                 if (++i == nentries)
5740                         i = 0;
5741         } while (i != first);
5742
5743         rc = sbuf_finish(sb);
5744         sbuf_delete(sb);
5745 done:
5746         free(buf, M_CXGBE);
5747         return (rc);
5748 }
5749
5750 static int
5751 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
5752 {
5753         struct adapter *sc = arg1;
5754         struct sbuf *sb;
5755         int rc;
5756         struct tp_fcoe_stats stats[4];
5757
5758         rc = sysctl_wire_old_buffer(req, 0);
5759         if (rc != 0)
5760                 return (rc);
5761
5762         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5763         if (sb == NULL)
5764                 return (ENOMEM);
5765
5766         t4_get_fcoe_stats(sc, 0, &stats[0]);
5767         t4_get_fcoe_stats(sc, 1, &stats[1]);
5768         t4_get_fcoe_stats(sc, 2, &stats[2]);
5769         t4_get_fcoe_stats(sc, 3, &stats[3]);
5770
5771         sbuf_printf(sb, "                   channel 0        channel 1        "
5772             "channel 2        channel 3\n");
5773         sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
5774             stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
5775             stats[3].octetsDDP);
5776         sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
5777             stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
5778         sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
5779             stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
5780             stats[3].framesDrop);
5781
5782         rc = sbuf_finish(sb);
5783         sbuf_delete(sb);
5784
5785         return (rc);
5786 }
5787
5788 static int
5789 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
5790 {
5791         struct adapter *sc = arg1;
5792         struct sbuf *sb;
5793         int rc, i;
5794         unsigned int map, kbps, ipg, mode;
5795         unsigned int pace_tab[NTX_SCHED];
5796
5797         rc = sysctl_wire_old_buffer(req, 0);
5798         if (rc != 0)
5799                 return (rc);
5800
5801         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5802         if (sb == NULL)
5803                 return (ENOMEM);
5804
5805         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
5806         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
5807         t4_read_pace_tbl(sc, pace_tab);
5808
5809         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
5810             "Class IPG (0.1 ns)   Flow IPG (us)");
5811
5812         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
5813                 t4_get_tx_sched(sc, i, &kbps, &ipg);
5814                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
5815                     (mode & (1 << i)) ? "flow" : "class", map & 3);
5816                 if (kbps)
5817                         sbuf_printf(sb, "%9u     ", kbps);
5818                 else
5819                         sbuf_printf(sb, " disabled     ");
5820
5821                 if (ipg)
5822                         sbuf_printf(sb, "%13u        ", ipg);
5823                 else
5824                         sbuf_printf(sb, "     disabled        ");
5825
5826                 if (pace_tab[i])
5827                         sbuf_printf(sb, "%10u", pace_tab[i]);
5828                 else
5829                         sbuf_printf(sb, "  disabled");
5830         }
5831
5832         rc = sbuf_finish(sb);
5833         sbuf_delete(sb);
5834
5835         return (rc);
5836 }
5837
5838 static int
5839 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
5840 {
5841         struct adapter *sc = arg1;
5842         struct sbuf *sb;
5843         int rc, i, j;
5844         uint64_t *p0, *p1;
5845         struct lb_port_stats s[2];
5846         static const char *stat_name[] = {
5847                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
5848                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
5849                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
5850                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
5851                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
5852                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
5853                 "BG2FramesTrunc:", "BG3FramesTrunc:"
5854         };
5855
5856         rc = sysctl_wire_old_buffer(req, 0);
5857         if (rc != 0)
5858                 return (rc);
5859
5860         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5861         if (sb == NULL)
5862                 return (ENOMEM);
5863
5864         memset(s, 0, sizeof(s));
5865
5866         for (i = 0; i < 4; i += 2) {
5867                 t4_get_lb_stats(sc, i, &s[0]);
5868                 t4_get_lb_stats(sc, i + 1, &s[1]);
5869
5870                 p0 = &s[0].octets;
5871                 p1 = &s[1].octets;
5872                 sbuf_printf(sb, "%s                       Loopback %u"
5873                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
5874
5875                 for (j = 0; j < nitems(stat_name); j++)
5876                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
5877                                    *p0++, *p1++);
5878         }
5879
5880         rc = sbuf_finish(sb);
5881         sbuf_delete(sb);
5882
5883         return (rc);
5884 }
5885
5886 static int
5887 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
5888 {
5889         int rc = 0;
5890         struct port_info *pi = arg1;
5891         struct sbuf *sb;
5892         static const char *linkdnreasons[] = {
5893                 "non-specific", "remote fault", "autoneg failed", "reserved3",
5894                 "PHY overheated", "unknown", "rx los", "reserved7"
5895         };
5896
5897         rc = sysctl_wire_old_buffer(req, 0);
5898         if (rc != 0)
5899                 return(rc);
5900         sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
5901         if (sb == NULL)
5902                 return (ENOMEM);
5903
5904         if (pi->linkdnrc < 0)
5905                 sbuf_printf(sb, "n/a");
5906         else if (pi->linkdnrc < nitems(linkdnreasons))
5907                 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
5908         else
5909                 sbuf_printf(sb, "%d", pi->linkdnrc);
5910
5911         rc = sbuf_finish(sb);
5912         sbuf_delete(sb);
5913
5914         return (rc);
5915 }
5916
5917 struct mem_desc {
5918         unsigned int base;
5919         unsigned int limit;
5920         unsigned int idx;
5921 };
5922
5923 static int
5924 mem_desc_cmp(const void *a, const void *b)
5925 {
5926         return ((const struct mem_desc *)a)->base -
5927                ((const struct mem_desc *)b)->base;
5928 }
5929
5930 static void
5931 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
5932     unsigned int to)
5933 {
5934         unsigned int size;
5935
5936         size = to - from + 1;
5937         if (size == 0)
5938                 return;
5939
5940         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
5941         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
5942 }
5943
5944 static int
5945 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
5946 {
5947         struct adapter *sc = arg1;
5948         struct sbuf *sb;
5949         int rc, i, n;
5950         uint32_t lo, hi, used, alloc;
5951         static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
5952         static const char *region[] = {
5953                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
5954                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
5955                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
5956                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
5957                 "RQUDP region:", "PBL region:", "TXPBL region:",
5958                 "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
5959                 "On-chip queues:"
5960         };
5961         struct mem_desc avail[4];
5962         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
5963         struct mem_desc *md = mem;
5964
5965         rc = sysctl_wire_old_buffer(req, 0);
5966         if (rc != 0)
5967                 return (rc);
5968
5969         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5970         if (sb == NULL)
5971                 return (ENOMEM);
5972
5973         for (i = 0; i < nitems(mem); i++) {
5974                 mem[i].limit = 0;
5975                 mem[i].idx = i;
5976         }
5977
5978         /* Find and sort the populated memory ranges */
5979         i = 0;
5980         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
5981         if (lo & F_EDRAM0_ENABLE) {
5982                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
5983                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
5984                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
5985                 avail[i].idx = 0;
5986                 i++;
5987         }
5988         if (lo & F_EDRAM1_ENABLE) {
5989                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
5990                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
5991                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
5992                 avail[i].idx = 1;
5993                 i++;
5994         }
5995         if (lo & F_EXT_MEM_ENABLE) {
5996                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
5997                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
5998                 avail[i].limit = avail[i].base +
5999                     (G_EXT_MEM_SIZE(hi) << 20);
6000                 avail[i].idx = is_t4(sc) ? 2 : 3;       /* Call it MC for T4 */
6001                 i++;
6002         }
6003         if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
6004                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
6005                 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
6006                 avail[i].limit = avail[i].base +
6007                     (G_EXT_MEM1_SIZE(hi) << 20);
6008                 avail[i].idx = 4;
6009                 i++;
6010         }
6011         if (!i)                                    /* no memory available */
6012                 return 0;
6013         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
6014
6015         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
6016         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
6017         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
6018         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
6019         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
6020         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
6021         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
6022         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
6023         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
6024
6025         /* the next few have explicit upper bounds */
6026         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
6027         md->limit = md->base - 1 +
6028                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
6029                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
6030         md++;
6031
6032         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
6033         md->limit = md->base - 1 +
6034                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
6035                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
6036         md++;
6037
6038         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6039                 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
6040                 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
6041                 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
6042         } else {
6043                 md->base = 0;
6044                 md->idx = nitems(region);  /* hide it */
6045         }
6046         md++;
6047
6048 #define ulp_region(reg) \
6049         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
6050         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
6051
6052         ulp_region(RX_ISCSI);
6053         ulp_region(RX_TDDP);
6054         ulp_region(TX_TPT);
6055         ulp_region(RX_STAG);
6056         ulp_region(RX_RQ);
6057         ulp_region(RX_RQUDP);
6058         ulp_region(RX_PBL);
6059         ulp_region(TX_PBL);
6060 #undef ulp_region
6061
6062         md->base = 0;
6063         md->idx = nitems(region);
6064         if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
6065                 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
6066                 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
6067                     A_SGE_DBVFIFO_SIZE))) << 2) - 1;
6068         }
6069         md++;
6070
6071         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
6072         md->limit = md->base + sc->tids.ntids - 1;
6073         md++;
6074         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
6075         md->limit = md->base + sc->tids.ntids - 1;
6076         md++;
6077
6078         md->base = sc->vres.ocq.start;
6079         if (sc->vres.ocq.size)
6080                 md->limit = md->base + sc->vres.ocq.size - 1;
6081         else
6082                 md->idx = nitems(region);  /* hide it */
6083         md++;
6084
6085         /* add any address-space holes, there can be up to 3 */
6086         for (n = 0; n < i - 1; n++)
6087                 if (avail[n].limit < avail[n + 1].base)
6088                         (md++)->base = avail[n].limit;
6089         if (avail[n].limit)
6090                 (md++)->base = avail[n].limit;
6091
6092         n = md - mem;
6093         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
6094
6095         for (lo = 0; lo < i; lo++)
6096                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
6097                                 avail[lo].limit - 1);
6098
6099         sbuf_printf(sb, "\n");
6100         for (i = 0; i < n; i++) {
6101                 if (mem[i].idx >= nitems(region))
6102                         continue;                        /* skip holes */
6103                 if (!mem[i].limit)
6104                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
6105                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
6106                                 mem[i].limit);
6107         }
6108
6109         sbuf_printf(sb, "\n");
6110         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
6111         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
6112         mem_region_show(sb, "uP RAM:", lo, hi);
6113
6114         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
6115         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
6116         mem_region_show(sb, "uP Extmem2:", lo, hi);
6117
6118         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
6119         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
6120                    G_PMRXMAXPAGE(lo),
6121                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
6122                    (lo & F_PMRXNUMCHN) ? 2 : 1);
6123
6124         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
6125         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
6126         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
6127                    G_PMTXMAXPAGE(lo),
6128                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
6129                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
6130         sbuf_printf(sb, "%u p-structs\n",
6131                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
6132
6133         for (i = 0; i < 4; i++) {
6134                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
6135                 if (is_t4(sc)) {
6136                         used = G_USED(lo);
6137                         alloc = G_ALLOC(lo);
6138                 } else {
6139                         used = G_T5_USED(lo);
6140                         alloc = G_T5_ALLOC(lo);
6141                 }
6142                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
6143                            i, used, alloc);
6144         }
6145         for (i = 0; i < 4; i++) {
6146                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
6147                 if (is_t4(sc)) {
6148                         used = G_USED(lo);
6149                         alloc = G_ALLOC(lo);
6150                 } else {
6151                         used = G_T5_USED(lo);
6152                         alloc = G_T5_ALLOC(lo);
6153                 }
6154                 sbuf_printf(sb,
6155                            "\nLoopback %d using %u pages out of %u allocated",
6156                            i, used, alloc);
6157         }
6158
6159         rc = sbuf_finish(sb);
6160         sbuf_delete(sb);
6161
6162         return (rc);
6163 }
6164
6165 static inline void
6166 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
6167 {
6168         *mask = x | y;
6169         y = htobe64(y);
6170         memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
6171 }
6172
6173 static int
6174 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
6175 {
6176         struct adapter *sc = arg1;
6177         struct sbuf *sb;
6178         int rc, i, n;
6179
6180         rc = sysctl_wire_old_buffer(req, 0);
6181         if (rc != 0)
6182                 return (rc);
6183
6184         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6185         if (sb == NULL)
6186                 return (ENOMEM);
6187
6188         sbuf_printf(sb,
6189             "Idx  Ethernet address     Mask     Vld Ports PF"
6190             "  VF              Replication             P0 P1 P2 P3  ML");
6191         n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
6192             NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
6193         for (i = 0; i < n; i++) {
6194                 uint64_t tcamx, tcamy, mask;
6195                 uint32_t cls_lo, cls_hi;
6196                 uint8_t addr[ETHER_ADDR_LEN];
6197
6198                 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
6199                 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
6200                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
6201                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
6202
6203                 if (tcamx & tcamy)
6204                         continue;
6205
6206                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
6207                 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
6208                            "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
6209                            addr[3], addr[4], addr[5], (uintmax_t)mask,
6210                            (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
6211                            G_PORTMAP(cls_hi), G_PF(cls_lo),
6212                            (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
6213
6214                 if (cls_lo & F_REPLICATE) {
6215                         struct fw_ldst_cmd ldst_cmd;
6216
6217                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
6218                         ldst_cmd.op_to_addrspace =
6219                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
6220                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
6221                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
6222                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
6223                         ldst_cmd.u.mps.fid_ctl =
6224                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
6225                                 V_FW_LDST_CMD_CTL(i));
6226
6227                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
6228                             "t4mps");
6229                         if (rc)
6230                                 break;
6231                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
6232                             sizeof(ldst_cmd), &ldst_cmd);
6233                         end_synchronized_op(sc, 0);
6234
6235                         if (rc != 0) {
6236                                 sbuf_printf(sb,
6237                                     " ------------ error %3u ------------", rc);
6238                                 rc = 0;
6239                         } else {
6240                                 sbuf_printf(sb, " %08x %08x %08x %08x",
6241                                     be32toh(ldst_cmd.u.mps.rplc127_96),
6242                                     be32toh(ldst_cmd.u.mps.rplc95_64),
6243                                     be32toh(ldst_cmd.u.mps.rplc63_32),
6244                                     be32toh(ldst_cmd.u.mps.rplc31_0));
6245                         }
6246                 } else
6247                         sbuf_printf(sb, "%36s", "");
6248
6249                 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
6250                     G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
6251                     G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
6252         }
6253
6254         if (rc)
6255                 (void) sbuf_finish(sb);
6256         else
6257                 rc = sbuf_finish(sb);
6258         sbuf_delete(sb);
6259
6260         return (rc);
6261 }
6262
6263 static int
6264 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
6265 {
6266         struct adapter *sc = arg1;
6267         struct sbuf *sb;
6268         int rc;
6269         uint16_t mtus[NMTUS];
6270
6271         rc = sysctl_wire_old_buffer(req, 0);
6272         if (rc != 0)
6273                 return (rc);
6274
6275         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6276         if (sb == NULL)
6277                 return (ENOMEM);
6278
6279         t4_read_mtu_tbl(sc, mtus, NULL);
6280
6281         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
6282             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
6283             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
6284             mtus[14], mtus[15]);
6285
6286         rc = sbuf_finish(sb);
6287         sbuf_delete(sb);
6288
6289         return (rc);
6290 }
6291
6292 static int
6293 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
6294 {
6295         struct adapter *sc = arg1;
6296         struct sbuf *sb;
6297         int rc, i;
6298         uint32_t cnt[PM_NSTATS];
6299         uint64_t cyc[PM_NSTATS];
6300         static const char *rx_stats[] = {
6301                 "Read:", "Write bypass:", "Write mem:", "Flush:"
6302         };
6303         static const char *tx_stats[] = {
6304                 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
6305         };
6306
6307         rc = sysctl_wire_old_buffer(req, 0);
6308         if (rc != 0)
6309                 return (rc);
6310
6311         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6312         if (sb == NULL)
6313                 return (ENOMEM);
6314
6315         t4_pmtx_get_stats(sc, cnt, cyc);
6316         sbuf_printf(sb, "                Tx pcmds             Tx bytes");
6317         for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
6318                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
6319                     cyc[i]);
6320
6321         t4_pmrx_get_stats(sc, cnt, cyc);
6322         sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
6323         for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
6324                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
6325                     cyc[i]);
6326
6327         rc = sbuf_finish(sb);
6328         sbuf_delete(sb);
6329
6330         return (rc);
6331 }
6332
6333 static int
6334 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
6335 {
6336         struct adapter *sc = arg1;
6337         struct sbuf *sb;
6338         int rc;
6339         struct tp_rdma_stats stats;
6340
6341         rc = sysctl_wire_old_buffer(req, 0);
6342         if (rc != 0)
6343                 return (rc);
6344
6345         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6346         if (sb == NULL)
6347                 return (ENOMEM);
6348
6349         t4_tp_get_rdma_stats(sc, &stats);
6350         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
6351         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
6352
6353         rc = sbuf_finish(sb);
6354         sbuf_delete(sb);
6355
6356         return (rc);
6357 }
6358
6359 static int
6360 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
6361 {
6362         struct adapter *sc = arg1;
6363         struct sbuf *sb;
6364         int rc;
6365         struct tp_tcp_stats v4, v6;
6366
6367         rc = sysctl_wire_old_buffer(req, 0);
6368         if (rc != 0)
6369                 return (rc);
6370
6371         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6372         if (sb == NULL)
6373                 return (ENOMEM);
6374
6375         t4_tp_get_tcp_stats(sc, &v4, &v6);
6376         sbuf_printf(sb,
6377             "                                IP                 IPv6\n");
6378         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
6379             v4.tcpOutRsts, v6.tcpOutRsts);
6380         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
6381             v4.tcpInSegs, v6.tcpInSegs);
6382         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
6383             v4.tcpOutSegs, v6.tcpOutSegs);
6384         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
6385             v4.tcpRetransSegs, v6.tcpRetransSegs);
6386
6387         rc = sbuf_finish(sb);
6388         sbuf_delete(sb);
6389
6390         return (rc);
6391 }
6392
6393 static int
6394 sysctl_tids(SYSCTL_HANDLER_ARGS)
6395 {
6396         struct adapter *sc = arg1;
6397         struct sbuf *sb;
6398         int rc;
6399         struct tid_info *t = &sc->tids;
6400
6401         rc = sysctl_wire_old_buffer(req, 0);
6402         if (rc != 0)
6403                 return (rc);
6404
6405         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6406         if (sb == NULL)
6407                 return (ENOMEM);
6408
6409         if (t->natids) {
6410                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
6411                     t->atids_in_use);
6412         }
6413
6414         if (t->ntids) {
6415                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6416                         uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
6417
6418                         if (b) {
6419                                 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
6420                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6421                                     t->ntids - 1);
6422                         } else {
6423                                 sbuf_printf(sb, "TID range: %u-%u",
6424                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6425                                     t->ntids - 1);
6426                         }
6427                 } else
6428                         sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
6429                 sbuf_printf(sb, ", in use: %u\n",
6430                     atomic_load_acq_int(&t->tids_in_use));
6431         }
6432
6433         if (t->nstids) {
6434                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
6435                     t->stid_base + t->nstids - 1, t->stids_in_use);
6436         }
6437
6438         if (t->nftids) {
6439                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
6440                     t->ftid_base + t->nftids - 1);
6441         }
6442
6443         if (t->netids) {
6444                 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
6445                     t->etid_base + t->netids - 1);
6446         }
6447
6448         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
6449             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
6450             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
6451
6452         rc = sbuf_finish(sb);
6453         sbuf_delete(sb);
6454
6455         return (rc);
6456 }
6457
6458 static int
6459 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
6460 {
6461         struct adapter *sc = arg1;
6462         struct sbuf *sb;
6463         int rc;
6464         struct tp_err_stats stats;
6465
6466         rc = sysctl_wire_old_buffer(req, 0);
6467         if (rc != 0)
6468                 return (rc);
6469
6470         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6471         if (sb == NULL)
6472                 return (ENOMEM);
6473
6474         t4_tp_get_err_stats(sc, &stats);
6475
6476         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
6477                       "channel 3\n");
6478         sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
6479             stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
6480             stats.macInErrs[3]);
6481         sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
6482             stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
6483             stats.hdrInErrs[3]);
6484         sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
6485             stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
6486             stats.tcpInErrs[3]);
6487         sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
6488             stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
6489             stats.tcp6InErrs[3]);
6490         sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
6491             stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
6492             stats.tnlCongDrops[3]);
6493         sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
6494             stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
6495             stats.tnlTxDrops[3]);
6496         sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
6497             stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
6498             stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
6499         sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
6500             stats.ofldChanDrops[0], stats.ofldChanDrops[1],
6501             stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
6502         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
6503             stats.ofldNoNeigh, stats.ofldCongDefer);
6504
6505         rc = sbuf_finish(sb);
6506         sbuf_delete(sb);
6507
6508         return (rc);
6509 }
6510
6511 struct field_desc {
6512         const char *name;
6513         u_int start;
6514         u_int width;
6515 };
6516
6517 static void
6518 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
6519 {
6520         char buf[32];
6521         int line_size = 0;
6522
6523         while (f->name) {
6524                 uint64_t mask = (1ULL << f->width) - 1;
6525                 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
6526                     ((uintmax_t)v >> f->start) & mask);
6527
6528                 if (line_size + len >= 79) {
6529                         line_size = 8;
6530                         sbuf_printf(sb, "\n        ");
6531                 }
6532                 sbuf_printf(sb, "%s ", buf);
6533                 line_size += len + 1;
6534                 f++;
6535         }
6536         sbuf_printf(sb, "\n");
6537 }
6538
6539 static struct field_desc tp_la0[] = {
6540         { "RcfOpCodeOut", 60, 4 },
6541         { "State", 56, 4 },
6542         { "WcfState", 52, 4 },
6543         { "RcfOpcSrcOut", 50, 2 },
6544         { "CRxError", 49, 1 },
6545         { "ERxError", 48, 1 },
6546         { "SanityFailed", 47, 1 },
6547         { "SpuriousMsg", 46, 1 },
6548         { "FlushInputMsg", 45, 1 },
6549         { "FlushInputCpl", 44, 1 },
6550         { "RssUpBit", 43, 1 },
6551         { "RssFilterHit", 42, 1 },
6552         { "Tid", 32, 10 },
6553         { "InitTcb", 31, 1 },
6554         { "LineNumber", 24, 7 },
6555         { "Emsg", 23, 1 },
6556         { "EdataOut", 22, 1 },
6557         { "Cmsg", 21, 1 },
6558         { "CdataOut", 20, 1 },
6559         { "EreadPdu", 19, 1 },
6560         { "CreadPdu", 18, 1 },
6561         { "TunnelPkt", 17, 1 },
6562         { "RcfPeerFin", 16, 1 },
6563         { "RcfReasonOut", 12, 4 },
6564         { "TxCchannel", 10, 2 },
6565         { "RcfTxChannel", 8, 2 },
6566         { "RxEchannel", 6, 2 },
6567         { "RcfRxChannel", 5, 1 },
6568         { "RcfDataOutSrdy", 4, 1 },
6569         { "RxDvld", 3, 1 },
6570         { "RxOoDvld", 2, 1 },
6571         { "RxCongestion", 1, 1 },
6572         { "TxCongestion", 0, 1 },
6573         { NULL }
6574 };
6575
6576 static struct field_desc tp_la1[] = {
6577         { "CplCmdIn", 56, 8 },
6578         { "CplCmdOut", 48, 8 },
6579         { "ESynOut", 47, 1 },
6580         { "EAckOut", 46, 1 },
6581         { "EFinOut", 45, 1 },
6582         { "ERstOut", 44, 1 },
6583         { "SynIn", 43, 1 },
6584         { "AckIn", 42, 1 },
6585         { "FinIn", 41, 1 },
6586         { "RstIn", 40, 1 },
6587         { "DataIn", 39, 1 },
6588         { "DataInVld", 38, 1 },
6589         { "PadIn", 37, 1 },
6590         { "RxBufEmpty", 36, 1 },
6591         { "RxDdp", 35, 1 },
6592         { "RxFbCongestion", 34, 1 },
6593         { "TxFbCongestion", 33, 1 },
6594         { "TxPktSumSrdy", 32, 1 },
6595         { "RcfUlpType", 28, 4 },
6596         { "Eread", 27, 1 },
6597         { "Ebypass", 26, 1 },
6598         { "Esave", 25, 1 },
6599         { "Static0", 24, 1 },
6600         { "Cread", 23, 1 },
6601         { "Cbypass", 22, 1 },
6602         { "Csave", 21, 1 },
6603         { "CPktOut", 20, 1 },
6604         { "RxPagePoolFull", 18, 2 },
6605         { "RxLpbkPkt", 17, 1 },
6606         { "TxLpbkPkt", 16, 1 },
6607         { "RxVfValid", 15, 1 },
6608         { "SynLearned", 14, 1 },
6609         { "SetDelEntry", 13, 1 },
6610         { "SetInvEntry", 12, 1 },
6611         { "CpcmdDvld", 11, 1 },
6612         { "CpcmdSave", 10, 1 },
6613         { "RxPstructsFull", 8, 2 },
6614         { "EpcmdDvld", 7, 1 },
6615         { "EpcmdFlush", 6, 1 },
6616         { "EpcmdTrimPrefix", 5, 1 },
6617         { "EpcmdTrimPostfix", 4, 1 },
6618         { "ERssIp4Pkt", 3, 1 },
6619         { "ERssIp6Pkt", 2, 1 },
6620         { "ERssTcpUdpPkt", 1, 1 },
6621         { "ERssFceFipPkt", 0, 1 },
6622         { NULL }
6623 };
6624
6625 static struct field_desc tp_la2[] = {
6626         { "CplCmdIn", 56, 8 },
6627         { "MpsVfVld", 55, 1 },
6628         { "MpsPf", 52, 3 },
6629         { "MpsVf", 44, 8 },
6630         { "SynIn", 43, 1 },
6631         { "AckIn", 42, 1 },
6632         { "FinIn", 41, 1 },
6633         { "RstIn", 40, 1 },
6634         { "DataIn", 39, 1 },
6635         { "DataInVld", 38, 1 },
6636         { "PadIn", 37, 1 },
6637         { "RxBufEmpty", 36, 1 },
6638         { "RxDdp", 35, 1 },
6639         { "RxFbCongestion", 34, 1 },
6640         { "TxFbCongestion", 33, 1 },
6641         { "TxPktSumSrdy", 32, 1 },
6642         { "RcfUlpType", 28, 4 },
6643         { "Eread", 27, 1 },
6644         { "Ebypass", 26, 1 },
6645         { "Esave", 25, 1 },
6646         { "Static0", 24, 1 },
6647         { "Cread", 23, 1 },
6648         { "Cbypass", 22, 1 },
6649         { "Csave", 21, 1 },
6650         { "CPktOut", 20, 1 },
6651         { "RxPagePoolFull", 18, 2 },
6652         { "RxLpbkPkt", 17, 1 },
6653         { "TxLpbkPkt", 16, 1 },
6654         { "RxVfValid", 15, 1 },
6655         { "SynLearned", 14, 1 },
6656         { "SetDelEntry", 13, 1 },
6657         { "SetInvEntry", 12, 1 },
6658         { "CpcmdDvld", 11, 1 },
6659         { "CpcmdSave", 10, 1 },
6660         { "RxPstructsFull", 8, 2 },
6661         { "EpcmdDvld", 7, 1 },
6662         { "EpcmdFlush", 6, 1 },
6663         { "EpcmdTrimPrefix", 5, 1 },
6664         { "EpcmdTrimPostfix", 4, 1 },
6665         { "ERssIp4Pkt", 3, 1 },
6666         { "ERssIp6Pkt", 2, 1 },
6667         { "ERssTcpUdpPkt", 1, 1 },
6668         { "ERssFceFipPkt", 0, 1 },
6669         { NULL }
6670 };
6671
6672 static void
6673 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
6674 {
6675
6676         field_desc_show(sb, *p, tp_la0);
6677 }
6678
6679 static void
6680 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
6681 {
6682
6683         if (idx)
6684                 sbuf_printf(sb, "\n");
6685         field_desc_show(sb, p[0], tp_la0);
6686         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6687                 field_desc_show(sb, p[1], tp_la0);
6688 }
6689
6690 static void
6691 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
6692 {
6693
6694         if (idx)
6695                 sbuf_printf(sb, "\n");
6696         field_desc_show(sb, p[0], tp_la0);
6697         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6698                 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
6699 }
6700
6701 static int
6702 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
6703 {
6704         struct adapter *sc = arg1;
6705         struct sbuf *sb;
6706         uint64_t *buf, *p;
6707         int rc;
6708         u_int i, inc;
6709         void (*show_func)(struct sbuf *, uint64_t *, int);
6710
6711         rc = sysctl_wire_old_buffer(req, 0);
6712         if (rc != 0)
6713                 return (rc);
6714
6715         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6716         if (sb == NULL)
6717                 return (ENOMEM);
6718
6719         buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
6720
6721         t4_tp_read_la(sc, buf, NULL);
6722         p = buf;
6723
6724         switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
6725         case 2:
6726                 inc = 2;
6727                 show_func = tp_la_show2;
6728                 break;
6729         case 3:
6730                 inc = 2;
6731                 show_func = tp_la_show3;
6732                 break;
6733         default:
6734                 inc = 1;
6735                 show_func = tp_la_show;
6736         }
6737
6738         for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
6739                 (*show_func)(sb, p, i);
6740
6741         rc = sbuf_finish(sb);
6742         sbuf_delete(sb);
6743         free(buf, M_CXGBE);
6744         return (rc);
6745 }
6746
6747 static int
6748 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
6749 {
6750         struct adapter *sc = arg1;
6751         struct sbuf *sb;
6752         int rc;
6753         u64 nrate[NCHAN], orate[NCHAN];
6754
6755         rc = sysctl_wire_old_buffer(req, 0);
6756         if (rc != 0)
6757                 return (rc);
6758
6759         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6760         if (sb == NULL)
6761                 return (ENOMEM);
6762
6763         t4_get_chan_txrate(sc, nrate, orate);
6764         sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
6765                  "channel 3\n");
6766         sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
6767             nrate[0], nrate[1], nrate[2], nrate[3]);
6768         sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
6769             orate[0], orate[1], orate[2], orate[3]);
6770
6771         rc = sbuf_finish(sb);
6772         sbuf_delete(sb);
6773
6774         return (rc);
6775 }
6776
6777 static int
6778 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
6779 {
6780         struct adapter *sc = arg1;
6781         struct sbuf *sb;
6782         uint32_t *buf, *p;
6783         int rc, i;
6784
6785         rc = sysctl_wire_old_buffer(req, 0);
6786         if (rc != 0)
6787                 return (rc);
6788
6789         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6790         if (sb == NULL)
6791                 return (ENOMEM);
6792
6793         buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
6794             M_ZERO | M_WAITOK);
6795
6796         t4_ulprx_read_la(sc, buf);
6797         p = buf;
6798
6799         sbuf_printf(sb, "      Pcmd        Type   Message"
6800             "                Data");
6801         for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
6802                 sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
6803                     p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
6804         }
6805
6806         rc = sbuf_finish(sb);
6807         sbuf_delete(sb);
6808         free(buf, M_CXGBE);
6809         return (rc);
6810 }
6811
6812 static int
6813 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
6814 {
6815         struct adapter *sc = arg1;
6816         struct sbuf *sb;
6817         int rc, v;
6818
6819         rc = sysctl_wire_old_buffer(req, 0);
6820         if (rc != 0)
6821                 return (rc);
6822
6823         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6824         if (sb == NULL)
6825                 return (ENOMEM);
6826
6827         v = t4_read_reg(sc, A_SGE_STAT_CFG);
6828         if (G_STATSOURCE_T5(v) == 7) {
6829                 if (G_STATMODE(v) == 0) {
6830                         sbuf_printf(sb, "total %d, incomplete %d",
6831                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6832                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6833                 } else if (G_STATMODE(v) == 1) {
6834                         sbuf_printf(sb, "total %d, data overflow %d",
6835                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6836                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6837                 }
6838         }
6839         rc = sbuf_finish(sb);
6840         sbuf_delete(sb);
6841
6842         return (rc);
6843 }
6844 #endif
6845
6846 static inline void
6847 txq_start(struct ifnet *ifp, struct sge_txq *txq)
6848 {
6849         struct buf_ring *br;
6850         struct mbuf *m;
6851
6852         TXQ_LOCK_ASSERT_OWNED(txq);
6853
6854         br = txq->br;
6855         m = txq->m ? txq->m : drbr_dequeue(ifp, br);
6856         if (m)
6857                 t4_eth_tx(ifp, txq, m);
6858 }
6859
6860 void
6861 t4_tx_callout(void *arg)
6862 {
6863         struct sge_eq *eq = arg;
6864         struct adapter *sc;
6865
6866         if (EQ_TRYLOCK(eq) == 0)
6867                 goto reschedule;
6868
6869         if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) {
6870                 EQ_UNLOCK(eq);
6871 reschedule:
6872                 if (__predict_true(!(eq->flags && EQ_DOOMED)))
6873                         callout_schedule(&eq->tx_callout, 1);
6874                 return;
6875         }
6876
6877         EQ_LOCK_ASSERT_OWNED(eq);
6878
6879         if (__predict_true((eq->flags & EQ_DOOMED) == 0)) {
6880
6881                 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
6882                         struct sge_txq *txq = arg;
6883                         struct port_info *pi = txq->ifp->if_softc;
6884
6885                         sc = pi->adapter;
6886                 } else {
6887                         struct sge_wrq *wrq = arg;
6888
6889                         sc = wrq->adapter;
6890                 }
6891
6892                 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task);
6893         }
6894
6895         EQ_UNLOCK(eq);
6896 }
6897
6898 void
6899 t4_tx_task(void *arg, int count)
6900 {
6901         struct sge_eq *eq = arg;
6902
6903         EQ_LOCK(eq);
6904         if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
6905                 struct sge_txq *txq = arg;
6906                 txq_start(txq->ifp, txq);
6907         } else {
6908                 struct sge_wrq *wrq = arg;
6909                 t4_wrq_tx_locked(wrq->adapter, wrq, NULL);
6910         }
6911         EQ_UNLOCK(eq);
6912 }
6913
6914 static uint32_t
6915 fconf_to_mode(uint32_t fconf)
6916 {
6917         uint32_t mode;
6918
6919         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
6920             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
6921
6922         if (fconf & F_FRAGMENTATION)
6923                 mode |= T4_FILTER_IP_FRAGMENT;
6924
6925         if (fconf & F_MPSHITTYPE)
6926                 mode |= T4_FILTER_MPS_HIT_TYPE;
6927
6928         if (fconf & F_MACMATCH)
6929                 mode |= T4_FILTER_MAC_IDX;
6930
6931         if (fconf & F_ETHERTYPE)
6932                 mode |= T4_FILTER_ETH_TYPE;
6933
6934         if (fconf & F_PROTOCOL)
6935                 mode |= T4_FILTER_IP_PROTO;
6936
6937         if (fconf & F_TOS)
6938                 mode |= T4_FILTER_IP_TOS;
6939
6940         if (fconf & F_VLAN)
6941                 mode |= T4_FILTER_VLAN;
6942
6943         if (fconf & F_VNIC_ID)
6944                 mode |= T4_FILTER_VNIC;
6945
6946         if (fconf & F_PORT)
6947                 mode |= T4_FILTER_PORT;
6948
6949         if (fconf & F_FCOE)
6950                 mode |= T4_FILTER_FCoE;
6951
6952         return (mode);
6953 }
6954
6955 static uint32_t
6956 mode_to_fconf(uint32_t mode)
6957 {
6958         uint32_t fconf = 0;
6959
6960         if (mode & T4_FILTER_IP_FRAGMENT)
6961                 fconf |= F_FRAGMENTATION;
6962
6963         if (mode & T4_FILTER_MPS_HIT_TYPE)
6964                 fconf |= F_MPSHITTYPE;
6965
6966         if (mode & T4_FILTER_MAC_IDX)
6967                 fconf |= F_MACMATCH;
6968
6969         if (mode & T4_FILTER_ETH_TYPE)
6970                 fconf |= F_ETHERTYPE;
6971
6972         if (mode & T4_FILTER_IP_PROTO)
6973                 fconf |= F_PROTOCOL;
6974
6975         if (mode & T4_FILTER_IP_TOS)
6976                 fconf |= F_TOS;
6977
6978         if (mode & T4_FILTER_VLAN)
6979                 fconf |= F_VLAN;
6980
6981         if (mode & T4_FILTER_VNIC)
6982                 fconf |= F_VNIC_ID;
6983
6984         if (mode & T4_FILTER_PORT)
6985                 fconf |= F_PORT;
6986
6987         if (mode & T4_FILTER_FCoE)
6988                 fconf |= F_FCOE;
6989
6990         return (fconf);
6991 }
6992
6993 static uint32_t
6994 fspec_to_fconf(struct t4_filter_specification *fs)
6995 {
6996         uint32_t fconf = 0;
6997
6998         if (fs->val.frag || fs->mask.frag)
6999                 fconf |= F_FRAGMENTATION;
7000
7001         if (fs->val.matchtype || fs->mask.matchtype)
7002                 fconf |= F_MPSHITTYPE;
7003
7004         if (fs->val.macidx || fs->mask.macidx)
7005                 fconf |= F_MACMATCH;
7006
7007         if (fs->val.ethtype || fs->mask.ethtype)
7008                 fconf |= F_ETHERTYPE;
7009
7010         if (fs->val.proto || fs->mask.proto)
7011                 fconf |= F_PROTOCOL;
7012
7013         if (fs->val.tos || fs->mask.tos)
7014                 fconf |= F_TOS;
7015
7016         if (fs->val.vlan_vld || fs->mask.vlan_vld)
7017                 fconf |= F_VLAN;
7018
7019         if (fs->val.vnic_vld || fs->mask.vnic_vld)
7020                 fconf |= F_VNIC_ID;
7021
7022         if (fs->val.iport || fs->mask.iport)
7023                 fconf |= F_PORT;
7024
7025         if (fs->val.fcoe || fs->mask.fcoe)
7026                 fconf |= F_FCOE;
7027
7028         return (fconf);
7029 }
7030
7031 static int
7032 get_filter_mode(struct adapter *sc, uint32_t *mode)
7033 {
7034         int rc;
7035         uint32_t fconf;
7036
7037         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7038             "t4getfm");
7039         if (rc)
7040                 return (rc);
7041
7042         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
7043             A_TP_VLAN_PRI_MAP);
7044
7045         if (sc->params.tp.vlan_pri_map != fconf) {
7046                 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
7047                     device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
7048                     fconf);
7049                 sc->params.tp.vlan_pri_map = fconf;
7050         }
7051
7052         *mode = fconf_to_mode(sc->params.tp.vlan_pri_map);
7053
7054         end_synchronized_op(sc, LOCK_HELD);
7055         return (0);
7056 }
7057
7058 static int
7059 set_filter_mode(struct adapter *sc, uint32_t mode)
7060 {
7061         uint32_t fconf;
7062         int rc;
7063
7064         fconf = mode_to_fconf(mode);
7065
7066         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7067             "t4setfm");
7068         if (rc)
7069                 return (rc);
7070
7071         if (sc->tids.ftids_in_use > 0) {
7072                 rc = EBUSY;
7073                 goto done;
7074         }
7075
7076 #ifdef TCP_OFFLOAD
7077         if (sc->offload_map) {
7078                 rc = EBUSY;
7079                 goto done;
7080         }
7081 #endif
7082
7083 #ifdef notyet
7084         rc = -t4_set_filter_mode(sc, fconf);
7085         if (rc == 0)
7086                 sc->filter_mode = fconf;
7087 #else
7088         rc = ENOTSUP;
7089 #endif
7090
7091 done:
7092         end_synchronized_op(sc, LOCK_HELD);
7093         return (rc);
7094 }
7095
7096 static inline uint64_t
7097 get_filter_hits(struct adapter *sc, uint32_t fid)
7098 {
7099         uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
7100         uint64_t hits;
7101
7102         memwin_info(sc, 0, &mw_base, NULL);
7103         off = position_memwin(sc, 0,
7104             tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
7105         if (is_t4(sc)) {
7106                 hits = t4_read_reg64(sc, mw_base + off + 16);
7107                 hits = be64toh(hits);
7108         } else {
7109                 hits = t4_read_reg(sc, mw_base + off + 24);
7110                 hits = be32toh(hits);
7111         }
7112
7113         return (hits);
7114 }
7115
7116 static int
7117 get_filter(struct adapter *sc, struct t4_filter *t)
7118 {
7119         int i, rc, nfilters = sc->tids.nftids;
7120         struct filter_entry *f;
7121
7122         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7123             "t4getf");
7124         if (rc)
7125                 return (rc);
7126
7127         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
7128             t->idx >= nfilters) {
7129                 t->idx = 0xffffffff;
7130                 goto done;
7131         }
7132
7133         f = &sc->tids.ftid_tab[t->idx];
7134         for (i = t->idx; i < nfilters; i++, f++) {
7135                 if (f->valid) {
7136                         t->idx = i;
7137                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
7138                         t->smtidx = f->smtidx;
7139                         if (f->fs.hitcnts)
7140                                 t->hits = get_filter_hits(sc, t->idx);
7141                         else
7142                                 t->hits = UINT64_MAX;
7143                         t->fs = f->fs;
7144
7145                         goto done;
7146                 }
7147         }
7148
7149         t->idx = 0xffffffff;
7150 done:
7151         end_synchronized_op(sc, LOCK_HELD);
7152         return (0);
7153 }
7154
7155 static int
7156 set_filter(struct adapter *sc, struct t4_filter *t)
7157 {
7158         unsigned int nfilters, nports;
7159         struct filter_entry *f;
7160         int i, rc;
7161
7162         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
7163         if (rc)
7164                 return (rc);
7165
7166         nfilters = sc->tids.nftids;
7167         nports = sc->params.nports;
7168
7169         if (nfilters == 0) {
7170                 rc = ENOTSUP;
7171                 goto done;
7172         }
7173
7174         if (!(sc->flags & FULL_INIT_DONE)) {
7175                 rc = EAGAIN;
7176                 goto done;
7177         }
7178
7179         if (t->idx >= nfilters) {
7180                 rc = EINVAL;
7181                 goto done;
7182         }
7183
7184         /* Validate against the global filter mode */
7185         if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
7186             sc->params.tp.vlan_pri_map) {
7187                 rc = E2BIG;
7188                 goto done;
7189         }
7190
7191         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
7192                 rc = EINVAL;
7193                 goto done;
7194         }
7195
7196         if (t->fs.val.iport >= nports) {
7197                 rc = EINVAL;
7198                 goto done;
7199         }
7200
7201         /* Can't specify an iq if not steering to it */
7202         if (!t->fs.dirsteer && t->fs.iq) {
7203                 rc = EINVAL;
7204                 goto done;
7205         }
7206
7207         /* IPv6 filter idx must be 4 aligned */
7208         if (t->fs.type == 1 &&
7209             ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
7210                 rc = EINVAL;
7211                 goto done;
7212         }
7213
7214         if (sc->tids.ftid_tab == NULL) {
7215                 KASSERT(sc->tids.ftids_in_use == 0,
7216                     ("%s: no memory allocated but filters_in_use > 0",
7217                     __func__));
7218
7219                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
7220                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
7221                 if (sc->tids.ftid_tab == NULL) {
7222                         rc = ENOMEM;
7223                         goto done;
7224                 }
7225                 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
7226         }
7227
7228         for (i = 0; i < 4; i++) {
7229                 f = &sc->tids.ftid_tab[t->idx + i];
7230
7231                 if (f->pending || f->valid) {
7232                         rc = EBUSY;
7233                         goto done;
7234                 }
7235                 if (f->locked) {
7236                         rc = EPERM;
7237                         goto done;
7238                 }
7239
7240                 if (t->fs.type == 0)
7241                         break;
7242         }
7243
7244         f = &sc->tids.ftid_tab[t->idx];
7245         f->fs = t->fs;
7246
7247         rc = set_filter_wr(sc, t->idx);
7248 done:
7249         end_synchronized_op(sc, 0);
7250
7251         if (rc == 0) {
7252                 mtx_lock(&sc->tids.ftid_lock);
7253                 for (;;) {
7254                         if (f->pending == 0) {
7255                                 rc = f->valid ? 0 : EIO;
7256                                 break;
7257                         }
7258
7259                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7260                             PCATCH, "t4setfw", 0)) {
7261                                 rc = EINPROGRESS;
7262                                 break;
7263                         }
7264                 }
7265                 mtx_unlock(&sc->tids.ftid_lock);
7266         }
7267         return (rc);
7268 }
7269
7270 static int
7271 del_filter(struct adapter *sc, struct t4_filter *t)
7272 {
7273         unsigned int nfilters;
7274         struct filter_entry *f;
7275         int rc;
7276
7277         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
7278         if (rc)
7279                 return (rc);
7280
7281         nfilters = sc->tids.nftids;
7282
7283         if (nfilters == 0) {
7284                 rc = ENOTSUP;
7285                 goto done;
7286         }
7287
7288         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
7289             t->idx >= nfilters) {
7290                 rc = EINVAL;
7291                 goto done;
7292         }
7293
7294         if (!(sc->flags & FULL_INIT_DONE)) {
7295                 rc = EAGAIN;
7296                 goto done;
7297         }
7298
7299         f = &sc->tids.ftid_tab[t->idx];
7300
7301         if (f->pending) {
7302                 rc = EBUSY;
7303                 goto done;
7304         }
7305         if (f->locked) {
7306                 rc = EPERM;
7307                 goto done;
7308         }
7309
7310         if (f->valid) {
7311                 t->fs = f->fs;  /* extra info for the caller */
7312                 rc = del_filter_wr(sc, t->idx);
7313         }
7314
7315 done:
7316         end_synchronized_op(sc, 0);
7317
7318         if (rc == 0) {
7319                 mtx_lock(&sc->tids.ftid_lock);
7320                 for (;;) {
7321                         if (f->pending == 0) {
7322                                 rc = f->valid ? EIO : 0;
7323                                 break;
7324                         }
7325
7326                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7327                             PCATCH, "t4delfw", 0)) {
7328                                 rc = EINPROGRESS;
7329                                 break;
7330                         }
7331                 }
7332                 mtx_unlock(&sc->tids.ftid_lock);
7333         }
7334
7335         return (rc);
7336 }
7337
7338 static void
7339 clear_filter(struct filter_entry *f)
7340 {
7341         if (f->l2t)
7342                 t4_l2t_release(f->l2t);
7343
7344         bzero(f, sizeof (*f));
7345 }
7346
7347 static int
7348 set_filter_wr(struct adapter *sc, int fidx)
7349 {
7350         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7351         struct wrqe *wr;
7352         struct fw_filter_wr *fwr;
7353         unsigned int ftid;
7354
7355         ASSERT_SYNCHRONIZED_OP(sc);
7356
7357         if (f->fs.newdmac || f->fs.newvlan) {
7358                 /* This filter needs an L2T entry; allocate one. */
7359                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
7360                 if (f->l2t == NULL)
7361                         return (EAGAIN);
7362                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
7363                     f->fs.dmac)) {
7364                         t4_l2t_release(f->l2t);
7365                         f->l2t = NULL;
7366                         return (ENOMEM);
7367                 }
7368         }
7369
7370         ftid = sc->tids.ftid_base + fidx;
7371
7372         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7373         if (wr == NULL)
7374                 return (ENOMEM);
7375
7376         fwr = wrtod(wr);
7377         bzero(fwr, sizeof (*fwr));
7378
7379         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
7380         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
7381         fwr->tid_to_iq =
7382             htobe32(V_FW_FILTER_WR_TID(ftid) |
7383                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
7384                 V_FW_FILTER_WR_NOREPLY(0) |
7385                 V_FW_FILTER_WR_IQ(f->fs.iq));
7386         fwr->del_filter_to_l2tix =
7387             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
7388                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
7389                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
7390                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
7391                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
7392                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
7393                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
7394                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
7395                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
7396                     f->fs.newvlan == VLAN_REWRITE) |
7397                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
7398                     f->fs.newvlan == VLAN_REWRITE) |
7399                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
7400                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
7401                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
7402                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
7403         fwr->ethtype = htobe16(f->fs.val.ethtype);
7404         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
7405         fwr->frag_to_ovlan_vldm =
7406             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
7407                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
7408                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
7409                 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
7410                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
7411                 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
7412         fwr->smac_sel = 0;
7413         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
7414             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
7415         fwr->maci_to_matchtypem =
7416             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
7417                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
7418                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
7419                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
7420                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
7421                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
7422                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
7423                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
7424         fwr->ptcl = f->fs.val.proto;
7425         fwr->ptclm = f->fs.mask.proto;
7426         fwr->ttyp = f->fs.val.tos;
7427         fwr->ttypm = f->fs.mask.tos;
7428         fwr->ivlan = htobe16(f->fs.val.vlan);
7429         fwr->ivlanm = htobe16(f->fs.mask.vlan);
7430         fwr->ovlan = htobe16(f->fs.val.vnic);
7431         fwr->ovlanm = htobe16(f->fs.mask.vnic);
7432         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
7433         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
7434         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
7435         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
7436         fwr->lp = htobe16(f->fs.val.dport);
7437         fwr->lpm = htobe16(f->fs.mask.dport);
7438         fwr->fp = htobe16(f->fs.val.sport);
7439         fwr->fpm = htobe16(f->fs.mask.sport);
7440         if (f->fs.newsmac)
7441                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
7442
7443         f->pending = 1;
7444         sc->tids.ftids_in_use++;
7445
7446         t4_wrq_tx(sc, wr);
7447         return (0);
7448 }
7449
7450 static int
7451 del_filter_wr(struct adapter *sc, int fidx)
7452 {
7453         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7454         struct wrqe *wr;
7455         struct fw_filter_wr *fwr;
7456         unsigned int ftid;
7457
7458         ftid = sc->tids.ftid_base + fidx;
7459
7460         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7461         if (wr == NULL)
7462                 return (ENOMEM);
7463         fwr = wrtod(wr);
7464         bzero(fwr, sizeof (*fwr));
7465
7466         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
7467
7468         f->pending = 1;
7469         t4_wrq_tx(sc, wr);
7470         return (0);
7471 }
7472
7473 int
7474 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
7475 {
7476         struct adapter *sc = iq->adapter;
7477         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
7478         unsigned int idx = GET_TID(rpl);
7479         unsigned int rc;
7480         struct filter_entry *f;
7481
7482         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
7483             rss->opcode));
7484
7485         if (is_ftid(sc, idx)) {
7486
7487                 idx -= sc->tids.ftid_base;
7488                 f = &sc->tids.ftid_tab[idx];
7489                 rc = G_COOKIE(rpl->cookie);
7490
7491                 mtx_lock(&sc->tids.ftid_lock);
7492                 if (rc == FW_FILTER_WR_FLT_ADDED) {
7493                         KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
7494                             __func__, idx));
7495                         f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
7496                         f->pending = 0;  /* asynchronous setup completed */
7497                         f->valid = 1;
7498                 } else {
7499                         if (rc != FW_FILTER_WR_FLT_DELETED) {
7500                                 /* Add or delete failed, display an error */
7501                                 log(LOG_ERR,
7502                                     "filter %u setup failed with error %u\n",
7503                                     idx, rc);
7504                         }
7505
7506                         clear_filter(f);
7507                         sc->tids.ftids_in_use--;
7508                 }
7509                 wakeup(&sc->tids.ftid_tab);
7510                 mtx_unlock(&sc->tids.ftid_lock);
7511         }
7512
7513         return (0);
7514 }
7515
7516 static int
7517 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
7518 {
7519         int rc;
7520
7521         if (cntxt->cid > M_CTXTQID)
7522                 return (EINVAL);
7523
7524         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
7525             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
7526                 return (EINVAL);
7527
7528         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
7529         if (rc)
7530                 return (rc);
7531
7532         if (sc->flags & FW_OK) {
7533                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
7534                     &cntxt->data[0]);
7535                 if (rc == 0)
7536                         goto done;
7537         }
7538
7539         /*
7540          * Read via firmware failed or wasn't even attempted.  Read directly via
7541          * the backdoor.
7542          */
7543         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
7544 done:
7545         end_synchronized_op(sc, 0);
7546         return (rc);
7547 }
7548
7549 static int
7550 load_fw(struct adapter *sc, struct t4_data *fw)
7551 {
7552         int rc;
7553         uint8_t *fw_data;
7554
7555         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
7556         if (rc)
7557                 return (rc);
7558
7559         if (sc->flags & FULL_INIT_DONE) {
7560                 rc = EBUSY;
7561                 goto done;
7562         }
7563
7564         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
7565         if (fw_data == NULL) {
7566                 rc = ENOMEM;
7567                 goto done;
7568         }
7569
7570         rc = copyin(fw->data, fw_data, fw->len);
7571         if (rc == 0)
7572                 rc = -t4_load_fw(sc, fw_data, fw->len);
7573
7574         free(fw_data, M_CXGBE);
7575 done:
7576         end_synchronized_op(sc, 0);
7577         return (rc);
7578 }
7579
7580 static int
7581 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
7582 {
7583         uint32_t addr, off, remaining, i, n;
7584         uint32_t *buf, *b;
7585         uint32_t mw_base, mw_aperture;
7586         int rc;
7587         uint8_t *dst;
7588
7589         rc = validate_mem_range(sc, mr->addr, mr->len);
7590         if (rc != 0)
7591                 return (rc);
7592
7593         memwin_info(sc, win, &mw_base, &mw_aperture);
7594         buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
7595         addr = mr->addr;
7596         remaining = mr->len;
7597         dst = (void *)mr->data;
7598
7599         while (remaining) {
7600                 off = position_memwin(sc, win, addr);
7601
7602                 /* number of bytes that we'll copy in the inner loop */
7603                 n = min(remaining, mw_aperture - off);
7604                 for (i = 0; i < n; i += 4)
7605                         *b++ = t4_read_reg(sc, mw_base + off + i);
7606
7607                 rc = copyout(buf, dst, n);
7608                 if (rc != 0)
7609                         break;
7610
7611                 b = buf;
7612                 dst += n;
7613                 remaining -= n;
7614                 addr += n;
7615         }
7616
7617         free(buf, M_CXGBE);
7618         return (rc);
7619 }
7620
7621 static int
7622 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
7623 {
7624         int rc;
7625
7626         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
7627                 return (EINVAL);
7628
7629         if (i2cd->len > sizeof(i2cd->data))
7630                 return (EFBIG);
7631
7632         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
7633         if (rc)
7634                 return (rc);
7635         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
7636             i2cd->offset, i2cd->len, &i2cd->data[0]);
7637         end_synchronized_op(sc, 0);
7638
7639         return (rc);
7640 }
7641
7642 static int
7643 in_range(int val, int lo, int hi)
7644 {
7645
7646         return (val < 0 || (val <= hi && val >= lo));
7647 }
7648
7649 static int
7650 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
7651 {
7652         int fw_subcmd, fw_type, rc;
7653
7654         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
7655         if (rc)
7656                 return (rc);
7657
7658         if (!(sc->flags & FULL_INIT_DONE)) {
7659                 rc = EAGAIN;
7660                 goto done;
7661         }
7662
7663         /*
7664          * Translate the cxgbetool parameters into T4 firmware parameters.  (The
7665          * sub-command and type are in common locations.)
7666          */
7667         if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
7668                 fw_subcmd = FW_SCHED_SC_CONFIG;
7669         else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
7670                 fw_subcmd = FW_SCHED_SC_PARAMS;
7671         else {
7672                 rc = EINVAL;
7673                 goto done;
7674         }
7675         if (p->type == SCHED_CLASS_TYPE_PACKET)
7676                 fw_type = FW_SCHED_TYPE_PKTSCHED;
7677         else {
7678                 rc = EINVAL;
7679                 goto done;
7680         }
7681
7682         if (fw_subcmd == FW_SCHED_SC_CONFIG) {
7683                 /* Vet our parameters ..*/
7684                 if (p->u.config.minmax < 0) {
7685                         rc = EINVAL;
7686                         goto done;
7687                 }
7688
7689                 /* And pass the request to the firmware ...*/
7690                 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1);
7691                 goto done;
7692         }
7693
7694         if (fw_subcmd == FW_SCHED_SC_PARAMS) {
7695                 int fw_level;
7696                 int fw_mode;
7697                 int fw_rateunit;
7698                 int fw_ratemode;
7699
7700                 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
7701                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
7702                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
7703                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
7704                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
7705                         fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
7706                 else {
7707                         rc = EINVAL;
7708                         goto done;
7709                 }
7710
7711                 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
7712                         fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
7713                 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
7714                         fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
7715                 else {
7716                         rc = EINVAL;
7717                         goto done;
7718                 }
7719
7720                 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
7721                         fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
7722                 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
7723                         fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
7724                 else {
7725                         rc = EINVAL;
7726                         goto done;
7727                 }
7728
7729                 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
7730                         fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
7731                 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
7732                         fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
7733                 else {
7734                         rc = EINVAL;
7735                         goto done;
7736                 }
7737
7738                 /* Vet our parameters ... */
7739                 if (!in_range(p->u.params.channel, 0, 3) ||
7740                     !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
7741                     !in_range(p->u.params.minrate, 0, 10000000) ||
7742                     !in_range(p->u.params.maxrate, 0, 10000000) ||
7743                     !in_range(p->u.params.weight, 0, 100)) {
7744                         rc = ERANGE;
7745                         goto done;
7746                 }
7747
7748                 /*
7749                  * Translate any unset parameters into the firmware's
7750                  * nomenclature and/or fail the call if the parameters
7751                  * are required ...
7752                  */
7753                 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
7754                     p->u.params.channel < 0 || p->u.params.cl < 0) {
7755                         rc = EINVAL;
7756                         goto done;
7757                 }
7758                 if (p->u.params.minrate < 0)
7759                         p->u.params.minrate = 0;
7760                 if (p->u.params.maxrate < 0) {
7761                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7762                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7763                                 rc = EINVAL;
7764                                 goto done;
7765                         } else
7766                                 p->u.params.maxrate = 0;
7767                 }
7768                 if (p->u.params.weight < 0) {
7769                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
7770                                 rc = EINVAL;
7771                                 goto done;
7772                         } else
7773                                 p->u.params.weight = 0;
7774                 }
7775                 if (p->u.params.pktsize < 0) {
7776                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7777                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7778                                 rc = EINVAL;
7779                                 goto done;
7780                         } else
7781                                 p->u.params.pktsize = 0;
7782                 }
7783
7784                 /* See what the firmware thinks of the request ... */
7785                 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
7786                     fw_rateunit, fw_ratemode, p->u.params.channel,
7787                     p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
7788                     p->u.params.weight, p->u.params.pktsize, 1);
7789                 goto done;
7790         }
7791
7792         rc = EINVAL;
7793 done:
7794         end_synchronized_op(sc, 0);
7795         return (rc);
7796 }
7797
7798 static int
7799 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
7800 {
7801         struct port_info *pi = NULL;
7802         struct sge_txq *txq;
7803         uint32_t fw_mnem, fw_queue, fw_class;
7804         int i, rc;
7805
7806         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
7807         if (rc)
7808                 return (rc);
7809
7810         if (!(sc->flags & FULL_INIT_DONE)) {
7811                 rc = EAGAIN;
7812                 goto done;
7813         }
7814
7815         if (p->port >= sc->params.nports) {
7816                 rc = EINVAL;
7817                 goto done;
7818         }
7819
7820         pi = sc->port[p->port];
7821         if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
7822                 rc = EINVAL;
7823                 goto done;
7824         }
7825
7826         /*
7827          * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
7828          * Scheduling Class in this case).
7829          */
7830         fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
7831             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
7832         fw_class = p->cl < 0 ? 0xffffffff : p->cl;
7833
7834         /*
7835          * If op.queue is non-negative, then we're only changing the scheduling
7836          * on a single specified TX queue.
7837          */
7838         if (p->queue >= 0) {
7839                 txq = &sc->sge.txq[pi->first_txq + p->queue];
7840                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7841                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7842                     &fw_class);
7843                 goto done;
7844         }
7845
7846         /*
7847          * Change the scheduling on all the TX queues for the
7848          * interface.
7849          */
7850         for_each_txq(pi, i, txq) {
7851                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7852                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7853                     &fw_class);
7854                 if (rc)
7855                         goto done;
7856         }
7857
7858         rc = 0;
7859 done:
7860         end_synchronized_op(sc, 0);
7861         return (rc);
7862 }
7863
7864 int
7865 t4_os_find_pci_capability(struct adapter *sc, int cap)
7866 {
7867         int i;
7868
7869         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
7870 }
7871
7872 int
7873 t4_os_pci_save_state(struct adapter *sc)
7874 {
7875         device_t dev;
7876         struct pci_devinfo *dinfo;
7877
7878         dev = sc->dev;
7879         dinfo = device_get_ivars(dev);
7880
7881         pci_cfg_save(dev, dinfo, 0);
7882         return (0);
7883 }
7884
7885 int
7886 t4_os_pci_restore_state(struct adapter *sc)
7887 {
7888         device_t dev;
7889         struct pci_devinfo *dinfo;
7890
7891         dev = sc->dev;
7892         dinfo = device_get_ivars(dev);
7893
7894         pci_cfg_restore(dev, dinfo);
7895         return (0);
7896 }
7897
7898 void
7899 t4_os_portmod_changed(const struct adapter *sc, int idx)
7900 {
7901         struct port_info *pi = sc->port[idx];
7902         static const char *mod_str[] = {
7903                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
7904         };
7905
7906         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
7907                 if_printf(pi->ifp, "transceiver unplugged.\n");
7908         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
7909                 if_printf(pi->ifp, "unknown transceiver inserted.\n");
7910         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
7911                 if_printf(pi->ifp, "unsupported transceiver inserted.\n");
7912         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
7913                 if_printf(pi->ifp, "%s transceiver inserted.\n",
7914                     mod_str[pi->mod_type]);
7915         } else {
7916                 if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
7917                     pi->mod_type);
7918         }
7919 }
7920
7921 void
7922 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
7923 {
7924         struct port_info *pi = sc->port[idx];
7925         struct ifnet *ifp = pi->ifp;
7926
7927         if (link_stat) {
7928                 pi->linkdnrc = -1;
7929                 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
7930                 if_link_state_change(ifp, LINK_STATE_UP);
7931         } else {
7932                 if (reason >= 0)
7933                         pi->linkdnrc = reason;
7934                 if_link_state_change(ifp, LINK_STATE_DOWN);
7935         }
7936 }
7937
7938 void
7939 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
7940 {
7941         struct adapter *sc;
7942
7943         sx_slock(&t4_list_lock);
7944         SLIST_FOREACH(sc, &t4_list, link) {
7945                 /*
7946                  * func should not make any assumptions about what state sc is
7947                  * in - the only guarantee is that sc->sc_lock is a valid lock.
7948                  */
7949                 func(sc, arg);
7950         }
7951         sx_sunlock(&t4_list_lock);
7952 }
7953
7954 static int
7955 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
7956 {
7957        return (0);
7958 }
7959
7960 static int
7961 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
7962 {
7963        return (0);
7964 }
7965
7966 static int
7967 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
7968     struct thread *td)
7969 {
7970         int rc;
7971         struct adapter *sc = dev->si_drv1;
7972
7973         rc = priv_check(td, PRIV_DRIVER);
7974         if (rc != 0)
7975                 return (rc);
7976
7977         switch (cmd) {
7978         case CHELSIO_T4_GETREG: {
7979                 struct t4_reg *edata = (struct t4_reg *)data;
7980
7981                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
7982                         return (EFAULT);
7983
7984                 if (edata->size == 4)
7985                         edata->val = t4_read_reg(sc, edata->addr);
7986                 else if (edata->size == 8)
7987                         edata->val = t4_read_reg64(sc, edata->addr);
7988                 else
7989                         return (EINVAL);
7990
7991                 break;
7992         }
7993         case CHELSIO_T4_SETREG: {
7994                 struct t4_reg *edata = (struct t4_reg *)data;
7995
7996                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
7997                         return (EFAULT);
7998
7999                 if (edata->size == 4) {
8000                         if (edata->val & 0xffffffff00000000)
8001                                 return (EINVAL);
8002                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
8003                 } else if (edata->size == 8)
8004                         t4_write_reg64(sc, edata->addr, edata->val);
8005                 else
8006                         return (EINVAL);
8007                 break;
8008         }
8009         case CHELSIO_T4_REGDUMP: {
8010                 struct t4_regdump *regs = (struct t4_regdump *)data;
8011                 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
8012                 uint8_t *buf;
8013
8014                 if (regs->len < reglen) {
8015                         regs->len = reglen; /* hint to the caller */
8016                         return (ENOBUFS);
8017                 }
8018
8019                 regs->len = reglen;
8020                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
8021                 t4_get_regs(sc, regs, buf);
8022                 rc = copyout(buf, regs->data, reglen);
8023                 free(buf, M_CXGBE);
8024                 break;
8025         }
8026         case CHELSIO_T4_GET_FILTER_MODE:
8027                 rc = get_filter_mode(sc, (uint32_t *)data);
8028                 break;
8029         case CHELSIO_T4_SET_FILTER_MODE:
8030                 rc = set_filter_mode(sc, *(uint32_t *)data);
8031                 break;
8032         case CHELSIO_T4_GET_FILTER:
8033                 rc = get_filter(sc, (struct t4_filter *)data);
8034                 break;
8035         case CHELSIO_T4_SET_FILTER:
8036                 rc = set_filter(sc, (struct t4_filter *)data);
8037                 break;
8038         case CHELSIO_T4_DEL_FILTER:
8039                 rc = del_filter(sc, (struct t4_filter *)data);
8040                 break;
8041         case CHELSIO_T4_GET_SGE_CONTEXT:
8042                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
8043                 break;
8044         case CHELSIO_T4_LOAD_FW:
8045                 rc = load_fw(sc, (struct t4_data *)data);
8046                 break;
8047         case CHELSIO_T4_GET_MEM:
8048                 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
8049                 break;
8050         case CHELSIO_T4_GET_I2C:
8051                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
8052                 break;
8053         case CHELSIO_T4_CLEAR_STATS: {
8054                 int i;
8055                 u_int port_id = *(uint32_t *)data;
8056                 struct port_info *pi;
8057
8058                 if (port_id >= sc->params.nports)
8059                         return (EINVAL);
8060                 pi = sc->port[port_id];
8061
8062                 /* MAC stats */
8063                 t4_clr_port_stats(sc, pi->tx_chan);
8064
8065                 if (pi->flags & PORT_INIT_DONE) {
8066                         struct sge_rxq *rxq;
8067                         struct sge_txq *txq;
8068                         struct sge_wrq *wrq;
8069
8070                         for_each_rxq(pi, i, rxq) {
8071 #if defined(INET) || defined(INET6)
8072                                 rxq->lro.lro_queued = 0;
8073                                 rxq->lro.lro_flushed = 0;
8074 #endif
8075                                 rxq->rxcsum = 0;
8076                                 rxq->vlan_extraction = 0;
8077                         }
8078
8079                         for_each_txq(pi, i, txq) {
8080                                 txq->txcsum = 0;
8081                                 txq->tso_wrs = 0;
8082                                 txq->vlan_insertion = 0;
8083                                 txq->imm_wrs = 0;
8084                                 txq->sgl_wrs = 0;
8085                                 txq->txpkt_wrs = 0;
8086                                 txq->txpkts_wrs = 0;
8087                                 txq->txpkts_pkts = 0;
8088                                 txq->br->br_drops = 0;
8089                                 txq->no_dmamap = 0;
8090                                 txq->no_desc = 0;
8091                         }
8092
8093 #ifdef TCP_OFFLOAD
8094                         /* nothing to clear for each ofld_rxq */
8095
8096                         for_each_ofld_txq(pi, i, wrq) {
8097                                 wrq->tx_wrs = 0;
8098                                 wrq->no_desc = 0;
8099                         }
8100 #endif
8101                         wrq = &sc->sge.ctrlq[pi->port_id];
8102                         wrq->tx_wrs = 0;
8103                         wrq->no_desc = 0;
8104                 }
8105                 break;
8106         }
8107         case CHELSIO_T4_SCHED_CLASS:
8108                 rc = set_sched_class(sc, (struct t4_sched_params *)data);
8109                 break;
8110         case CHELSIO_T4_SCHED_QUEUE:
8111                 rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
8112                 break;
8113         case CHELSIO_T4_GET_TRACER:
8114                 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
8115                 break;
8116         case CHELSIO_T4_SET_TRACER:
8117                 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
8118                 break;
8119         default:
8120                 rc = EINVAL;
8121         }
8122
8123         return (rc);
8124 }
8125
8126 #ifdef TCP_OFFLOAD
8127 void
8128 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask,
8129     const unsigned int *pgsz_order)
8130 {
8131         struct port_info *pi = ifp->if_softc;
8132         struct adapter *sc = pi->adapter;
8133
8134         t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
8135         t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |
8136                 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) |
8137                 V_HPZ3(pgsz_order[3]));
8138 }
8139
8140 static int
8141 toe_capability(struct port_info *pi, int enable)
8142 {
8143         int rc;
8144         struct adapter *sc = pi->adapter;
8145
8146         ASSERT_SYNCHRONIZED_OP(sc);
8147
8148         if (!is_offload(sc))
8149                 return (ENODEV);
8150
8151         if (enable) {
8152                 if (!(sc->flags & FULL_INIT_DONE)) {
8153                         rc = cxgbe_init_synchronized(pi);
8154                         if (rc)
8155                                 return (rc);
8156                 }
8157
8158                 if (isset(&sc->offload_map, pi->port_id))
8159                         return (0);
8160
8161                 if (!(sc->flags & TOM_INIT_DONE)) {
8162                         rc = t4_activate_uld(sc, ULD_TOM);
8163                         if (rc == EAGAIN) {
8164                                 log(LOG_WARNING,
8165                                     "You must kldload t4_tom.ko before trying "
8166                                     "to enable TOE on a cxgbe interface.\n");
8167                         }
8168                         if (rc != 0)
8169                                 return (rc);
8170                         KASSERT(sc->tom_softc != NULL,
8171                             ("%s: TOM activated but softc NULL", __func__));
8172                         KASSERT(sc->flags & TOM_INIT_DONE,
8173                             ("%s: TOM activated but flag not set", __func__));
8174                 }
8175
8176                 setbit(&sc->offload_map, pi->port_id);
8177         } else {
8178                 if (!isset(&sc->offload_map, pi->port_id))
8179                         return (0);
8180
8181                 KASSERT(sc->flags & TOM_INIT_DONE,
8182                     ("%s: TOM never initialized?", __func__));
8183                 clrbit(&sc->offload_map, pi->port_id);
8184         }
8185
8186         return (0);
8187 }
8188
8189 /*
8190  * Add an upper layer driver to the global list.
8191  */
8192 int
8193 t4_register_uld(struct uld_info *ui)
8194 {
8195         int rc = 0;
8196         struct uld_info *u;
8197
8198         sx_xlock(&t4_uld_list_lock);
8199         SLIST_FOREACH(u, &t4_uld_list, link) {
8200             if (u->uld_id == ui->uld_id) {
8201                     rc = EEXIST;
8202                     goto done;
8203             }
8204         }
8205
8206         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
8207         ui->refcount = 0;
8208 done:
8209         sx_xunlock(&t4_uld_list_lock);
8210         return (rc);
8211 }
8212
8213 int
8214 t4_unregister_uld(struct uld_info *ui)
8215 {
8216         int rc = EINVAL;
8217         struct uld_info *u;
8218
8219         sx_xlock(&t4_uld_list_lock);
8220
8221         SLIST_FOREACH(u, &t4_uld_list, link) {
8222             if (u == ui) {
8223                     if (ui->refcount > 0) {
8224                             rc = EBUSY;
8225                             goto done;
8226                     }
8227
8228                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
8229                     rc = 0;
8230                     goto done;
8231             }
8232         }
8233 done:
8234         sx_xunlock(&t4_uld_list_lock);
8235         return (rc);
8236 }
8237
8238 int
8239 t4_activate_uld(struct adapter *sc, int id)
8240 {
8241         int rc = EAGAIN;
8242         struct uld_info *ui;
8243
8244         ASSERT_SYNCHRONIZED_OP(sc);
8245
8246         sx_slock(&t4_uld_list_lock);
8247
8248         SLIST_FOREACH(ui, &t4_uld_list, link) {
8249                 if (ui->uld_id == id) {
8250                         rc = ui->activate(sc);
8251                         if (rc == 0)
8252                                 ui->refcount++;
8253                         goto done;
8254                 }
8255         }
8256 done:
8257         sx_sunlock(&t4_uld_list_lock);
8258
8259         return (rc);
8260 }
8261
8262 int
8263 t4_deactivate_uld(struct adapter *sc, int id)
8264 {
8265         int rc = EINVAL;
8266         struct uld_info *ui;
8267
8268         ASSERT_SYNCHRONIZED_OP(sc);
8269
8270         sx_slock(&t4_uld_list_lock);
8271
8272         SLIST_FOREACH(ui, &t4_uld_list, link) {
8273                 if (ui->uld_id == id) {
8274                         rc = ui->deactivate(sc);
8275                         if (rc == 0)
8276                                 ui->refcount--;
8277                         goto done;
8278                 }
8279         }
8280 done:
8281         sx_sunlock(&t4_uld_list_lock);
8282
8283         return (rc);
8284 }
8285 #endif
8286
8287 /*
8288  * Come up with reasonable defaults for some of the tunables, provided they're
8289  * not set by the user (in which case we'll use the values as is).
8290  */
8291 static void
8292 tweak_tunables(void)
8293 {
8294         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
8295
8296         if (t4_ntxq10g < 1)
8297                 t4_ntxq10g = min(nc, NTXQ_10G);
8298
8299         if (t4_ntxq1g < 1)
8300                 t4_ntxq1g = min(nc, NTXQ_1G);
8301
8302         if (t4_nrxq10g < 1)
8303                 t4_nrxq10g = min(nc, NRXQ_10G);
8304
8305         if (t4_nrxq1g < 1)
8306                 t4_nrxq1g = min(nc, NRXQ_1G);
8307
8308 #ifdef TCP_OFFLOAD
8309         if (t4_nofldtxq10g < 1)
8310                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
8311
8312         if (t4_nofldtxq1g < 1)
8313                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
8314
8315         if (t4_nofldrxq10g < 1)
8316                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
8317
8318         if (t4_nofldrxq1g < 1)
8319                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
8320
8321         if (t4_toecaps_allowed == -1)
8322                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
8323 #else
8324         if (t4_toecaps_allowed == -1)
8325                 t4_toecaps_allowed = 0;
8326 #endif
8327
8328 #ifdef DEV_NETMAP
8329         if (t4_nnmtxq10g < 1)
8330                 t4_nnmtxq10g = min(nc, NNMTXQ_10G);
8331
8332         if (t4_nnmtxq1g < 1)
8333                 t4_nnmtxq1g = min(nc, NNMTXQ_1G);
8334
8335         if (t4_nnmrxq10g < 1)
8336                 t4_nnmrxq10g = min(nc, NNMRXQ_10G);
8337
8338         if (t4_nnmrxq1g < 1)
8339                 t4_nnmrxq1g = min(nc, NNMRXQ_1G);
8340 #endif
8341
8342         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
8343                 t4_tmr_idx_10g = TMR_IDX_10G;
8344
8345         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
8346                 t4_pktc_idx_10g = PKTC_IDX_10G;
8347
8348         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
8349                 t4_tmr_idx_1g = TMR_IDX_1G;
8350
8351         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
8352                 t4_pktc_idx_1g = PKTC_IDX_1G;
8353
8354         if (t4_qsize_txq < 128)
8355                 t4_qsize_txq = 128;
8356
8357         if (t4_qsize_rxq < 128)
8358                 t4_qsize_rxq = 128;
8359         while (t4_qsize_rxq & 7)
8360                 t4_qsize_rxq++;
8361
8362         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
8363 }
8364
8365 static struct sx mlu;   /* mod load unload */
8366 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
8367
8368 static int
8369 mod_event(module_t mod, int cmd, void *arg)
8370 {
8371         int rc = 0;
8372         static int loaded = 0;
8373
8374         switch (cmd) {
8375         case MOD_LOAD:
8376                 sx_xlock(&mlu);
8377                 if (loaded++ == 0) {
8378                         t4_sge_modload();
8379                         sx_init(&t4_list_lock, "T4/T5 adapters");
8380                         SLIST_INIT(&t4_list);
8381 #ifdef TCP_OFFLOAD
8382                         sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
8383                         SLIST_INIT(&t4_uld_list);
8384 #endif
8385                         t4_tracer_modload();
8386                         tweak_tunables();
8387                 }
8388                 sx_xunlock(&mlu);
8389                 break;
8390
8391         case MOD_UNLOAD:
8392                 sx_xlock(&mlu);
8393                 if (--loaded == 0) {
8394                         int tries;
8395
8396                         sx_slock(&t4_list_lock);
8397                         if (!SLIST_EMPTY(&t4_list)) {
8398                                 rc = EBUSY;
8399                                 sx_sunlock(&t4_list_lock);
8400                                 goto done_unload;
8401                         }
8402 #ifdef TCP_OFFLOAD
8403                         sx_slock(&t4_uld_list_lock);
8404                         if (!SLIST_EMPTY(&t4_uld_list)) {
8405                                 rc = EBUSY;
8406                                 sx_sunlock(&t4_uld_list_lock);
8407                                 sx_sunlock(&t4_list_lock);
8408                                 goto done_unload;
8409                         }
8410 #endif
8411                         tries = 0;
8412                         while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
8413                                 uprintf("%ju clusters with custom free routine "
8414                                     "still is use.\n", t4_sge_extfree_refs());
8415                                 pause("t4unload", 2 * hz);
8416                         }
8417 #ifdef TCP_OFFLOAD
8418                         sx_sunlock(&t4_uld_list_lock);
8419 #endif
8420                         sx_sunlock(&t4_list_lock);
8421
8422                         if (t4_sge_extfree_refs() == 0) {
8423                                 t4_tracer_modunload();
8424 #ifdef TCP_OFFLOAD
8425                                 sx_destroy(&t4_uld_list_lock);
8426 #endif
8427                                 sx_destroy(&t4_list_lock);
8428                                 t4_sge_modunload();
8429                                 loaded = 0;
8430                         } else {
8431                                 rc = EBUSY;
8432                                 loaded++;       /* undo earlier decrement */
8433                         }
8434                 }
8435 done_unload:
8436                 sx_xunlock(&mlu);
8437                 break;
8438         }
8439
8440         return (rc);
8441 }
8442
8443 static devclass_t t4_devclass, t5_devclass;
8444 static devclass_t cxgbe_devclass, cxl_devclass;
8445
8446 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
8447 MODULE_VERSION(t4nex, 1);
8448 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
8449
8450 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
8451 MODULE_VERSION(t5nex, 1);
8452 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
8453
8454 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
8455 MODULE_VERSION(cxgbe, 1);
8456
8457 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
8458 MODULE_VERSION(cxl, 1);