sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/conf.h>
  36 #include <sys/priv.h>
  37 #include <sys/kernel.h>
  38 #include <sys/bus.h>
  39 #include <sys/module.h>
  40 #include <sys/malloc.h>
  41 #include <sys/queue.h>
  42 #include <sys/taskqueue.h>
  43 #include <sys/pciio.h>
  44 #include <dev/pci/pcireg.h>
  45 #include <dev/pci/pcivar.h>
  46 #include <dev/pci/pci_private.h>
  47 #include <sys/firmware.h>
  48 #include <sys/sbuf.h>
  49 #include <sys/smp.h>
  50 #include <sys/socket.h>
  51 #include <sys/sockio.h>
  52 #include <sys/sysctl.h>
  53 #include <net/ethernet.h>
  54 #include <net/if.h>
  55 #include <net/if_types.h>
  56 #include <net/if_dl.h>
  57 #include <net/if_vlan_var.h>
  58 #if defined(__i386__) || defined(__amd64__)
  59 #include <vm/vm.h>
  60 #include <vm/pmap.h>
  61 #endif
  62
  63 #include "common/common.h"
  64 #include "common/t4_msg.h"
  65 #include "common/t4_regs.h"
  66 #include "common/t4_regs_values.h"
  67 #include "t4_ioctl.h"
  68 #include "t4_l2t.h"
  69
  70 /* T4 bus driver interface */
  71 static int t4_probe(device_t);
  72 static int t4_attach(device_t);
  73 static int t4_detach(device_t);
  74 static device_method_t t4_methods[] = {
  75         DEVMETHOD(device_probe,         t4_probe),
  76         DEVMETHOD(device_attach,        t4_attach),
  77         DEVMETHOD(device_detach,        t4_detach),
  78
  79         DEVMETHOD_END
  80 };
  81 static driver_t t4_driver = {
  82         "t4nex",
  83         t4_methods,
  84         sizeof(struct adapter)
  85 };
  86
  87
  88 /* T4 port (cxgbe) interface */
  89 static int cxgbe_probe(device_t);
  90 static int cxgbe_attach(device_t);
  91 static int cxgbe_detach(device_t);
  92 static device_method_t cxgbe_methods[] = {
  93         DEVMETHOD(device_probe,         cxgbe_probe),
  94         DEVMETHOD(device_attach,        cxgbe_attach),
  95         DEVMETHOD(device_detach,        cxgbe_detach),
  96         { 0, 0 }
  97 };
  98 static driver_t cxgbe_driver = {
  99         "cxgbe",
 100         cxgbe_methods,
 101         sizeof(struct port_info)
 102 };
 103
 104 static d_ioctl_t t4_ioctl;
 105 static d_open_t t4_open;
 106 static d_close_t t4_close;
 107
 108 static struct cdevsw t4_cdevsw = {
 109        .d_version = D_VERSION,
 110        .d_flags = 0,
 111        .d_open = t4_open,
 112        .d_close = t4_close,
 113        .d_ioctl = t4_ioctl,
 114        .d_name = "t4nex",
 115 };
 116
 117 /* T5 bus driver interface */
 118 static int t5_probe(device_t);
 119 static device_method_t t5_methods[] = {
 120         DEVMETHOD(device_probe,         t5_probe),
 121         DEVMETHOD(device_attach,        t4_attach),
 122         DEVMETHOD(device_detach,        t4_detach),
 123
 124         DEVMETHOD_END
 125 };
 126 static driver_t t5_driver = {
 127         "t5nex",
 128         t5_methods,
 129         sizeof(struct adapter)
 130 };
 131
 132
 133 /* T5 port (cxl) interface */
 134 static driver_t cxl_driver = {
 135         "cxl",
 136         cxgbe_methods,
 137         sizeof(struct port_info)
 138 };
 139
 140 static struct cdevsw t5_cdevsw = {
 141        .d_version = D_VERSION,
 142        .d_flags = 0,
 143        .d_open = t4_open,
 144        .d_close = t4_close,
 145        .d_ioctl = t4_ioctl,
 146        .d_name = "t5nex",
 147 };
 148
 149 /* ifnet + media interface */
 150 static void cxgbe_init(void *);
 151 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 152 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 153 static void cxgbe_qflush(struct ifnet *);
 154 static int cxgbe_media_change(struct ifnet *);
 155 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 156
 157 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
 158
 159 /*
 160  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 161  * then ADAPTER_LOCK, then t4_uld_list_lock.
 162  */
 163 static struct sx t4_list_lock;
 164 SLIST_HEAD(, adapter) t4_list;
 165 #ifdef TCP_OFFLOAD
 166 static struct sx t4_uld_list_lock;
 167 SLIST_HEAD(, uld_info) t4_uld_list;
 168 #endif
 169
 170 /*
 171  * Tunables.  See tweak_tunables() too.
 172  *
 173  * Each tunable is set to a default value here if it's known at compile-time.
 174  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
 175  * provide a reasonable default when the driver is loaded.
 176  *
 177  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
 178  * T5 are under hw.cxl.
 179  */
 180
 181 /*
 182  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 183  */
 184 #define NTXQ_10G 16
 185 static int t4_ntxq10g = -1;
 186 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 187
 188 #define NRXQ_10G 8
 189 static int t4_nrxq10g = -1;
 190 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 191
 192 #define NTXQ_1G 4
 193 static int t4_ntxq1g = -1;
 194 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 195
 196 #define NRXQ_1G 2
 197 static int t4_nrxq1g = -1;
 198 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 199
 200 static int t4_rsrv_noflowq = 0;
 201 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
 202
 203 #ifdef TCP_OFFLOAD
 204 #define NOFLDTXQ_10G 8
 205 static int t4_nofldtxq10g = -1;
 206 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 207
 208 #define NOFLDRXQ_10G 2
 209 static int t4_nofldrxq10g = -1;
 210 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 211
 212 #define NOFLDTXQ_1G 2
 213 static int t4_nofldtxq1g = -1;
 214 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 215
 216 #define NOFLDRXQ_1G 1
 217 static int t4_nofldrxq1g = -1;
 218 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 219 #endif
 220
 221 #ifdef DEV_NETMAP
 222 #define NNMTXQ_10G 2
 223 static int t4_nnmtxq10g = -1;
 224 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g);
 225
 226 #define NNMRXQ_10G 2
 227 static int t4_nnmrxq10g = -1;
 228 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g);
 229
 230 #define NNMTXQ_1G 1
 231 static int t4_nnmtxq1g = -1;
 232 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g);
 233
 234 #define NNMRXQ_1G 1
 235 static int t4_nnmrxq1g = -1;
 236 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g);
 237 #endif
 238
 239 /*
 240  * Holdoff parameters for 10G and 1G ports.
 241  */
 242 #define TMR_IDX_10G 1
 243 static int t4_tmr_idx_10g = TMR_IDX_10G;
 244 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 245
 246 #define PKTC_IDX_10G (-1)
 247 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 248 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 249
 250 #define TMR_IDX_1G 1
 251 static int t4_tmr_idx_1g = TMR_IDX_1G;
 252 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 253
 254 #define PKTC_IDX_1G (-1)
 255 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 256 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 257
 258 /*
 259  * Size (# of entries) of each tx and rx queue.
 260  */
 261 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 262 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 263
 264 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 265 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 266
 267 /*
 268  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 269  */
 270 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 271 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 272
 273 /*
 274  * Configuration file.
 275  */
 276 #define DEFAULT_CF      "default"
 277 #define FLASH_CF        "flash"
 278 #define UWIRE_CF        "uwire"
 279 #define FPGA_CF         "fpga"
 280 static char t4_cfg_file[32] = DEFAULT_CF;
 281 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 282
 283 /*
 284  * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively).
 285  * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
 286  * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
 287  *            mark or when signalled to do so, 0 to never emit PAUSE.
 288  */
 289 static int t4_pause_settings = PAUSE_TX | PAUSE_RX;
 290 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings);
 291
 292 /*
 293  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
 294  * encouraged respectively).
 295  */
 296 static unsigned int t4_fw_install = 1;
 297 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 298
 299 /*
 300  * ASIC features that will be used.  Disable the ones you don't want so that the
 301  * chip resources aren't wasted on features that will not be used.
 302  */
 303 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 304 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 305
 306 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 307 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 308
 309 static int t4_toecaps_allowed = -1;
 310 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 311
 312 static int t4_rdmacaps_allowed = 0;
 313 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 314
 315 static int t4_iscsicaps_allowed = 0;
 316 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 317
 318 static int t4_fcoecaps_allowed = 0;
 319 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 320
 321 static int t5_write_combine = 0;
 322 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 323
 324 struct intrs_and_queues {
 325         uint16_t intr_type;     /* INTx, MSI, or MSI-X */
 326         uint16_t nirq;          /* Total # of vectors */
 327         uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
 328         uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */
 329         uint16_t ntxq10g;       /* # of NIC txq's for each 10G port */
 330         uint16_t nrxq10g;       /* # of NIC rxq's for each 10G port */
 331         uint16_t ntxq1g;        /* # of NIC txq's for each 1G port */
 332         uint16_t nrxq1g;        /* # of NIC rxq's for each 1G port */
 333         uint16_t rsrv_noflowq;  /* Flag whether to reserve queue 0 */
 334 #ifdef TCP_OFFLOAD
 335         uint16_t nofldtxq10g;   /* # of TOE txq's for each 10G port */
 336         uint16_t nofldrxq10g;   /* # of TOE rxq's for each 10G port */
 337         uint16_t nofldtxq1g;    /* # of TOE txq's for each 1G port */
 338         uint16_t nofldrxq1g;    /* # of TOE rxq's for each 1G port */
 339 #endif
 340 #ifdef DEV_NETMAP
 341         uint16_t nnmtxq10g;     /* # of netmap txq's for each 10G port */
 342         uint16_t nnmrxq10g;     /* # of netmap rxq's for each 10G port */
 343         uint16_t nnmtxq1g;      /* # of netmap txq's for each 1G port */
 344         uint16_t nnmrxq1g;      /* # of netmap rxq's for each 1G port */
 345 #endif
 346 };
 347
 348 struct filter_entry {
 349         uint32_t valid:1;       /* filter allocated and valid */
 350         uint32_t locked:1;      /* filter is administratively locked */
 351         uint32_t pending:1;     /* filter action is pending firmware reply */
 352         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 353         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 354
 355         struct t4_filter_specification fs;
 356 };
 357
 358 static int map_bars_0_and_4(struct adapter *);
 359 static int map_bar_2(struct adapter *);
 360 static void setup_memwin(struct adapter *);
 361 static int validate_mem_range(struct adapter *, uint32_t, int);
 362 static int fwmtype_to_hwmtype(int);
 363 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
 364     uint32_t *);
 365 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
 366 static uint32_t position_memwin(struct adapter *, int, uint32_t);
 367 static int cfg_itype_and_nqueues(struct adapter *, int, int,
 368     struct intrs_and_queues *);
 369 static int prep_firmware(struct adapter *);
 370 static int partition_resources(struct adapter *, const struct firmware *,
 371     const char *);
 372 static int get_params__pre_init(struct adapter *);
 373 static int get_params__post_init(struct adapter *);
 374 static int set_params__post_init(struct adapter *);
 375 static void t4_set_desc(struct adapter *);
 376 static void build_medialist(struct port_info *, struct ifmedia *);
 377 static int cxgbe_init_synchronized(struct port_info *);
 378 static int cxgbe_uninit_synchronized(struct port_info *);
 379 static int setup_intr_handlers(struct adapter *);
 380 static void quiesce_eq(struct adapter *, struct sge_eq *);
 381 static void quiesce_iq(struct adapter *, struct sge_iq *);
 382 static void quiesce_fl(struct adapter *, struct sge_fl *);
 383 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 384     driver_intr_t *, void *, char *);
 385 static int t4_free_irq(struct adapter *, struct irq *);
 386 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
 387     unsigned int);
 388 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 389 static void cxgbe_tick(void *);
 390 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 391 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
 392     struct mbuf *);
 393 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 394 static int fw_msg_not_handled(struct adapter *, const __be64 *);
 395 static int t4_sysctls(struct adapter *);
 396 static int cxgbe_sysctls(struct port_info *);
 397 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 398 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 399 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
 400 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 401 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 402 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 403 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 404 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 405 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
 406 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 407 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 408 #ifdef SBUF_DRAIN
 409 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 410 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 411 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 412 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
 413 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
 414 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 415 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 416 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 417 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 418 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 419 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 420 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 421 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
 422 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 423 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
 424 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 425 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 426 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 427 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 428 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 429 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 430 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
 431 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 432 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 433 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 434 #endif
 435 static inline void txq_start(struct ifnet *, struct sge_txq *);
 436 static uint32_t fconf_to_mode(uint32_t);
 437 static uint32_t mode_to_fconf(uint32_t);
 438 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 439 static int get_filter_mode(struct adapter *, uint32_t *);
 440 static int set_filter_mode(struct adapter *, uint32_t);
 441 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 442 static int get_filter(struct adapter *, struct t4_filter *);
 443 static int set_filter(struct adapter *, struct t4_filter *);
 444 static int del_filter(struct adapter *, struct t4_filter *);
 445 static void clear_filter(struct filter_entry *);
 446 static int set_filter_wr(struct adapter *, int);
 447 static int del_filter_wr(struct adapter *, int);
 448 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 449 static int load_fw(struct adapter *, struct t4_data *);
 450 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 451 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 452 static int set_sched_class(struct adapter *, struct t4_sched_params *);
 453 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
 454 #ifdef TCP_OFFLOAD
 455 static int toe_capability(struct port_info *, int);
 456 #endif
 457 static int mod_event(module_t, int, void *);
 458
 459 struct {
 460         uint16_t device;
 461         char *desc;
 462 } t4_pciids[] = {
 463         {0xa000, "Chelsio Terminator 4 FPGA"},
 464         {0x4400, "Chelsio T440-dbg"},
 465         {0x4401, "Chelsio T420-CR"},
 466         {0x4402, "Chelsio T422-CR"},
 467         {0x4403, "Chelsio T440-CR"},
 468         {0x4404, "Chelsio T420-BCH"},
 469         {0x4405, "Chelsio T440-BCH"},
 470         {0x4406, "Chelsio T440-CH"},
 471         {0x4407, "Chelsio T420-SO"},
 472         {0x4408, "Chelsio T420-CX"},
 473         {0x4409, "Chelsio T420-BT"},
 474         {0x440a, "Chelsio T404-BT"},
 475         {0x440e, "Chelsio T440-LP-CR"},
 476 }, t5_pciids[] = {
 477         {0xb000, "Chelsio Terminator 5 FPGA"},
 478         {0x5400, "Chelsio T580-dbg"},
 479         {0x5401,  "Chelsio T520-CR"},           /* 2 x 10G */
 480         {0x5402,  "Chelsio T522-CR"},           /* 2 x 10G, 2 X 1G */
 481         {0x5403,  "Chelsio T540-CR"},           /* 4 x 10G */
 482         {0x5407,  "Chelsio T520-SO"},           /* 2 x 10G, nomem */
 483         {0x5409,  "Chelsio T520-BT"},           /* 2 x 10GBaseT */
 484         {0x540a,  "Chelsio T504-BT"},           /* 4 x 1G */
 485         {0x540d,  "Chelsio T580-CR"},           /* 2 x 40G */
 486         {0x540e,  "Chelsio T540-LP-CR"},        /* 4 x 10G */
 487         {0x5410,  "Chelsio T580-LP-CR"},        /* 2 x 40G */
 488         {0x5411,  "Chelsio T520-LL-CR"},        /* 2 x 10G */
 489         {0x5412,  "Chelsio T560-CR"},           /* 1 x 40G, 2 x 10G */
 490         {0x5414,  "Chelsio T580-LP-SO-CR"},     /* 2 x 40G, nomem */
 491         {0x5415,  "Chelsio T502-BT"},           /* 2 x 1G */
 492 #ifdef notyet
 493         {0x5404,  "Chelsio T520-BCH"},
 494         {0x5405,  "Chelsio T540-BCH"},
 495         {0x5406,  "Chelsio T540-CH"},
 496         {0x5408,  "Chelsio T520-CX"},
 497         {0x540b,  "Chelsio B520-SR"},
 498         {0x540c,  "Chelsio B504-BT"},
 499         {0x540f,  "Chelsio Amsterdam"},
 500         {0x5413,  "Chelsio T580-CHR"},
 501 #endif
 502 };
 503
 504 #ifdef TCP_OFFLOAD
 505 /*
 506  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 507  * exactly the same for both rxq and ofld_rxq.
 508  */
 509 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 510 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 511 #endif
 512
 513 /* No easy way to include t4_msg.h before adapter.h so we check this way */
 514 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
 515 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
 516
 517 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
 518
 519 static int
 520 t4_probe(device_t dev)
 521 {
 522         int i;
 523         uint16_t v = pci_get_vendor(dev);
 524         uint16_t d = pci_get_device(dev);
 525         uint8_t f = pci_get_function(dev);
 526
 527         if (v != PCI_VENDOR_ID_CHELSIO)
 528                 return (ENXIO);
 529
 530         /* Attach only to PF0 of the FPGA */
 531         if (d == 0xa000 && f != 0)
 532                 return (ENXIO);
 533
 534         for (i = 0; i < nitems(t4_pciids); i++) {
 535                 if (d == t4_pciids[i].device) {
 536                         device_set_desc(dev, t4_pciids[i].desc);
 537                         return (BUS_PROBE_DEFAULT);
 538                 }
 539         }
 540
 541         return (ENXIO);
 542 }
 543
 544 static int
 545 t5_probe(device_t dev)
 546 {
 547         int i;
 548         uint16_t v = pci_get_vendor(dev);
 549         uint16_t d = pci_get_device(dev);
 550         uint8_t f = pci_get_function(dev);
 551
 552         if (v != PCI_VENDOR_ID_CHELSIO)
 553                 return (ENXIO);
 554
 555         /* Attach only to PF0 of the FPGA */
 556         if (d == 0xb000 && f != 0)
 557                 return (ENXIO);
 558
 559         for (i = 0; i < nitems(t5_pciids); i++) {
 560                 if (d == t5_pciids[i].device) {
 561                         device_set_desc(dev, t5_pciids[i].desc);
 562                         return (BUS_PROBE_DEFAULT);
 563                 }
 564         }
 565
 566         return (ENXIO);
 567 }
 568
 569 static int
 570 t4_attach(device_t dev)
 571 {
 572         struct adapter *sc;
 573         int rc = 0, i, n10g, n1g, rqidx, tqidx;
 574         struct intrs_and_queues iaq;
 575         struct sge *s;
 576 #ifdef TCP_OFFLOAD
 577         int ofld_rqidx, ofld_tqidx;
 578 #endif
 579 #ifdef DEV_NETMAP
 580         int nm_rqidx, nm_tqidx;
 581 #endif
 582
 583         sc = device_get_softc(dev);
 584         sc->dev = dev;
 585
 586         pci_enable_busmaster(dev);
 587         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 588                 uint32_t v;
 589
 590                 pci_set_max_read_req(dev, 4096);
 591                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 592                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 593                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 594
 595                 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
 596         }
 597
 598         sc->traceq = -1;
 599         mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
 600         snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
 601             device_get_nameunit(dev));
 602
 603         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 604             device_get_nameunit(dev));
 605         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 606         sx_xlock(&t4_list_lock);
 607         SLIST_INSERT_HEAD(&t4_list, sc, link);
 608         sx_xunlock(&t4_list_lock);
 609
 610         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 611         TAILQ_INIT(&sc->sfl);
 612         callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
 613
 614         rc = map_bars_0_and_4(sc);
 615         if (rc != 0)
 616                 goto done; /* error message displayed already */
 617
 618         /*
 619          * This is the real PF# to which we're attaching.  Works from within PCI
 620          * passthrough environments too, where pci_get_function() could return a
 621          * different PF# depending on the passthrough configuration.  We need to
 622          * use the real PF# in all our communication with the firmware.
 623          */
 624         sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 625         sc->mbox = sc->pf;
 626
 627         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 628         sc->an_handler = an_not_handled;
 629         for (i = 0; i < nitems(sc->cpl_handler); i++)
 630                 sc->cpl_handler[i] = cpl_not_handled;
 631         for (i = 0; i < nitems(sc->fw_msg_handler); i++)
 632                 sc->fw_msg_handler[i] = fw_msg_not_handled;
 633         t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 634         t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
 635         t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt);
 636         t4_init_sge_cpl_handlers(sc);
 637
 638         /* Prepare the adapter for operation */
 639         rc = -t4_prep_adapter(sc);
 640         if (rc != 0) {
 641                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 642                 goto done;
 643         }
 644
 645         /*
 646          * Do this really early, with the memory windows set up even before the
 647          * character device.  The userland tool's register i/o and mem read
 648          * will work even in "recovery mode".
 649          */
 650         setup_memwin(sc);
 651         sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
 652             device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
 653             device_get_nameunit(dev));
 654         if (sc->cdev == NULL)
 655                 device_printf(dev, "failed to create nexus char device.\n");
 656         else
 657                 sc->cdev->si_drv1 = sc;
 658
 659         /* Go no further if recovery mode has been requested. */
 660         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 661                 device_printf(dev, "recovery mode.\n");
 662                 goto done;
 663         }
 664
 665         /* Prepare the firmware for operation */
 666         rc = prep_firmware(sc);
 667         if (rc != 0)
 668                 goto done; /* error message displayed already */
 669
 670         rc = get_params__post_init(sc);
 671         if (rc != 0)
 672                 goto done; /* error message displayed already */
 673
 674         rc = set_params__post_init(sc);
 675         if (rc != 0)
 676                 goto done; /* error message displayed already */
 677
 678         rc = map_bar_2(sc);
 679         if (rc != 0)
 680                 goto done; /* error message displayed already */
 681
 682         rc = t4_create_dma_tag(sc);
 683         if (rc != 0)
 684                 goto done; /* error message displayed already */
 685
 686         /*
 687          * First pass over all the ports - allocate VIs and initialize some
 688          * basic parameters like mac address, port type, etc.  We also figure
 689          * out whether a port is 10G or 1G and use that information when
 690          * calculating how many interrupts to attempt to allocate.
 691          */
 692         n10g = n1g = 0;
 693         for_each_port(sc, i) {
 694                 struct port_info *pi;
 695
 696                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 697                 sc->port[i] = pi;
 698
 699                 /* These must be set before t4_port_init */
 700                 pi->adapter = sc;
 701                 pi->port_id = i;
 702
 703                 /* Allocate the vi and initialize parameters like mac addr */
 704                 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 705                 if (rc != 0) {
 706                         device_printf(dev, "unable to initialize port %d: %d\n",
 707                             i, rc);
 708                         free(pi, M_CXGBE);
 709                         sc->port[i] = NULL;
 710                         goto done;
 711                 }
 712
 713                 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX);
 714                 pi->link_cfg.requested_fc |= t4_pause_settings;
 715                 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX);
 716                 pi->link_cfg.fc |= t4_pause_settings;
 717
 718                 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
 719                 if (rc != 0) {
 720                         device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
 721                         free(pi, M_CXGBE);
 722                         sc->port[i] = NULL;
 723                         goto done;
 724                 }
 725
 726                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 727                     device_get_nameunit(dev), i);
 728                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 729                 sc->chan_map[pi->tx_chan] = i;
 730
 731                 if (is_10G_port(pi) || is_40G_port(pi)) {
 732                         n10g++;
 733                         pi->tmr_idx = t4_tmr_idx_10g;
 734                         pi->pktc_idx = t4_pktc_idx_10g;
 735                 } else {
 736                         n1g++;
 737                         pi->tmr_idx = t4_tmr_idx_1g;
 738                         pi->pktc_idx = t4_pktc_idx_1g;
 739                 }
 740
 741                 pi->xact_addr_filt = -1;
 742                 pi->linkdnrc = -1;
 743
 744                 pi->qsize_rxq = t4_qsize_rxq;
 745                 pi->qsize_txq = t4_qsize_txq;
 746
 747                 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
 748                 if (pi->dev == NULL) {
 749                         device_printf(dev,
 750                             "failed to add device for port %d.\n", i);
 751                         rc = ENXIO;
 752                         goto done;
 753                 }
 754                 device_set_softc(pi->dev, pi);
 755         }
 756
 757         /*
 758          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 759          */
 760         rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
 761         if (rc != 0)
 762                 goto done; /* error message displayed already */
 763
 764         sc->intr_type = iaq.intr_type;
 765         sc->intr_count = iaq.nirq;
 766
 767         s = &sc->sge;
 768         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 769         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 770         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
 771         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 772         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
 773 #ifdef TCP_OFFLOAD
 774         if (is_offload(sc)) {
 775                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 776                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 777                 s->neq += s->nofldtxq + s->nofldrxq;
 778                 s->niq += s->nofldrxq;
 779
 780                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 781                     M_CXGBE, M_ZERO | M_WAITOK);
 782                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 783                     M_CXGBE, M_ZERO | M_WAITOK);
 784         }
 785 #endif
 786 #ifdef DEV_NETMAP
 787         s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g;
 788         s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g;
 789         s->neq += s->nnmtxq + s->nnmrxq;
 790         s->niq += s->nnmrxq;
 791
 792         s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
 793             M_CXGBE, M_ZERO | M_WAITOK);
 794         s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
 795             M_CXGBE, M_ZERO | M_WAITOK);
 796 #endif
 797
 798         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 799             M_ZERO | M_WAITOK);
 800         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 801             M_ZERO | M_WAITOK);
 802         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 803             M_ZERO | M_WAITOK);
 804         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 805             M_ZERO | M_WAITOK);
 806         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 807             M_ZERO | M_WAITOK);
 808
 809         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 810             M_ZERO | M_WAITOK);
 811
 812         t4_init_l2t(sc, M_WAITOK);
 813
 814         /*
 815          * Second pass over the ports.  This time we know the number of rx and
 816          * tx queues that each port should get.
 817          */
 818         rqidx = tqidx = 0;
 819 #ifdef TCP_OFFLOAD
 820         ofld_rqidx = ofld_tqidx = 0;
 821 #endif
 822 #ifdef DEV_NETMAP
 823         nm_rqidx = nm_tqidx = 0;
 824 #endif
 825         for_each_port(sc, i) {
 826                 struct port_info *pi = sc->port[i];
 827
 828                 if (pi == NULL)
 829                         continue;
 830
 831                 pi->first_rxq = rqidx;
 832                 pi->first_txq = tqidx;
 833                 if (is_10G_port(pi) || is_40G_port(pi)) {
 834                         pi->flags |= iaq.intr_flags_10g;
 835                         pi->nrxq = iaq.nrxq10g;
 836                         pi->ntxq = iaq.ntxq10g;
 837                 } else {
 838                         pi->flags |= iaq.intr_flags_1g;
 839                         pi->nrxq = iaq.nrxq1g;
 840                         pi->ntxq = iaq.ntxq1g;
 841                 }
 842
 843                 if (pi->ntxq > 1)
 844                         pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
 845                 else
 846                         pi->rsrv_noflowq = 0;
 847
 848                 rqidx += pi->nrxq;
 849                 tqidx += pi->ntxq;
 850 #ifdef TCP_OFFLOAD
 851                 if (is_offload(sc)) {
 852                         pi->first_ofld_rxq = ofld_rqidx;
 853                         pi->first_ofld_txq = ofld_tqidx;
 854                         if (is_10G_port(pi) || is_40G_port(pi)) {
 855                                 pi->nofldrxq = iaq.nofldrxq10g;
 856                                 pi->nofldtxq = iaq.nofldtxq10g;
 857                         } else {
 858                                 pi->nofldrxq = iaq.nofldrxq1g;
 859                                 pi->nofldtxq = iaq.nofldtxq1g;
 860                         }
 861                         ofld_rqidx += pi->nofldrxq;
 862                         ofld_tqidx += pi->nofldtxq;
 863                 }
 864 #endif
 865 #ifdef DEV_NETMAP
 866                 pi->first_nm_rxq = nm_rqidx;
 867                 pi->first_nm_txq = nm_tqidx;
 868                 if (is_10G_port(pi) || is_40G_port(pi)) {
 869                         pi->nnmrxq = iaq.nnmrxq10g;
 870                         pi->nnmtxq = iaq.nnmtxq10g;
 871                 } else {
 872                         pi->nnmrxq = iaq.nnmrxq1g;
 873                         pi->nnmtxq = iaq.nnmtxq1g;
 874                 }
 875                 nm_rqidx += pi->nnmrxq;
 876                 nm_tqidx += pi->nnmtxq;
 877 #endif
 878         }
 879
 880         rc = setup_intr_handlers(sc);
 881         if (rc != 0) {
 882                 device_printf(dev,
 883                     "failed to setup interrupt handlers: %d\n", rc);
 884                 goto done;
 885         }
 886
 887         rc = bus_generic_attach(dev);
 888         if (rc != 0) {
 889                 device_printf(dev,
 890                     "failed to attach all child ports: %d\n", rc);
 891                 goto done;
 892         }
 893
 894         device_printf(dev,
 895             "PCIe x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
 896             sc->params.pci.width, sc->params.nports, sc->intr_count,
 897             sc->intr_type == INTR_MSIX ? "MSI-X" :
 898             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
 899             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
 900
 901         t4_set_desc(sc);
 902
 903 done:
 904         if (rc != 0 && sc->cdev) {
 905                 /* cdev was created and so cxgbetool works; recover that way. */
 906                 device_printf(dev,
 907                     "error during attach, adapter is now in recovery mode.\n");
 908                 rc = 0;
 909         }
 910
 911         if (rc != 0)
 912                 t4_detach(dev);
 913         else
 914                 t4_sysctls(sc);
 915
 916         return (rc);
 917 }
 918
 919 /*
 920  * Idempotent
 921  */
 922 static int
 923 t4_detach(device_t dev)
 924 {
 925         struct adapter *sc;
 926         struct port_info *pi;
 927         int i, rc;
 928
 929         sc = device_get_softc(dev);
 930
 931         if (sc->flags & FULL_INIT_DONE)
 932                 t4_intr_disable(sc);
 933
 934         if (sc->cdev) {
 935                 destroy_dev(sc->cdev);
 936                 sc->cdev = NULL;
 937         }
 938
 939         rc = bus_generic_detach(dev);
 940         if (rc) {
 941                 device_printf(dev,
 942                     "failed to detach child devices: %d\n", rc);
 943                 return (rc);
 944         }
 945
 946         for (i = 0; i < sc->intr_count; i++)
 947                 t4_free_irq(sc, &sc->irq[i]);
 948
 949         for (i = 0; i < MAX_NPORTS; i++) {
 950                 pi = sc->port[i];
 951                 if (pi) {
 952                         t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid);
 953                         if (pi->dev)
 954                                 device_delete_child(dev, pi->dev);
 955
 956                         mtx_destroy(&pi->pi_lock);
 957                         free(pi, M_CXGBE);
 958                 }
 959         }
 960
 961         if (sc->flags & FULL_INIT_DONE)
 962                 adapter_full_uninit(sc);
 963
 964         if (sc->flags & FW_OK)
 965                 t4_fw_bye(sc, sc->mbox);
 966
 967         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
 968                 pci_release_msi(dev);
 969
 970         if (sc->regs_res)
 971                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
 972                     sc->regs_res);
 973
 974         if (sc->udbs_res)
 975                 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
 976                     sc->udbs_res);
 977
 978         if (sc->msix_res)
 979                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
 980                     sc->msix_res);
 981
 982         if (sc->l2t)
 983                 t4_free_l2t(sc->l2t);
 984
 985 #ifdef TCP_OFFLOAD
 986         free(sc->sge.ofld_rxq, M_CXGBE);
 987         free(sc->sge.ofld_txq, M_CXGBE);
 988 #endif
 989 #ifdef DEV_NETMAP
 990         free(sc->sge.nm_rxq, M_CXGBE);
 991         free(sc->sge.nm_txq, M_CXGBE);
 992 #endif
 993         free(sc->irq, M_CXGBE);
 994         free(sc->sge.rxq, M_CXGBE);
 995         free(sc->sge.txq, M_CXGBE);
 996         free(sc->sge.ctrlq, M_CXGBE);
 997         free(sc->sge.iqmap, M_CXGBE);
 998         free(sc->sge.eqmap, M_CXGBE);
 999         free(sc->tids.ftid_tab, M_CXGBE);
1000         t4_destroy_dma_tag(sc);
1001         if (mtx_initialized(&sc->sc_lock)) {
1002                 sx_xlock(&t4_list_lock);
1003                 SLIST_REMOVE(&t4_list, sc, adapter, link);
1004                 sx_xunlock(&t4_list_lock);
1005                 mtx_destroy(&sc->sc_lock);
1006         }
1007
1008         if (mtx_initialized(&sc->tids.ftid_lock))
1009                 mtx_destroy(&sc->tids.ftid_lock);
1010         if (mtx_initialized(&sc->sfl_lock))
1011                 mtx_destroy(&sc->sfl_lock);
1012         if (mtx_initialized(&sc->ifp_lock))
1013                 mtx_destroy(&sc->ifp_lock);
1014
1015         bzero(sc, sizeof(*sc));
1016
1017         return (0);
1018 }
1019
1020 static int
1021 cxgbe_probe(device_t dev)
1022 {
1023         char buf[128];
1024         struct port_info *pi = device_get_softc(dev);
1025
1026         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
1027         device_set_desc_copy(dev, buf);
1028
1029         return (BUS_PROBE_DEFAULT);
1030 }
1031
1032 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
1033     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1034     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
1035 #define T4_CAP_ENABLE (T4_CAP)
1036
1037 static int
1038 cxgbe_attach(device_t dev)
1039 {
1040         struct port_info *pi = device_get_softc(dev);
1041         struct ifnet *ifp;
1042         char *s;
1043         int n, o;
1044
1045         /* Allocate an ifnet and set it up */
1046         ifp = if_alloc(IFT_ETHER);
1047         if (ifp == NULL) {
1048                 device_printf(dev, "Cannot allocate ifnet\n");
1049                 return (ENOMEM);
1050         }
1051         pi->ifp = ifp;
1052         ifp->if_softc = pi;
1053
1054         callout_init(&pi->tick, CALLOUT_MPSAFE);
1055
1056         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1057         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1058
1059         ifp->if_init = cxgbe_init;
1060         ifp->if_ioctl = cxgbe_ioctl;
1061         ifp->if_transmit = cxgbe_transmit;
1062         ifp->if_qflush = cxgbe_qflush;
1063
1064         ifp->if_capabilities = T4_CAP;
1065 #ifdef TCP_OFFLOAD
1066         if (is_offload(pi->adapter))
1067                 ifp->if_capabilities |= IFCAP_TOE;
1068 #endif
1069         ifp->if_capenable = T4_CAP_ENABLE;
1070         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1071             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1072
1073         ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1074         ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS;
1075         ifp->if_hw_tsomaxsegsize = 65536;
1076
1077         /* Initialize ifmedia for this port */
1078         ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1079             cxgbe_media_status);
1080         build_medialist(pi, &pi->media);
1081
1082         pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1083             EVENTHANDLER_PRI_ANY);
1084
1085         ether_ifattach(ifp, pi->hw_addr);
1086
1087         n = 128;
1088         s = malloc(n, M_CXGBE, M_WAITOK);
1089         o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq);
1090         MPASS(n > o);
1091 #ifdef TCP_OFFLOAD
1092         if (is_offload(pi->adapter)) {
1093                 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)",
1094                     pi->nofldtxq, pi->nofldrxq);
1095                 MPASS(n > o);
1096         }
1097 #endif
1098 #ifdef DEV_NETMAP
1099         o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq,
1100             pi->nnmrxq);
1101         MPASS(n > o);
1102 #endif
1103         device_printf(dev, "%s\n", s);
1104         free(s, M_CXGBE);
1105
1106 #ifdef DEV_NETMAP
1107         /* nm_media handled here to keep implementation private to this file */
1108         ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change,
1109             cxgbe_media_status);
1110         build_medialist(pi, &pi->nm_media);
1111         create_netmap_ifnet(pi);        /* logs errors it something fails */
1112 #endif
1113         cxgbe_sysctls(pi);
1114
1115         return (0);
1116 }
1117
1118 static int
1119 cxgbe_detach(device_t dev)
1120 {
1121         struct port_info *pi = device_get_softc(dev);
1122         struct adapter *sc = pi->adapter;
1123         struct ifnet *ifp = pi->ifp;
1124
1125         /* Tell if_ioctl and if_init that the port is going away */
1126         ADAPTER_LOCK(sc);
1127         SET_DOOMED(pi);
1128         wakeup(&sc->flags);
1129         while (IS_BUSY(sc))
1130                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
1131         SET_BUSY(sc);
1132 #ifdef INVARIANTS
1133         sc->last_op = "t4detach";
1134         sc->last_op_thr = curthread;
1135 #endif
1136         ADAPTER_UNLOCK(sc);
1137
1138         if (pi->flags & HAS_TRACEQ) {
1139                 sc->traceq = -1;        /* cloner should not create ifnet */
1140                 t4_tracer_port_detach(sc);
1141         }
1142
1143         if (pi->vlan_c)
1144                 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
1145
1146         PORT_LOCK(pi);
1147         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1148         callout_stop(&pi->tick);
1149         PORT_UNLOCK(pi);
1150         callout_drain(&pi->tick);
1151
1152         /* Let detach proceed even if these fail. */
1153         cxgbe_uninit_synchronized(pi);
1154         port_full_uninit(pi);
1155
1156         ifmedia_removeall(&pi->media);
1157         ether_ifdetach(pi->ifp);
1158         if_free(pi->ifp);
1159
1160 #ifdef DEV_NETMAP
1161         /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */
1162         destroy_netmap_ifnet(pi);
1163 #endif
1164
1165         ADAPTER_LOCK(sc);
1166         CLR_BUSY(sc);
1167         wakeup(&sc->flags);
1168         ADAPTER_UNLOCK(sc);
1169
1170         return (0);
1171 }
1172
1173 static void
1174 cxgbe_init(void *arg)
1175 {
1176         struct port_info *pi = arg;
1177         struct adapter *sc = pi->adapter;
1178
1179         if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
1180                 return;
1181         cxgbe_init_synchronized(pi);
1182         end_synchronized_op(sc, 0);
1183 }
1184
1185 static int
1186 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1187 {
1188         int rc = 0, mtu, flags, can_sleep;
1189         struct port_info *pi = ifp->if_softc;
1190         struct adapter *sc = pi->adapter;
1191         struct ifreq *ifr = (struct ifreq *)data;
1192         uint32_t mask;
1193
1194         switch (cmd) {
1195         case SIOCSIFMTU:
1196                 mtu = ifr->ifr_mtu;
1197                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1198                         return (EINVAL);
1199
1200                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
1201                 if (rc)
1202                         return (rc);
1203                 ifp->if_mtu = mtu;
1204                 if (pi->flags & PORT_INIT_DONE) {
1205                         t4_update_fl_bufsize(ifp);
1206                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1207                                 rc = update_mac_settings(ifp, XGMAC_MTU);
1208                 }
1209                 end_synchronized_op(sc, 0);
1210                 break;
1211
1212         case SIOCSIFFLAGS:
1213                 can_sleep = 0;
1214 redo_sifflags:
1215                 rc = begin_synchronized_op(sc, pi,
1216                     can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg");
1217                 if (rc)
1218                         return (rc);
1219
1220                 if (ifp->if_flags & IFF_UP) {
1221                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1222                                 flags = pi->if_flags;
1223                                 if ((ifp->if_flags ^ flags) &
1224                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1225                                         if (can_sleep == 1) {
1226                                                 end_synchronized_op(sc, 0);
1227                                                 can_sleep = 0;
1228                                                 goto redo_sifflags;
1229                                         }
1230                                         rc = update_mac_settings(ifp,
1231                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1232                                 }
1233                         } else {
1234                                 if (can_sleep == 0) {
1235                                         end_synchronized_op(sc, LOCK_HELD);
1236                                         can_sleep = 1;
1237                                         goto redo_sifflags;
1238                                 }
1239                                 rc = cxgbe_init_synchronized(pi);
1240                         }
1241                         pi->if_flags = ifp->if_flags;
1242                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1243                         if (can_sleep == 0) {
1244                                 end_synchronized_op(sc, LOCK_HELD);
1245                                 can_sleep = 1;
1246                                 goto redo_sifflags;
1247                         }
1248                         rc = cxgbe_uninit_synchronized(pi);
1249                 }
1250                 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD);
1251                 break;
1252
1253         case SIOCADDMULTI:
1254         case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1255                 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
1256                 if (rc)
1257                         return (rc);
1258                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1259                         rc = update_mac_settings(ifp, XGMAC_MCADDRS);
1260                 end_synchronized_op(sc, LOCK_HELD);
1261                 break;
1262
1263         case SIOCSIFCAP:
1264                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
1265                 if (rc)
1266                         return (rc);
1267
1268                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1269                 if (mask & IFCAP_TXCSUM) {
1270                         ifp->if_capenable ^= IFCAP_TXCSUM;
1271                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1272
1273                         if (IFCAP_TSO4 & ifp->if_capenable &&
1274                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1275                                 ifp->if_capenable &= ~IFCAP_TSO4;
1276                                 if_printf(ifp,
1277                                     "tso4 disabled due to -txcsum.\n");
1278                         }
1279                 }
1280                 if (mask & IFCAP_TXCSUM_IPV6) {
1281                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1282                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1283
1284                         if (IFCAP_TSO6 & ifp->if_capenable &&
1285                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1286                                 ifp->if_capenable &= ~IFCAP_TSO6;
1287                                 if_printf(ifp,
1288                                     "tso6 disabled due to -txcsum6.\n");
1289                         }
1290                 }
1291                 if (mask & IFCAP_RXCSUM)
1292                         ifp->if_capenable ^= IFCAP_RXCSUM;
1293                 if (mask & IFCAP_RXCSUM_IPV6)
1294                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1295
1296                 /*
1297                  * Note that we leave CSUM_TSO alone (it is always set).  The
1298                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1299                  * sending a TSO request our way, so it's sufficient to toggle
1300                  * IFCAP_TSOx only.
1301                  */
1302                 if (mask & IFCAP_TSO4) {
1303                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1304                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1305                                 if_printf(ifp, "enable txcsum first.\n");
1306                                 rc = EAGAIN;
1307                                 goto fail;
1308                         }
1309                         ifp->if_capenable ^= IFCAP_TSO4;
1310                 }
1311                 if (mask & IFCAP_TSO6) {
1312                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1313                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1314                                 if_printf(ifp, "enable txcsum6 first.\n");
1315                                 rc = EAGAIN;
1316                                 goto fail;
1317                         }
1318                         ifp->if_capenable ^= IFCAP_TSO6;
1319                 }
1320                 if (mask & IFCAP_LRO) {
1321 #if defined(INET) || defined(INET6)
1322                         int i;
1323                         struct sge_rxq *rxq;
1324
1325                         ifp->if_capenable ^= IFCAP_LRO;
1326                         for_each_rxq(pi, i, rxq) {
1327                                 if (ifp->if_capenable & IFCAP_LRO)
1328                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1329                                 else
1330                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1331                         }
1332 #endif
1333                 }
1334 #ifdef TCP_OFFLOAD
1335                 if (mask & IFCAP_TOE) {
1336                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1337
1338                         rc = toe_capability(pi, enable);
1339                         if (rc != 0)
1340                                 goto fail;
1341
1342                         ifp->if_capenable ^= mask;
1343                 }
1344 #endif
1345                 if (mask & IFCAP_VLAN_HWTAGGING) {
1346                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1347                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1348                                 rc = update_mac_settings(ifp, XGMAC_VLANEX);
1349                 }
1350                 if (mask & IFCAP_VLAN_MTU) {
1351                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1352
1353                         /* Need to find out how to disable auto-mtu-inflation */
1354                 }
1355                 if (mask & IFCAP_VLAN_HWTSO)
1356                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1357                 if (mask & IFCAP_VLAN_HWCSUM)
1358                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1359
1360 #ifdef VLAN_CAPABILITIES
1361                 VLAN_CAPABILITIES(ifp);
1362 #endif
1363 fail:
1364                 end_synchronized_op(sc, 0);
1365                 break;
1366
1367         case SIOCSIFMEDIA:
1368         case SIOCGIFMEDIA:
1369                 ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1370                 break;
1371
1372         default:
1373                 rc = ether_ioctl(ifp, cmd, data);
1374         }
1375
1376         return (rc);
1377 }
1378
1379 static int
1380 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1381 {
1382         struct port_info *pi = ifp->if_softc;
1383         struct adapter *sc = pi->adapter;
1384         struct sge_txq *txq = &sc->sge.txq[pi->first_txq];
1385         struct buf_ring *br;
1386         int rc;
1387
1388         M_ASSERTPKTHDR(m);
1389
1390         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1391                 m_freem(m);
1392                 return (ENETDOWN);
1393         }
1394
1395         /* check if flowid is set */
1396         if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
1397                 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq))
1398                     + pi->rsrv_noflowq);
1399         br = txq->br;
1400
1401         if (TXQ_TRYLOCK(txq) == 0) {
1402                 struct sge_eq *eq = &txq->eq;
1403
1404                 /*
1405                  * It is possible that t4_eth_tx finishes up and releases the
1406                  * lock between the TRYLOCK above and the drbr_enqueue here.  We
1407                  * need to make sure that this mbuf doesn't just sit there in
1408                  * the drbr.
1409                  */
1410
1411                 rc = drbr_enqueue(ifp, br, m);
1412                 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 &&
1413                     !(eq->flags & EQ_DOOMED))
1414                         callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq);
1415                 return (rc);
1416         }
1417
1418         /*
1419          * txq->m is the mbuf that is held up due to a temporary shortage of
1420          * resources and it should be put on the wire first.  Then what's in
1421          * drbr and finally the mbuf that was just passed in to us.
1422          *
1423          * Return code should indicate the fate of the mbuf that was passed in
1424          * this time.
1425          */
1426
1427         TXQ_LOCK_ASSERT_OWNED(txq);
1428         if (drbr_needs_enqueue(ifp, br) || txq->m) {
1429
1430                 /* Queued for transmission. */
1431
1432                 rc = drbr_enqueue(ifp, br, m);
1433                 m = txq->m ? txq->m : drbr_dequeue(ifp, br);
1434                 (void) t4_eth_tx(ifp, txq, m);
1435                 TXQ_UNLOCK(txq);
1436                 return (rc);
1437         }
1438
1439         /* Direct transmission. */
1440         rc = t4_eth_tx(ifp, txq, m);
1441         if (rc != 0 && txq->m)
1442                 rc = 0; /* held, will be transmitted soon (hopefully) */
1443
1444         TXQ_UNLOCK(txq);
1445         return (rc);
1446 }
1447
1448 static void
1449 cxgbe_qflush(struct ifnet *ifp)
1450 {
1451         struct port_info *pi = ifp->if_softc;
1452         struct sge_txq *txq;
1453         int i;
1454         struct mbuf *m;
1455
1456         /* queues do not exist if !PORT_INIT_DONE. */
1457         if (pi->flags & PORT_INIT_DONE) {
1458                 for_each_txq(pi, i, txq) {
1459                         TXQ_LOCK(txq);
1460                         m_freem(txq->m);
1461                         txq->m = NULL;
1462                         while ((m = buf_ring_dequeue_sc(txq->br)) != NULL)
1463                                 m_freem(m);
1464                         TXQ_UNLOCK(txq);
1465                 }
1466         }
1467         if_qflush(ifp);
1468 }
1469
1470 static int
1471 cxgbe_media_change(struct ifnet *ifp)
1472 {
1473         struct port_info *pi = ifp->if_softc;
1474
1475         device_printf(pi->dev, "%s unimplemented.\n", __func__);
1476
1477         return (EOPNOTSUPP);
1478 }
1479
1480 static void
1481 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1482 {
1483         struct port_info *pi = ifp->if_softc;
1484         struct ifmedia *media = NULL;
1485         struct ifmedia_entry *cur;
1486         int speed = pi->link_cfg.speed;
1487 #ifdef INVARIANTS
1488         int data = (pi->port_type << 8) | pi->mod_type;
1489 #endif
1490
1491         if (ifp == pi->ifp)
1492                 media = &pi->media;
1493 #ifdef DEV_NETMAP
1494         else if (ifp == pi->nm_ifp)
1495                 media = &pi->nm_media;
1496 #endif
1497         MPASS(media != NULL);
1498
1499         cur = media->ifm_cur;
1500         MPASS(cur->ifm_data == data);
1501
1502         ifmr->ifm_status = IFM_AVALID;
1503         if (!pi->link_cfg.link_ok)
1504                 return;
1505
1506         ifmr->ifm_status |= IFM_ACTIVE;
1507
1508         /* active and current will differ iff current media is autoselect. */
1509         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1510                 return;
1511
1512         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1513         if (speed == SPEED_10000)
1514                 ifmr->ifm_active |= IFM_10G_T;
1515         else if (speed == SPEED_1000)
1516                 ifmr->ifm_active |= IFM_1000_T;
1517         else if (speed == SPEED_100)
1518                 ifmr->ifm_active |= IFM_100_TX;
1519         else if (speed == SPEED_10)
1520                 ifmr->ifm_active |= IFM_10_T;
1521         else
1522                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1523                             speed));
1524 }
1525
1526 void
1527 t4_fatal_err(struct adapter *sc)
1528 {
1529         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1530         t4_intr_disable(sc);
1531         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1532             device_get_nameunit(sc->dev));
1533 }
1534
1535 static int
1536 map_bars_0_and_4(struct adapter *sc)
1537 {
1538         sc->regs_rid = PCIR_BAR(0);
1539         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1540             &sc->regs_rid, RF_ACTIVE);
1541         if (sc->regs_res == NULL) {
1542                 device_printf(sc->dev, "cannot map registers.\n");
1543                 return (ENXIO);
1544         }
1545         sc->bt = rman_get_bustag(sc->regs_res);
1546         sc->bh = rman_get_bushandle(sc->regs_res);
1547         sc->mmio_len = rman_get_size(sc->regs_res);
1548         setbit(&sc->doorbells, DOORBELL_KDB);
1549
1550         sc->msix_rid = PCIR_BAR(4);
1551         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1552             &sc->msix_rid, RF_ACTIVE);
1553         if (sc->msix_res == NULL) {
1554                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1555                 return (ENXIO);
1556         }
1557
1558         return (0);
1559 }
1560
1561 static int
1562 map_bar_2(struct adapter *sc)
1563 {
1564
1565         /*
1566          * T4: only iWARP driver uses the userspace doorbells.  There is no need
1567          * to map it if RDMA is disabled.
1568          */
1569         if (is_t4(sc) && sc->rdmacaps == 0)
1570                 return (0);
1571
1572         sc->udbs_rid = PCIR_BAR(2);
1573         sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1574             &sc->udbs_rid, RF_ACTIVE);
1575         if (sc->udbs_res == NULL) {
1576                 device_printf(sc->dev, "cannot map doorbell BAR.\n");
1577                 return (ENXIO);
1578         }
1579         sc->udbs_base = rman_get_virtual(sc->udbs_res);
1580
1581         if (is_t5(sc)) {
1582                 setbit(&sc->doorbells, DOORBELL_UDB);
1583 #if defined(__i386__) || defined(__amd64__)
1584                 if (t5_write_combine) {
1585                         int rc;
1586
1587                         /*
1588                          * Enable write combining on BAR2.  This is the
1589                          * userspace doorbell BAR and is split into 128B
1590                          * (UDBS_SEG_SIZE) doorbell regions, each associated
1591                          * with an egress queue.  The first 64B has the doorbell
1592                          * and the second 64B can be used to submit a tx work
1593                          * request with an implicit doorbell.
1594                          */
1595
1596                         rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
1597                             rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
1598                         if (rc == 0) {
1599                                 clrbit(&sc->doorbells, DOORBELL_UDB);
1600                                 setbit(&sc->doorbells, DOORBELL_WCWR);
1601                                 setbit(&sc->doorbells, DOORBELL_UDBWC);
1602                         } else {
1603                                 device_printf(sc->dev,
1604                                     "couldn't enable write combining: %d\n",
1605                                     rc);
1606                         }
1607
1608                         t4_write_reg(sc, A_SGE_STAT_CFG,
1609                             V_STATSOURCE_T5(7) | V_STATMODE(0));
1610                 }
1611 #endif
1612         }
1613
1614         return (0);
1615 }
1616
1617 static const struct memwin t4_memwin[] = {
1618         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1619         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1620         { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
1621 };
1622
1623 static const struct memwin t5_memwin[] = {
1624         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1625         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1626         { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1627 };
1628
1629 static void
1630 setup_memwin(struct adapter *sc)
1631 {
1632         const struct memwin *mw;
1633         int i, n;
1634         uint32_t bar0;
1635
1636         if (is_t4(sc)) {
1637                 /*
1638                  * Read low 32b of bar0 indirectly via the hardware backdoor
1639                  * mechanism.  Works from within PCI passthrough environments
1640                  * too, where rman_get_start() can return a different value.  We
1641                  * need to program the T4 memory window decoders with the actual
1642                  * addresses that will be coming across the PCIe link.
1643                  */
1644                 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1645                 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1646
1647                 mw = &t4_memwin[0];
1648                 n = nitems(t4_memwin);
1649         } else {
1650                 /* T5 uses the relative offset inside the PCIe BAR */
1651                 bar0 = 0;
1652
1653                 mw = &t5_memwin[0];
1654                 n = nitems(t5_memwin);
1655         }
1656
1657         for (i = 0; i < n; i++, mw++) {
1658                 t4_write_reg(sc,
1659                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
1660                     (mw->base + bar0) | V_BIR(0) |
1661                     V_WINDOW(ilog2(mw->aperture) - 10));
1662         }
1663
1664         /* flush */
1665         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1666 }
1667
1668 /*
1669  * Verify that the memory range specified by the addr/len pair is valid and lies
1670  * entirely within a single region (EDCx or MCx).
1671  */
1672 static int
1673 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
1674 {
1675         uint32_t em, addr_len, maddr, mlen;
1676
1677         /* Memory can only be accessed in naturally aligned 4 byte units */
1678         if (addr & 3 || len & 3 || len == 0)
1679                 return (EINVAL);
1680
1681         /* Enabled memories */
1682         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1683         if (em & F_EDRAM0_ENABLE) {
1684                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1685                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1686                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1687                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1688                     addr + len <= maddr + mlen)
1689                         return (0);
1690         }
1691         if (em & F_EDRAM1_ENABLE) {
1692                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1693                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1694                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1695                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1696                     addr + len <= maddr + mlen)
1697                         return (0);
1698         }
1699         if (em & F_EXT_MEM_ENABLE) {
1700                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1701                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1702                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1703                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1704                     addr + len <= maddr + mlen)
1705                         return (0);
1706         }
1707         if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
1708                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1709                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1710                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1711                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1712                     addr + len <= maddr + mlen)
1713                         return (0);
1714         }
1715
1716         return (EFAULT);
1717 }
1718
1719 static int
1720 fwmtype_to_hwmtype(int mtype)
1721 {
1722
1723         switch (mtype) {
1724         case FW_MEMTYPE_EDC0:
1725                 return (MEM_EDC0);
1726         case FW_MEMTYPE_EDC1:
1727                 return (MEM_EDC1);
1728         case FW_MEMTYPE_EXTMEM:
1729                 return (MEM_MC0);
1730         case FW_MEMTYPE_EXTMEM1:
1731                 return (MEM_MC1);
1732         default:
1733                 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
1734         }
1735 }
1736
1737 /*
1738  * Verify that the memory range specified by the memtype/offset/len pair is
1739  * valid and lies entirely within the memtype specified.  The global address of
1740  * the start of the range is returned in addr.
1741  */
1742 static int
1743 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1744     uint32_t *addr)
1745 {
1746         uint32_t em, addr_len, maddr, mlen;
1747
1748         /* Memory can only be accessed in naturally aligned 4 byte units */
1749         if (off & 3 || len & 3 || len == 0)
1750                 return (EINVAL);
1751
1752         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1753         switch (fwmtype_to_hwmtype(mtype)) {
1754         case MEM_EDC0:
1755                 if (!(em & F_EDRAM0_ENABLE))
1756                         return (EINVAL);
1757                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1758                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1759                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1760                 break;
1761         case MEM_EDC1:
1762                 if (!(em & F_EDRAM1_ENABLE))
1763                         return (EINVAL);
1764                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1765                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1766                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1767                 break;
1768         case MEM_MC:
1769                 if (!(em & F_EXT_MEM_ENABLE))
1770                         return (EINVAL);
1771                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1772                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1773                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1774                 break;
1775         case MEM_MC1:
1776                 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
1777                         return (EINVAL);
1778                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1779                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1780                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1781                 break;
1782         default:
1783                 return (EINVAL);
1784         }
1785
1786         if (mlen > 0 && off < mlen && off + len <= mlen) {
1787                 *addr = maddr + off;    /* global address */
1788                 return (0);
1789         }
1790
1791         return (EFAULT);
1792 }
1793
1794 static void
1795 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1796 {
1797         const struct memwin *mw;
1798
1799         if (is_t4(sc)) {
1800                 KASSERT(win >= 0 && win < nitems(t4_memwin),
1801                     ("%s: incorrect memwin# (%d)", __func__, win));
1802                 mw = &t4_memwin[win];
1803         } else {
1804                 KASSERT(win >= 0 && win < nitems(t5_memwin),
1805                     ("%s: incorrect memwin# (%d)", __func__, win));
1806                 mw = &t5_memwin[win];
1807         }
1808
1809         if (base != NULL)
1810                 *base = mw->base;
1811         if (aperture != NULL)
1812                 *aperture = mw->aperture;
1813 }
1814
1815 /*
1816  * Positions the memory window such that it can be used to access the specified
1817  * address in the chip's address space.  The return value is the offset of addr
1818  * from the start of the window.
1819  */
1820 static uint32_t
1821 position_memwin(struct adapter *sc, int n, uint32_t addr)
1822 {
1823         uint32_t start, pf;
1824         uint32_t reg;
1825
1826         KASSERT(n >= 0 && n <= 3,
1827             ("%s: invalid window %d.", __func__, n));
1828         KASSERT((addr & 3) == 0,
1829             ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
1830
1831         if (is_t4(sc)) {
1832                 pf = 0;
1833                 start = addr & ~0xf;    /* start must be 16B aligned */
1834         } else {
1835                 pf = V_PFNUM(sc->pf);
1836                 start = addr & ~0x7f;   /* start must be 128B aligned */
1837         }
1838         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1839
1840         t4_write_reg(sc, reg, start | pf);
1841         t4_read_reg(sc, reg);
1842
1843         return (addr - start);
1844 }
1845
1846 static int
1847 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1848     struct intrs_and_queues *iaq)
1849 {
1850         int rc, itype, navail, nrxq10g, nrxq1g, n;
1851         int nofldrxq10g = 0, nofldrxq1g = 0;
1852         int nnmrxq10g = 0, nnmrxq1g = 0;
1853
1854         bzero(iaq, sizeof(*iaq));
1855
1856         iaq->ntxq10g = t4_ntxq10g;
1857         iaq->ntxq1g = t4_ntxq1g;
1858         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1859         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1860         iaq->rsrv_noflowq = t4_rsrv_noflowq;
1861 #ifdef TCP_OFFLOAD
1862         if (is_offload(sc)) {
1863                 iaq->nofldtxq10g = t4_nofldtxq10g;
1864                 iaq->nofldtxq1g = t4_nofldtxq1g;
1865                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1866                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1867         }
1868 #endif
1869 #ifdef DEV_NETMAP
1870         iaq->nnmtxq10g = t4_nnmtxq10g;
1871         iaq->nnmtxq1g = t4_nnmtxq1g;
1872         iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g;
1873         iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g;
1874 #endif
1875
1876         for (itype = INTR_MSIX; itype; itype >>= 1) {
1877
1878                 if ((itype & t4_intr_types) == 0)
1879                         continue;       /* not allowed */
1880
1881                 if (itype == INTR_MSIX)
1882                         navail = pci_msix_count(sc->dev);
1883                 else if (itype == INTR_MSI)
1884                         navail = pci_msi_count(sc->dev);
1885                 else
1886                         navail = 1;
1887 restart:
1888                 if (navail == 0)
1889                         continue;
1890
1891                 iaq->intr_type = itype;
1892                 iaq->intr_flags_10g = 0;
1893                 iaq->intr_flags_1g = 0;
1894
1895                 /*
1896                  * Best option: an interrupt vector for errors, one for the
1897                  * firmware event queue, and one for every rxq (NIC, TOE, and
1898                  * netmap).
1899                  */
1900                 iaq->nirq = T4_EXTRA_INTR;
1901                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g);
1902                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g);
1903                 if (iaq->nirq <= navail &&
1904                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
1905                         iaq->intr_flags_10g = INTR_ALL;
1906                         iaq->intr_flags_1g = INTR_ALL;
1907                         goto allocate;
1908                 }
1909
1910                 /*
1911                  * Second best option: a vector for errors, one for the firmware
1912                  * event queue, and vectors for either all the NIC rx queues or
1913                  * all the TOE rx queues.  The queues that don't get vectors
1914                  * will forward their interrupts to those that do.
1915                  *
1916                  * Note: netmap rx queues cannot be created early and so they
1917                  * can't be setup to receive forwarded interrupts for others.
1918                  */
1919                 iaq->nirq = T4_EXTRA_INTR;
1920                 if (nrxq10g >= nofldrxq10g) {
1921                         iaq->intr_flags_10g = INTR_RXQ;
1922                         iaq->nirq += n10g * nrxq10g;
1923 #ifdef DEV_NETMAP
1924                         iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g);
1925 #endif
1926                 } else {
1927                         iaq->intr_flags_10g = INTR_OFLD_RXQ;
1928                         iaq->nirq += n10g * nofldrxq10g;
1929 #ifdef DEV_NETMAP
1930                         iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g);
1931 #endif
1932                 }
1933                 if (nrxq1g >= nofldrxq1g) {
1934                         iaq->intr_flags_1g = INTR_RXQ;
1935                         iaq->nirq += n1g * nrxq1g;
1936 #ifdef DEV_NETMAP
1937                         iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g);
1938 #endif
1939                 } else {
1940                         iaq->intr_flags_1g = INTR_OFLD_RXQ;
1941                         iaq->nirq += n1g * nofldrxq1g;
1942 #ifdef DEV_NETMAP
1943                         iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g);
1944 #endif
1945                 }
1946                 if (iaq->nirq <= navail &&
1947                     (itype != INTR_MSI || powerof2(iaq->nirq)))
1948                         goto allocate;
1949
1950                 /*
1951                  * Next best option: an interrupt vector for errors, one for the
1952                  * firmware event queue, and at least one per port.  At this
1953                  * point we know we'll have to downsize nrxq and/or nofldrxq
1954                  * and/or nnmrxq to fit what's available to us.
1955                  */
1956                 iaq->nirq = T4_EXTRA_INTR;
1957                 iaq->nirq += n10g + n1g;
1958                 if (iaq->nirq <= navail) {
1959                         int leftover = navail - iaq->nirq;
1960
1961                         if (n10g > 0) {
1962                                 int target = max(nrxq10g, nofldrxq10g);
1963
1964                                 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
1965                                     INTR_RXQ : INTR_OFLD_RXQ;
1966
1967                                 n = 1;
1968                                 while (n < target && leftover >= n10g) {
1969                                         leftover -= n10g;
1970                                         iaq->nirq += n10g;
1971                                         n++;
1972                                 }
1973                                 iaq->nrxq10g = min(n, nrxq10g);
1974 #ifdef TCP_OFFLOAD
1975                                 iaq->nofldrxq10g = min(n, nofldrxq10g);
1976 #endif
1977 #ifdef DEV_NETMAP
1978                                 iaq->nnmrxq10g = min(n, nnmrxq10g);
1979 #endif
1980                         }
1981
1982                         if (n1g > 0) {
1983                                 int target = max(nrxq1g, nofldrxq1g);
1984
1985                                 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
1986                                     INTR_RXQ : INTR_OFLD_RXQ;
1987
1988                                 n = 1;
1989                                 while (n < target && leftover >= n1g) {
1990                                         leftover -= n1g;
1991                                         iaq->nirq += n1g;
1992                                         n++;
1993                                 }
1994                                 iaq->nrxq1g = min(n, nrxq1g);
1995 #ifdef TCP_OFFLOAD
1996                                 iaq->nofldrxq1g = min(n, nofldrxq1g);
1997 #endif
1998 #ifdef DEV_NETMAP
1999                                 iaq->nnmrxq1g = min(n, nnmrxq1g);
2000 #endif
2001                         }
2002
2003                         if (itype != INTR_MSI || powerof2(iaq->nirq))
2004                                 goto allocate;
2005                 }
2006
2007                 /*
2008                  * Least desirable option: one interrupt vector for everything.
2009                  */
2010                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
2011                 iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
2012 #ifdef TCP_OFFLOAD
2013                 if (is_offload(sc))
2014                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2015 #endif
2016 #ifdef DEV_NETMAP
2017                 iaq->nnmrxq10g = iaq->nnmrxq1g = 1;
2018 #endif
2019
2020 allocate:
2021                 navail = iaq->nirq;
2022                 rc = 0;
2023                 if (itype == INTR_MSIX)
2024                         rc = pci_alloc_msix(sc->dev, &navail);
2025                 else if (itype == INTR_MSI)
2026                         rc = pci_alloc_msi(sc->dev, &navail);
2027
2028                 if (rc == 0) {
2029                         if (navail == iaq->nirq)
2030                                 return (0);
2031
2032                         /*
2033                          * Didn't get the number requested.  Use whatever number
2034                          * the kernel is willing to allocate (it's in navail).
2035                          */
2036                         device_printf(sc->dev, "fewer vectors than requested, "
2037                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
2038                             itype, iaq->nirq, navail);
2039                         pci_release_msi(sc->dev);
2040                         goto restart;
2041                 }
2042
2043                 device_printf(sc->dev,
2044                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
2045                     itype, rc, iaq->nirq, navail);
2046         }
2047
2048         device_printf(sc->dev,
2049             "failed to find a usable interrupt type.  "
2050             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
2051             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
2052
2053         return (ENXIO);
2054 }
2055
2056 #define FW_VERSION(chip) ( \
2057     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
2058     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
2059     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
2060     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
2061 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
2062
2063 struct fw_info {
2064         uint8_t chip;
2065         char *kld_name;
2066         char *fw_mod_name;
2067         struct fw_hdr fw_hdr;   /* XXX: waste of space, need a sparse struct */
2068 } fw_info[] = {
2069         {
2070                 .chip = CHELSIO_T4,
2071                 .kld_name = "t4fw_cfg",
2072                 .fw_mod_name = "t4fw",
2073                 .fw_hdr = {
2074                         .chip = FW_HDR_CHIP_T4,
2075                         .fw_ver = htobe32_const(FW_VERSION(T4)),
2076                         .intfver_nic = FW_INTFVER(T4, NIC),
2077                         .intfver_vnic = FW_INTFVER(T4, VNIC),
2078                         .intfver_ofld = FW_INTFVER(T4, OFLD),
2079                         .intfver_ri = FW_INTFVER(T4, RI),
2080                         .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
2081                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
2082                         .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
2083                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
2084                 },
2085         }, {
2086                 .chip = CHELSIO_T5,
2087                 .kld_name = "t5fw_cfg",
2088                 .fw_mod_name = "t5fw",
2089                 .fw_hdr = {
2090                         .chip = FW_HDR_CHIP_T5,
2091                         .fw_ver = htobe32_const(FW_VERSION(T5)),
2092                         .intfver_nic = FW_INTFVER(T5, NIC),
2093                         .intfver_vnic = FW_INTFVER(T5, VNIC),
2094                         .intfver_ofld = FW_INTFVER(T5, OFLD),
2095                         .intfver_ri = FW_INTFVER(T5, RI),
2096                         .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
2097                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
2098                         .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
2099                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
2100                 },
2101         }
2102 };
2103
2104 static struct fw_info *
2105 find_fw_info(int chip)
2106 {
2107         int i;
2108
2109         for (i = 0; i < nitems(fw_info); i++) {
2110                 if (fw_info[i].chip == chip)
2111                         return (&fw_info[i]);
2112         }
2113         return (NULL);
2114 }
2115
2116 /*
2117  * Is the given firmware API compatible with the one the driver was compiled
2118  * with?
2119  */
2120 static int
2121 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
2122 {
2123
2124         /* short circuit if it's the exact same firmware version */
2125         if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
2126                 return (1);
2127
2128         /*
2129          * XXX: Is this too conservative?  Perhaps I should limit this to the
2130          * features that are supported in the driver.
2131          */
2132 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
2133         if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
2134             SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
2135             SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
2136                 return (1);
2137 #undef SAME_INTF
2138
2139         return (0);
2140 }
2141
2142 /*
2143  * The firmware in the KLD is usable, but should it be installed?  This routine
2144  * explains itself in detail if it indicates the KLD firmware should be
2145  * installed.
2146  */
2147 static int
2148 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
2149 {
2150         const char *reason;
2151
2152         if (!card_fw_usable) {
2153                 reason = "incompatible or unusable";
2154                 goto install;
2155         }
2156
2157         if (k > c) {
2158                 reason = "older than the version bundled with this driver";
2159                 goto install;
2160         }
2161
2162         if (t4_fw_install == 2 && k != c) {
2163                 reason = "different than the version bundled with this driver";
2164                 goto install;
2165         }
2166
2167         return (0);
2168
2169 install:
2170         if (t4_fw_install == 0) {
2171                 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2172                     "but the driver is prohibited from installing a different "
2173                     "firmware on the card.\n",
2174                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2175                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2176
2177                 return (0);
2178         }
2179
2180         device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2181             "installing firmware %u.%u.%u.%u on card.\n",
2182             G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2183             G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2184             G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2185             G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2186
2187         return (1);
2188 }
2189 /*
2190  * Establish contact with the firmware and determine if we are the master driver
2191  * or not, and whether we are responsible for chip initialization.
2192  */
2193 static int
2194 prep_firmware(struct adapter *sc)
2195 {
2196         const struct firmware *fw = NULL, *default_cfg;
2197         int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2198         enum dev_state state;
2199         struct fw_info *fw_info;
2200         struct fw_hdr *card_fw;         /* fw on the card */
2201         const struct fw_hdr *kld_fw;    /* fw in the KLD */
2202         const struct fw_hdr *drv_fw;    /* fw header the driver was compiled
2203                                            against */
2204
2205         /* Contact firmware. */
2206         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2207         if (rc < 0 || state == DEV_STATE_ERR) {
2208                 rc = -rc;
2209                 device_printf(sc->dev,
2210                     "failed to connect to the firmware: %d, %d.\n", rc, state);
2211                 return (rc);
2212         }
2213         pf = rc;
2214         if (pf == sc->mbox)
2215                 sc->flags |= MASTER_PF;
2216         else if (state == DEV_STATE_UNINIT) {
2217                 /*
2218                  * We didn't get to be the master so we definitely won't be
2219                  * configuring the chip.  It's a bug if someone else hasn't
2220                  * configured it already.
2221                  */
2222                 device_printf(sc->dev, "couldn't be master(%d), "
2223                     "device not already initialized either(%d).\n", rc, state);
2224                 return (EDOOFUS);
2225         }
2226
2227         /* This is the firmware whose headers the driver was compiled against */
2228         fw_info = find_fw_info(chip_id(sc));
2229         if (fw_info == NULL) {
2230                 device_printf(sc->dev,
2231                     "unable to look up firmware information for chip %d.\n",
2232                     chip_id(sc));
2233                 return (EINVAL);
2234         }
2235         drv_fw = &fw_info->fw_hdr;
2236
2237         /*
2238          * The firmware KLD contains many modules.  The KLD name is also the
2239          * name of the module that contains the default config file.
2240          */
2241         default_cfg = firmware_get(fw_info->kld_name);
2242
2243         /* Read the header of the firmware on the card */
2244         card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2245         rc = -t4_read_flash(sc, FLASH_FW_START,
2246             sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2247         if (rc == 0)
2248                 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2249         else {
2250                 device_printf(sc->dev,
2251                     "Unable to read card's firmware header: %d\n", rc);
2252                 card_fw_usable = 0;
2253         }
2254
2255         /* This is the firmware in the KLD */
2256         fw = firmware_get(fw_info->fw_mod_name);
2257         if (fw != NULL) {
2258                 kld_fw = (const void *)fw->data;
2259                 kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2260         } else {
2261                 kld_fw = NULL;
2262                 kld_fw_usable = 0;
2263         }
2264
2265         if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2266             (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2267                 /*
2268                  * Common case: the firmware on the card is an exact match and
2269                  * the KLD is an exact match too, or the KLD is
2270                  * absent/incompatible.  Note that t4_fw_install = 2 is ignored
2271                  * here -- use cxgbetool loadfw if you want to reinstall the
2272                  * same firmware as the one on the card.
2273                  */
2274         } else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2275             should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2276             be32toh(card_fw->fw_ver))) {
2277
2278                 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
2279                 if (rc != 0) {
2280                         device_printf(sc->dev,
2281                             "failed to install firmware: %d\n", rc);
2282                         goto done;
2283                 }
2284
2285                 /* Installed successfully, update the cached header too. */
2286                 memcpy(card_fw, kld_fw, sizeof(*card_fw));
2287                 card_fw_usable = 1;
2288                 need_fw_reset = 0;      /* already reset as part of load_fw */
2289         }
2290
2291         if (!card_fw_usable) {
2292                 uint32_t d, c, k;
2293
2294                 d = ntohl(drv_fw->fw_ver);
2295                 c = ntohl(card_fw->fw_ver);
2296                 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
2297
2298                 device_printf(sc->dev, "Cannot find a usable firmware: "
2299                     "fw_install %d, chip state %d, "
2300                     "driver compiled with %d.%d.%d.%d, "
2301                     "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
2302                     t4_fw_install, state,
2303                     G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
2304                     G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
2305                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2306                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
2307                     G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2308                     G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2309                 rc = EINVAL;
2310                 goto done;
2311         }
2312
2313         /* We're using whatever's on the card and it's known to be good. */
2314         sc->params.fw_vers = ntohl(card_fw->fw_ver);
2315         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
2316             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2317             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2318             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2319             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2320         t4_get_tp_version(sc, &sc->params.tp_vers);
2321
2322         /* Reset device */
2323         if (need_fw_reset &&
2324             (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
2325                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
2326                 if (rc != ETIMEDOUT && rc != EIO)
2327                         t4_fw_bye(sc, sc->mbox);
2328                 goto done;
2329         }
2330         sc->flags |= FW_OK;
2331
2332         rc = get_params__pre_init(sc);
2333         if (rc != 0)
2334                 goto done; /* error message displayed already */
2335
2336         /* Partition adapter resources as specified in the config file. */
2337         if (state == DEV_STATE_UNINIT) {
2338
2339                 KASSERT(sc->flags & MASTER_PF,
2340                     ("%s: trying to change chip settings when not master.",
2341                     __func__));
2342
2343                 rc = partition_resources(sc, default_cfg, fw_info->kld_name);
2344                 if (rc != 0)
2345                         goto done;      /* error message displayed already */
2346
2347                 t4_tweak_chip_settings(sc);
2348
2349                 /* get basic stuff going */
2350                 rc = -t4_fw_initialize(sc, sc->mbox);
2351                 if (rc != 0) {
2352                         device_printf(sc->dev, "fw init failed: %d.\n", rc);
2353                         goto done;
2354                 }
2355         } else {
2356                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
2357                 sc->cfcsum = 0;
2358         }
2359
2360 done:
2361         free(card_fw, M_CXGBE);
2362         if (fw != NULL)
2363                 firmware_put(fw, FIRMWARE_UNLOAD);
2364         if (default_cfg != NULL)
2365                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
2366
2367         return (rc);
2368 }
2369
2370 #define FW_PARAM_DEV(param) \
2371         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2372          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2373 #define FW_PARAM_PFVF(param) \
2374         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2375          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2376
2377 /*
2378  * Partition chip resources for use between various PFs, VFs, etc.
2379  */
2380 static int
2381 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
2382     const char *name_prefix)
2383 {
2384         const struct firmware *cfg = NULL;
2385         int rc = 0;
2386         struct fw_caps_config_cmd caps;
2387         uint32_t mtype, moff, finicsum, cfcsum;
2388
2389         /*
2390          * Figure out what configuration file to use.  Pick the default config
2391          * file for the card if the user hasn't specified one explicitly.
2392          */
2393         snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
2394         if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
2395                 /* Card specific overrides go here. */
2396                 if (pci_get_device(sc->dev) == 0x440a)
2397                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
2398                 if (is_fpga(sc))
2399                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
2400         }
2401
2402         /*
2403          * We need to load another module if the profile is anything except
2404          * "default" or "flash".
2405          */
2406         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
2407             strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2408                 char s[32];
2409
2410                 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
2411                 cfg = firmware_get(s);
2412                 if (cfg == NULL) {
2413                         if (default_cfg != NULL) {
2414                                 device_printf(sc->dev,
2415                                     "unable to load module \"%s\" for "
2416                                     "configuration profile \"%s\", will use "
2417                                     "the default config file instead.\n",
2418                                     s, sc->cfg_file);
2419                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2420                                     "%s", DEFAULT_CF);
2421                         } else {
2422                                 device_printf(sc->dev,
2423                                     "unable to load module \"%s\" for "
2424                                     "configuration profile \"%s\", will use "
2425                                     "the config file on the card's flash "
2426                                     "instead.\n", s, sc->cfg_file);
2427                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2428                                     "%s", FLASH_CF);
2429                         }
2430                 }
2431         }
2432
2433         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
2434             default_cfg == NULL) {
2435                 device_printf(sc->dev,
2436                     "default config file not available, will use the config "
2437                     "file on the card's flash instead.\n");
2438                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
2439         }
2440
2441         if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2442                 u_int cflen, i, n;
2443                 const uint32_t *cfdata;
2444                 uint32_t param, val, addr, off, mw_base, mw_aperture;
2445
2446                 KASSERT(cfg != NULL || default_cfg != NULL,
2447                     ("%s: no config to upload", __func__));
2448
2449                 /*
2450                  * Ask the firmware where it wants us to upload the config file.
2451                  */
2452                 param = FW_PARAM_DEV(CF);
2453                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2454                 if (rc != 0) {
2455                         /* No support for config file?  Shouldn't happen. */
2456                         device_printf(sc->dev,
2457                             "failed to query config file location: %d.\n", rc);
2458                         goto done;
2459                 }
2460                 mtype = G_FW_PARAMS_PARAM_Y(val);
2461                 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
2462
2463                 /*
2464                  * XXX: sheer laziness.  We deliberately added 4 bytes of
2465                  * useless stuffing/comments at the end of the config file so
2466                  * it's ok to simply throw away the last remaining bytes when
2467                  * the config file is not an exact multiple of 4.  This also
2468                  * helps with the validate_mt_off_len check.
2469                  */
2470                 if (cfg != NULL) {
2471                         cflen = cfg->datasize & ~3;
2472                         cfdata = cfg->data;
2473                 } else {
2474                         cflen = default_cfg->datasize & ~3;
2475                         cfdata = default_cfg->data;
2476                 }
2477
2478                 if (cflen > FLASH_CFG_MAX_SIZE) {
2479                         device_printf(sc->dev,
2480                             "config file too long (%d, max allowed is %d).  "
2481                             "Will try to use the config on the card, if any.\n",
2482                             cflen, FLASH_CFG_MAX_SIZE);
2483                         goto use_config_on_flash;
2484                 }
2485
2486                 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
2487                 if (rc != 0) {
2488                         device_printf(sc->dev,
2489                             "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
2490                             "Will try to use the config on the card, if any.\n",
2491                             __func__, mtype, moff, cflen, rc);
2492                         goto use_config_on_flash;
2493                 }
2494
2495                 memwin_info(sc, 2, &mw_base, &mw_aperture);
2496                 while (cflen) {
2497                         off = position_memwin(sc, 2, addr);
2498                         n = min(cflen, mw_aperture - off);
2499                         for (i = 0; i < n; i += 4)
2500                                 t4_write_reg(sc, mw_base + off + i, *cfdata++);
2501                         cflen -= n;
2502                         addr += n;
2503                 }
2504         } else {
2505 use_config_on_flash:
2506                 mtype = FW_MEMTYPE_FLASH;
2507                 moff = t4_flash_cfg_addr(sc);
2508         }
2509
2510         bzero(&caps, sizeof(caps));
2511         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2512             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2513         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
2514             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
2515             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
2516         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2517         if (rc != 0) {
2518                 device_printf(sc->dev,
2519                     "failed to pre-process config file: %d "
2520                     "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
2521                 goto done;
2522         }
2523
2524         finicsum = be32toh(caps.finicsum);
2525         cfcsum = be32toh(caps.cfcsum);
2526         if (finicsum != cfcsum) {
2527                 device_printf(sc->dev,
2528                     "WARNING: config file checksum mismatch: %08x %08x\n",
2529                     finicsum, cfcsum);
2530         }
2531         sc->cfcsum = cfcsum;
2532
2533 #define LIMIT_CAPS(x) do { \
2534         caps.x &= htobe16(t4_##x##_allowed); \
2535 } while (0)
2536
2537         /*
2538          * Let the firmware know what features will (not) be used so it can tune
2539          * things accordingly.
2540          */
2541         LIMIT_CAPS(linkcaps);
2542         LIMIT_CAPS(niccaps);
2543         LIMIT_CAPS(toecaps);
2544         LIMIT_CAPS(rdmacaps);
2545         LIMIT_CAPS(iscsicaps);
2546         LIMIT_CAPS(fcoecaps);
2547 #undef LIMIT_CAPS
2548
2549         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2550             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
2551         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2552         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
2553         if (rc != 0) {
2554                 device_printf(sc->dev,
2555                     "failed to process config file: %d.\n", rc);
2556         }
2557 done:
2558         if (cfg != NULL)
2559                 firmware_put(cfg, FIRMWARE_UNLOAD);
2560         return (rc);
2561 }
2562
2563 /*
2564  * Retrieve parameters that are needed (or nice to have) very early.
2565  */
2566 static int
2567 get_params__pre_init(struct adapter *sc)
2568 {
2569         int rc;
2570         uint32_t param[2], val[2];
2571         struct fw_devlog_cmd cmd;
2572         struct devlog_params *dlog = &sc->params.devlog;
2573
2574         param[0] = FW_PARAM_DEV(PORTVEC);
2575         param[1] = FW_PARAM_DEV(CCLK);
2576         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2577         if (rc != 0) {
2578                 device_printf(sc->dev,
2579                     "failed to query parameters (pre_init): %d.\n", rc);
2580                 return (rc);
2581         }
2582
2583         sc->params.portvec = val[0];
2584         sc->params.nports = bitcount32(val[0]);
2585         sc->params.vpd.cclk = val[1];
2586
2587         /* Read device log parameters. */
2588         bzero(&cmd, sizeof(cmd));
2589         cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
2590             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2591         cmd.retval_len16 = htobe32(FW_LEN16(cmd));
2592         rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
2593         if (rc != 0) {
2594                 device_printf(sc->dev,
2595                     "failed to get devlog parameters: %d.\n", rc);
2596                 bzero(dlog, sizeof (*dlog));
2597                 rc = 0; /* devlog isn't critical for device operation */
2598         } else {
2599                 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
2600                 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
2601                 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
2602                 dlog->size = be32toh(cmd.memsize_devlog);
2603         }
2604
2605         return (rc);
2606 }
2607
2608 /*
2609  * Retrieve various parameters that are of interest to the driver.  The device
2610  * has been initialized by the firmware at this point.
2611  */
2612 static int
2613 get_params__post_init(struct adapter *sc)
2614 {
2615         int rc;
2616         uint32_t param[7], val[7];
2617         struct fw_caps_config_cmd caps;
2618
2619         param[0] = FW_PARAM_PFVF(IQFLINT_START);
2620         param[1] = FW_PARAM_PFVF(EQ_START);
2621         param[2] = FW_PARAM_PFVF(FILTER_START);
2622         param[3] = FW_PARAM_PFVF(FILTER_END);
2623         param[4] = FW_PARAM_PFVF(L2T_START);
2624         param[5] = FW_PARAM_PFVF(L2T_END);
2625         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2626         if (rc != 0) {
2627                 device_printf(sc->dev,
2628                     "failed to query parameters (post_init): %d.\n", rc);
2629                 return (rc);
2630         }
2631
2632         sc->sge.iq_start = val[0];
2633         sc->sge.eq_start = val[1];
2634         sc->tids.ftid_base = val[2];
2635         sc->tids.nftids = val[3] - val[2] + 1;
2636         sc->params.ftid_min = val[2];
2637         sc->params.ftid_max = val[3];
2638         sc->vres.l2t.start = val[4];
2639         sc->vres.l2t.size = val[5] - val[4] + 1;
2640         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
2641             ("%s: L2 table size (%u) larger than expected (%u)",
2642             __func__, sc->vres.l2t.size, L2T_SIZE));
2643
2644         /* get capabilites */
2645         bzero(&caps, sizeof(caps));
2646         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2647             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2648         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2649         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2650         if (rc != 0) {
2651                 device_printf(sc->dev,
2652                     "failed to get card capabilities: %d.\n", rc);
2653                 return (rc);
2654         }
2655
2656 #define READ_CAPS(x) do { \
2657         sc->x = htobe16(caps.x); \
2658 } while (0)
2659         READ_CAPS(linkcaps);
2660         READ_CAPS(niccaps);
2661         READ_CAPS(toecaps);
2662         READ_CAPS(rdmacaps);
2663         READ_CAPS(iscsicaps);
2664         READ_CAPS(fcoecaps);
2665
2666         if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
2667                 param[0] = FW_PARAM_PFVF(ETHOFLD_START);
2668                 param[1] = FW_PARAM_PFVF(ETHOFLD_END);
2669                 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2670                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
2671                 if (rc != 0) {
2672                         device_printf(sc->dev,
2673                             "failed to query NIC parameters: %d.\n", rc);
2674                         return (rc);
2675                 }
2676                 sc->tids.etid_base = val[0];
2677                 sc->params.etid_min = val[0];
2678                 sc->tids.netids = val[1] - val[0] + 1;
2679                 sc->params.netids = sc->tids.netids;
2680                 sc->params.eo_wr_cred = val[2];
2681                 sc->params.ethoffload = 1;
2682         }
2683
2684         if (sc->toecaps) {
2685                 /* query offload-related parameters */
2686                 param[0] = FW_PARAM_DEV(NTID);
2687                 param[1] = FW_PARAM_PFVF(SERVER_START);
2688                 param[2] = FW_PARAM_PFVF(SERVER_END);
2689                 param[3] = FW_PARAM_PFVF(TDDP_START);
2690                 param[4] = FW_PARAM_PFVF(TDDP_END);
2691                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2692                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2693                 if (rc != 0) {
2694                         device_printf(sc->dev,
2695                             "failed to query TOE parameters: %d.\n", rc);
2696                         return (rc);
2697                 }
2698                 sc->tids.ntids = val[0];
2699                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
2700                 sc->tids.stid_base = val[1];
2701                 sc->tids.nstids = val[2] - val[1] + 1;
2702                 sc->vres.ddp.start = val[3];
2703                 sc->vres.ddp.size = val[4] - val[3] + 1;
2704                 sc->params.ofldq_wr_cred = val[5];
2705                 sc->params.offload = 1;
2706         }
2707         if (sc->rdmacaps) {
2708                 param[0] = FW_PARAM_PFVF(STAG_START);
2709                 param[1] = FW_PARAM_PFVF(STAG_END);
2710                 param[2] = FW_PARAM_PFVF(RQ_START);
2711                 param[3] = FW_PARAM_PFVF(RQ_END);
2712                 param[4] = FW_PARAM_PFVF(PBL_START);
2713                 param[5] = FW_PARAM_PFVF(PBL_END);
2714                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2715                 if (rc != 0) {
2716                         device_printf(sc->dev,
2717                             "failed to query RDMA parameters(1): %d.\n", rc);
2718                         return (rc);
2719                 }
2720                 sc->vres.stag.start = val[0];
2721                 sc->vres.stag.size = val[1] - val[0] + 1;
2722                 sc->vres.rq.start = val[2];
2723                 sc->vres.rq.size = val[3] - val[2] + 1;
2724                 sc->vres.pbl.start = val[4];
2725                 sc->vres.pbl.size = val[5] - val[4] + 1;
2726
2727                 param[0] = FW_PARAM_PFVF(SQRQ_START);
2728                 param[1] = FW_PARAM_PFVF(SQRQ_END);
2729                 param[2] = FW_PARAM_PFVF(CQ_START);
2730                 param[3] = FW_PARAM_PFVF(CQ_END);
2731                 param[4] = FW_PARAM_PFVF(OCQ_START);
2732                 param[5] = FW_PARAM_PFVF(OCQ_END);
2733                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2734                 if (rc != 0) {
2735                         device_printf(sc->dev,
2736                             "failed to query RDMA parameters(2): %d.\n", rc);
2737                         return (rc);
2738                 }
2739                 sc->vres.qp.start = val[0];
2740                 sc->vres.qp.size = val[1] - val[0] + 1;
2741                 sc->vres.cq.start = val[2];
2742                 sc->vres.cq.size = val[3] - val[2] + 1;
2743                 sc->vres.ocq.start = val[4];
2744                 sc->vres.ocq.size = val[5] - val[4] + 1;
2745         }
2746         if (sc->iscsicaps) {
2747                 param[0] = FW_PARAM_PFVF(ISCSI_START);
2748                 param[1] = FW_PARAM_PFVF(ISCSI_END);
2749                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2750                 if (rc != 0) {
2751                         device_printf(sc->dev,
2752                             "failed to query iSCSI parameters: %d.\n", rc);
2753                         return (rc);
2754                 }
2755                 sc->vres.iscsi.start = val[0];
2756                 sc->vres.iscsi.size = val[1] - val[0] + 1;
2757         }
2758
2759         /*
2760          * We've got the params we wanted to query via the firmware.  Now grab
2761          * some others directly from the chip.
2762          */
2763         rc = t4_read_chip_settings(sc);
2764
2765         return (rc);
2766 }
2767
2768 static int
2769 set_params__post_init(struct adapter *sc)
2770 {
2771         uint32_t param, val;
2772
2773         /* ask for encapsulated CPLs */
2774         param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
2775         val = 1;
2776         (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2777
2778         return (0);
2779 }
2780
2781 #undef FW_PARAM_PFVF
2782 #undef FW_PARAM_DEV
2783
2784 static void
2785 t4_set_desc(struct adapter *sc)
2786 {
2787         char buf[128];
2788         struct adapter_params *p = &sc->params;
2789
2790         snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
2791             "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
2792             chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
2793
2794         device_set_desc_copy(sc->dev, buf);
2795 }
2796
2797 static void
2798 build_medialist(struct port_info *pi, struct ifmedia *media)
2799 {
2800         int data, m;
2801
2802         PORT_LOCK(pi);
2803
2804         ifmedia_removeall(media);
2805
2806         m = IFM_ETHER | IFM_FDX;
2807         data = (pi->port_type << 8) | pi->mod_type;
2808
2809         switch(pi->port_type) {
2810         case FW_PORT_TYPE_BT_XFI:
2811         case FW_PORT_TYPE_BT_XAUI:
2812                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2813                 /* fall through */
2814
2815         case FW_PORT_TYPE_BT_SGMII:
2816                 ifmedia_add(media, m | IFM_1000_T, data, NULL);
2817                 ifmedia_add(media, m | IFM_100_TX, data, NULL);
2818                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL);
2819                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2820                 break;
2821
2822         case FW_PORT_TYPE_CX4:
2823                 ifmedia_add(media, m | IFM_10G_CX4, data, NULL);
2824                 ifmedia_set(media, m | IFM_10G_CX4);
2825                 break;
2826
2827         case FW_PORT_TYPE_QSFP_10G:
2828         case FW_PORT_TYPE_SFP:
2829         case FW_PORT_TYPE_FIBER_XFI:
2830         case FW_PORT_TYPE_FIBER_XAUI:
2831                 switch (pi->mod_type) {
2832
2833                 case FW_PORT_MOD_TYPE_LR:
2834                         ifmedia_add(media, m | IFM_10G_LR, data, NULL);
2835                         ifmedia_set(media, m | IFM_10G_LR);
2836                         break;
2837
2838                 case FW_PORT_MOD_TYPE_SR:
2839                         ifmedia_add(media, m | IFM_10G_SR, data, NULL);
2840                         ifmedia_set(media, m | IFM_10G_SR);
2841                         break;
2842
2843                 case FW_PORT_MOD_TYPE_LRM:
2844                         ifmedia_add(media, m | IFM_10G_LRM, data, NULL);
2845                         ifmedia_set(media, m | IFM_10G_LRM);
2846                         break;
2847
2848                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2849                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2850                         ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL);
2851                         ifmedia_set(media, m | IFM_10G_TWINAX);
2852                         break;
2853
2854                 case FW_PORT_MOD_TYPE_NONE:
2855                         m &= ~IFM_FDX;
2856                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2857                         ifmedia_set(media, m | IFM_NONE);
2858                         break;
2859
2860                 case FW_PORT_MOD_TYPE_NA:
2861                 case FW_PORT_MOD_TYPE_ER:
2862                 default:
2863                         device_printf(pi->dev,
2864                             "unknown port_type (%d), mod_type (%d)\n",
2865                             pi->port_type, pi->mod_type);
2866                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2867                         ifmedia_set(media, m | IFM_UNKNOWN);
2868                         break;
2869                 }
2870                 break;
2871
2872         case FW_PORT_TYPE_QSFP:
2873                 switch (pi->mod_type) {
2874
2875                 case FW_PORT_MOD_TYPE_LR:
2876                         ifmedia_add(media, m | IFM_40G_LR4, data, NULL);
2877                         ifmedia_set(media, m | IFM_40G_LR4);
2878                         break;
2879
2880                 case FW_PORT_MOD_TYPE_SR:
2881                         ifmedia_add(media, m | IFM_40G_SR4, data, NULL);
2882                         ifmedia_set(media, m | IFM_40G_SR4);
2883                         break;
2884
2885                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2886                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2887                         ifmedia_add(media, m | IFM_40G_CR4, data, NULL);
2888                         ifmedia_set(media, m | IFM_40G_CR4);
2889                         break;
2890
2891                 case FW_PORT_MOD_TYPE_NONE:
2892                         m &= ~IFM_FDX;
2893                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2894                         ifmedia_set(media, m | IFM_NONE);
2895                         break;
2896
2897                 default:
2898                         device_printf(pi->dev,
2899                             "unknown port_type (%d), mod_type (%d)\n",
2900                             pi->port_type, pi->mod_type);
2901                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2902                         ifmedia_set(media, m | IFM_UNKNOWN);
2903                         break;
2904                 }
2905                 break;
2906
2907         default:
2908                 device_printf(pi->dev,
2909                     "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
2910                     pi->mod_type);
2911                 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2912                 ifmedia_set(media, m | IFM_UNKNOWN);
2913                 break;
2914         }
2915
2916         PORT_UNLOCK(pi);
2917 }
2918
2919 #define FW_MAC_EXACT_CHUNK      7
2920
2921 /*
2922  * Program the port's XGMAC based on parameters in ifnet.  The caller also
2923  * indicates which parameters should be programmed (the rest are left alone).
2924  */
2925 int
2926 update_mac_settings(struct ifnet *ifp, int flags)
2927 {
2928         int rc = 0;
2929         struct port_info *pi = ifp->if_softc;
2930         struct adapter *sc = pi->adapter;
2931         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
2932         uint16_t viid = 0xffff;
2933         int16_t *xact_addr_filt = NULL;
2934
2935         ASSERT_SYNCHRONIZED_OP(sc);
2936         KASSERT(flags, ("%s: not told what to update.", __func__));
2937
2938         if (ifp == pi->ifp) {
2939                 viid = pi->viid;
2940                 xact_addr_filt = &pi->xact_addr_filt;
2941         }
2942 #ifdef DEV_NETMAP
2943         else if (ifp == pi->nm_ifp) {
2944                 viid = pi->nm_viid;
2945                 xact_addr_filt = &pi->nm_xact_addr_filt;
2946         }
2947 #endif
2948         if (flags & XGMAC_MTU)
2949                 mtu = ifp->if_mtu;
2950
2951         if (flags & XGMAC_PROMISC)
2952                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
2953
2954         if (flags & XGMAC_ALLMULTI)
2955                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
2956
2957         if (flags & XGMAC_VLANEX)
2958                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
2959
2960         if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
2961                 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti,
2962                     1, vlanex, false);
2963                 if (rc) {
2964                         if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
2965                             rc);
2966                         return (rc);
2967                 }
2968         }
2969
2970         if (flags & XGMAC_UCADDR) {
2971                 uint8_t ucaddr[ETHER_ADDR_LEN];
2972
2973                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
2974                 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr,
2975                     true, true);
2976                 if (rc < 0) {
2977                         rc = -rc;
2978                         if_printf(ifp, "change_mac failed: %d\n", rc);
2979                         return (rc);
2980                 } else {
2981                         *xact_addr_filt = rc;
2982                         rc = 0;
2983                 }
2984         }
2985
2986         if (flags & XGMAC_MCADDRS) {
2987                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
2988                 int del = 1;
2989                 uint64_t hash = 0;
2990                 struct ifmultiaddr *ifma;
2991                 int i = 0, j;
2992
2993                 if_maddr_rlock(ifp);
2994                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2995                         if (ifma->ifma_addr->sa_family != AF_LINK)
2996                                 continue;
2997                         mcaddr[i++] =
2998                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
2999
3000                         if (i == FW_MAC_EXACT_CHUNK) {
3001                                 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del,
3002                                     i, mcaddr, NULL, &hash, 0);
3003                                 if (rc < 0) {
3004                                         rc = -rc;
3005                                         for (j = 0; j < i; j++) {
3006                                                 if_printf(ifp,
3007                                                     "failed to add mc address"
3008                                                     " %02x:%02x:%02x:"
3009                                                     "%02x:%02x:%02x rc=%d\n",
3010                                                     mcaddr[j][0], mcaddr[j][1],
3011                                                     mcaddr[j][2], mcaddr[j][3],
3012                                                     mcaddr[j][4], mcaddr[j][5],
3013                                                     rc);
3014                                         }
3015                                         goto mcfail;
3016                                 }
3017                                 del = 0;
3018                                 i = 0;
3019                         }
3020                 }
3021                 if (i > 0) {
3022                         rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i,
3023                             mcaddr, NULL, &hash, 0);
3024                         if (rc < 0) {
3025                                 rc = -rc;
3026                                 for (j = 0; j < i; j++) {
3027                                         if_printf(ifp,
3028                                             "failed to add mc address"
3029                                             " %02x:%02x:%02x:"
3030                                             "%02x:%02x:%02x rc=%d\n",
3031                                             mcaddr[j][0], mcaddr[j][1],
3032                                             mcaddr[j][2], mcaddr[j][3],
3033                                             mcaddr[j][4], mcaddr[j][5],
3034                                             rc);
3035                                 }
3036                                 goto mcfail;
3037                         }
3038                 }
3039
3040                 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0);
3041                 if (rc != 0)
3042                         if_printf(ifp, "failed to set mc address hash: %d", rc);
3043 mcfail:
3044                 if_maddr_runlock(ifp);
3045         }
3046
3047         return (rc);
3048 }
3049
3050 /*
3051  * {begin|end}_synchronized_op must be called from the same thread.
3052  */
3053 int
3054 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
3055     char *wmesg)
3056 {
3057         int rc, pri;
3058
3059 #ifdef WITNESS
3060         /* the caller thinks it's ok to sleep, but is it really? */
3061         if (flags & SLEEP_OK)
3062                 pause("t4slptst", 1);
3063 #endif
3064
3065         if (INTR_OK)
3066                 pri = PCATCH;
3067         else
3068                 pri = 0;
3069
3070         ADAPTER_LOCK(sc);
3071         for (;;) {
3072
3073                 if (pi && IS_DOOMED(pi)) {
3074                         rc = ENXIO;
3075                         goto done;
3076                 }
3077
3078                 if (!IS_BUSY(sc)) {
3079                         rc = 0;
3080                         break;
3081                 }
3082
3083                 if (!(flags & SLEEP_OK)) {
3084                         rc = EBUSY;
3085                         goto done;
3086                 }
3087
3088                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
3089                         rc = EINTR;
3090                         goto done;
3091                 }
3092         }
3093
3094         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
3095         SET_BUSY(sc);
3096 #ifdef INVARIANTS
3097         sc->last_op = wmesg;
3098         sc->last_op_thr = curthread;
3099 #endif
3100
3101 done:
3102         if (!(flags & HOLD_LOCK) || rc)
3103                 ADAPTER_UNLOCK(sc);
3104
3105         return (rc);
3106 }
3107
3108 /*
3109  * {begin|end}_synchronized_op must be called from the same thread.
3110  */
3111 void
3112 end_synchronized_op(struct adapter *sc, int flags)
3113 {
3114
3115         if (flags & LOCK_HELD)
3116                 ADAPTER_LOCK_ASSERT_OWNED(sc);
3117         else
3118                 ADAPTER_LOCK(sc);
3119
3120         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
3121         CLR_BUSY(sc);
3122         wakeup(&sc->flags);
3123         ADAPTER_UNLOCK(sc);
3124 }
3125
3126 static int
3127 cxgbe_init_synchronized(struct port_info *pi)
3128 {
3129         struct adapter *sc = pi->adapter;
3130         struct ifnet *ifp = pi->ifp;
3131         int rc = 0;
3132
3133         ASSERT_SYNCHRONIZED_OP(sc);
3134
3135         if (isset(&sc->open_device_map, pi->port_id)) {
3136                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
3137                     ("mismatch between open_device_map and if_drv_flags"));
3138                 return (0);     /* already running */
3139         }
3140
3141         if (!(sc->flags & FULL_INIT_DONE) &&
3142             ((rc = adapter_full_init(sc)) != 0))
3143                 return (rc);    /* error message displayed already */
3144
3145         if (!(pi->flags & PORT_INIT_DONE) &&
3146             ((rc = port_full_init(pi)) != 0))
3147                 return (rc); /* error message displayed already */
3148
3149         rc = update_mac_settings(ifp, XGMAC_ALL);
3150         if (rc)
3151                 goto done;      /* error message displayed already */
3152
3153         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
3154         if (rc != 0) {
3155                 if_printf(ifp, "enable_vi failed: %d\n", rc);
3156                 goto done;
3157         }
3158
3159         /*
3160          * The first iq of the first port to come up is used for tracing.
3161          */
3162         if (sc->traceq < 0) {
3163                 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
3164                 t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
3165                     A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
3166                     V_QUEUENUMBER(sc->traceq));
3167                 pi->flags |= HAS_TRACEQ;
3168         }
3169
3170         /* all ok */
3171         setbit(&sc->open_device_map, pi->port_id);
3172         PORT_LOCK(pi);
3173         ifp->if_drv_flags |= IFF_DRV_RUNNING;
3174         PORT_UNLOCK(pi);
3175
3176         callout_reset(&pi->tick, hz, cxgbe_tick, pi);
3177 done:
3178         if (rc != 0)
3179                 cxgbe_uninit_synchronized(pi);
3180
3181         return (rc);
3182 }
3183
3184 /*
3185  * Idempotent.
3186  */
3187 static int
3188 cxgbe_uninit_synchronized(struct port_info *pi)
3189 {
3190         struct adapter *sc = pi->adapter;
3191         struct ifnet *ifp = pi->ifp;
3192         int rc;
3193
3194         ASSERT_SYNCHRONIZED_OP(sc);
3195
3196         /*
3197          * Disable the VI so that all its data in either direction is discarded
3198          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
3199          * tick) intact as the TP can deliver negative advice or data that it's
3200          * holding in its RAM (for an offloaded connection) even after the VI is
3201          * disabled.
3202          */
3203         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
3204         if (rc) {
3205                 if_printf(ifp, "disable_vi failed: %d\n", rc);
3206                 return (rc);
3207         }
3208
3209         clrbit(&sc->open_device_map, pi->port_id);
3210         PORT_LOCK(pi);
3211         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3212         PORT_UNLOCK(pi);
3213
3214         pi->link_cfg.link_ok = 0;
3215         pi->link_cfg.speed = 0;
3216         pi->linkdnrc = -1;
3217         t4_os_link_changed(sc, pi->port_id, 0, -1);
3218
3219         return (0);
3220 }
3221
3222 /*
3223  * It is ok for this function to fail midway and return right away.  t4_detach
3224  * will walk the entire sc->irq list and clean up whatever is valid.
3225  */
3226 static int
3227 setup_intr_handlers(struct adapter *sc)
3228 {
3229         int rc, rid, p, q;
3230         char s[8];
3231         struct irq *irq;
3232         struct port_info *pi;
3233         struct sge_rxq *rxq;
3234 #ifdef TCP_OFFLOAD
3235         struct sge_ofld_rxq *ofld_rxq;
3236 #endif
3237 #ifdef DEV_NETMAP
3238         struct sge_nm_rxq *nm_rxq;
3239 #endif
3240
3241         /*
3242          * Setup interrupts.
3243          */
3244         irq = &sc->irq[0];
3245         rid = sc->intr_type == INTR_INTX ? 0 : 1;
3246         if (sc->intr_count == 1)
3247                 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
3248
3249         /* Multiple interrupts. */
3250         KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
3251             ("%s: too few intr.", __func__));
3252
3253         /* The first one is always error intr */
3254         rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
3255         if (rc != 0)
3256                 return (rc);
3257         irq++;
3258         rid++;
3259
3260         /* The second one is always the firmware event queue */
3261         rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
3262         if (rc != 0)
3263                 return (rc);
3264         irq++;
3265         rid++;
3266
3267         for_each_port(sc, p) {
3268                 pi = sc->port[p];
3269
3270                 if (pi->flags & INTR_RXQ) {
3271                         for_each_rxq(pi, q, rxq) {
3272                                 snprintf(s, sizeof(s), "%d.%d", p, q);
3273                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
3274                                     s);
3275                                 if (rc != 0)
3276                                         return (rc);
3277                                 irq++;
3278                                 rid++;
3279                         }
3280                 }
3281 #ifdef TCP_OFFLOAD
3282                 if (pi->flags & INTR_OFLD_RXQ) {
3283                         for_each_ofld_rxq(pi, q, ofld_rxq) {
3284                                 snprintf(s, sizeof(s), "%d,%d", p, q);
3285                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr,
3286                                     ofld_rxq, s);
3287                                 if (rc != 0)
3288                                         return (rc);
3289                                 irq++;
3290                                 rid++;
3291                         }
3292                 }
3293 #endif
3294 #ifdef DEV_NETMAP
3295                 if (pi->flags & INTR_NM_RXQ) {
3296                         for_each_nm_rxq(pi, q, nm_rxq) {
3297                                 snprintf(s, sizeof(s), "%d-%d", p, q);
3298                                 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr,
3299                                     nm_rxq, s);
3300                                 if (rc != 0)
3301                                         return (rc);
3302                                 irq++;
3303                                 rid++;
3304                         }
3305                 }
3306 #endif
3307         }
3308         MPASS(irq == &sc->irq[sc->intr_count]);
3309
3310         return (0);
3311 }
3312
3313 int
3314 adapter_full_init(struct adapter *sc)
3315 {
3316         int rc, i;
3317
3318         ASSERT_SYNCHRONIZED_OP(sc);
3319         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3320         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
3321             ("%s: FULL_INIT_DONE already", __func__));
3322
3323         /*
3324          * queues that belong to the adapter (not any particular port).
3325          */
3326         rc = t4_setup_adapter_queues(sc);
3327         if (rc != 0)
3328                 goto done;
3329
3330         for (i = 0; i < nitems(sc->tq); i++) {
3331                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
3332                     taskqueue_thread_enqueue, &sc->tq[i]);
3333                 if (sc->tq[i] == NULL) {
3334                         device_printf(sc->dev,
3335                             "failed to allocate task queue %d\n", i);
3336                         rc = ENOMEM;
3337                         goto done;
3338                 }
3339                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
3340                     device_get_nameunit(sc->dev), i);
3341         }
3342
3343         t4_intr_enable(sc);
3344         sc->flags |= FULL_INIT_DONE;
3345 done:
3346         if (rc != 0)
3347                 adapter_full_uninit(sc);
3348
3349         return (rc);
3350 }
3351
3352 int
3353 adapter_full_uninit(struct adapter *sc)
3354 {
3355         int i;
3356
3357         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3358
3359         t4_teardown_adapter_queues(sc);
3360
3361         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
3362                 taskqueue_free(sc->tq[i]);
3363                 sc->tq[i] = NULL;
3364         }
3365
3366         sc->flags &= ~FULL_INIT_DONE;
3367
3368         return (0);
3369 }
3370
3371 int
3372 port_full_init(struct port_info *pi)
3373 {
3374         struct adapter *sc = pi->adapter;
3375         struct ifnet *ifp = pi->ifp;
3376         uint16_t *rss;
3377         struct sge_rxq *rxq;
3378         int rc, i, j;
3379
3380         ASSERT_SYNCHRONIZED_OP(sc);
3381         KASSERT((pi->flags & PORT_INIT_DONE) == 0,
3382             ("%s: PORT_INIT_DONE already", __func__));
3383
3384         sysctl_ctx_init(&pi->ctx);
3385         pi->flags |= PORT_SYSCTL_CTX;
3386
3387         /*
3388          * Allocate tx/rx/fl queues for this port.
3389          */
3390         rc = t4_setup_port_queues(pi);
3391         if (rc != 0)
3392                 goto done;      /* error message displayed already */
3393
3394         /*
3395          * Setup RSS for this port.  Save a copy of the RSS table for later use.
3396          */
3397         rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
3398         for (i = 0; i < pi->rss_size;) {
3399                 for_each_rxq(pi, j, rxq) {
3400                         rss[i++] = rxq->iq.abs_id;
3401                         if (i == pi->rss_size)
3402                                 break;
3403                 }
3404         }
3405
3406         rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
3407             pi->rss_size);
3408         if (rc != 0) {
3409                 if_printf(ifp, "rss_config failed: %d\n", rc);
3410                 goto done;
3411         }
3412
3413         pi->rss = rss;
3414         pi->flags |= PORT_INIT_DONE;
3415 done:
3416         if (rc != 0)
3417                 port_full_uninit(pi);
3418
3419         return (rc);
3420 }
3421
3422 /*
3423  * Idempotent.
3424  */
3425 int
3426 port_full_uninit(struct port_info *pi)
3427 {
3428         struct adapter *sc = pi->adapter;
3429         int i;
3430         struct sge_rxq *rxq;
3431         struct sge_txq *txq;
3432 #ifdef TCP_OFFLOAD
3433         struct sge_ofld_rxq *ofld_rxq;
3434         struct sge_wrq *ofld_txq;
3435 #endif
3436
3437         if (pi->flags & PORT_INIT_DONE) {
3438
3439                 /* Need to quiesce queues.  XXX: ctrl queues? */
3440
3441                 for_each_txq(pi, i, txq) {
3442                         quiesce_eq(sc, &txq->eq);
3443                 }
3444
3445 #ifdef TCP_OFFLOAD
3446                 for_each_ofld_txq(pi, i, ofld_txq) {
3447                         quiesce_eq(sc, &ofld_txq->eq);
3448                 }
3449 #endif
3450
3451                 for_each_rxq(pi, i, rxq) {
3452                         quiesce_iq(sc, &rxq->iq);
3453                         quiesce_fl(sc, &rxq->fl);
3454                 }
3455
3456 #ifdef TCP_OFFLOAD
3457                 for_each_ofld_rxq(pi, i, ofld_rxq) {
3458                         quiesce_iq(sc, &ofld_rxq->iq);
3459                         quiesce_fl(sc, &ofld_rxq->fl);
3460                 }
3461 #endif
3462                 free(pi->rss, M_CXGBE);
3463         }
3464
3465         t4_teardown_port_queues(pi);
3466         pi->flags &= ~PORT_INIT_DONE;
3467
3468         return (0);
3469 }
3470
3471 static void
3472 quiesce_eq(struct adapter *sc, struct sge_eq *eq)
3473 {
3474         EQ_LOCK(eq);
3475         eq->flags |= EQ_DOOMED;
3476
3477         /*
3478          * Wait for the response to a credit flush if one's
3479          * pending.
3480          */
3481         while (eq->flags & EQ_CRFLUSHED)
3482                 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0);
3483         EQ_UNLOCK(eq);
3484
3485         callout_drain(&eq->tx_callout); /* XXX: iffy */
3486         pause("callout", 10);           /* Still iffy */
3487
3488         taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task);
3489 }
3490
3491 static void
3492 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
3493 {
3494         (void) sc;      /* unused */
3495
3496         /* Synchronize with the interrupt handler */
3497         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
3498                 pause("iqfree", 1);
3499 }
3500
3501 static void
3502 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
3503 {
3504         mtx_lock(&sc->sfl_lock);
3505         FL_LOCK(fl);
3506         fl->flags |= FL_DOOMED;
3507         FL_UNLOCK(fl);
3508         mtx_unlock(&sc->sfl_lock);
3509
3510         callout_drain(&sc->sfl_callout);
3511         KASSERT((fl->flags & FL_STARVING) == 0,
3512             ("%s: still starving", __func__));
3513 }
3514
3515 static int
3516 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
3517     driver_intr_t *handler, void *arg, char *name)
3518 {
3519         int rc;
3520
3521         irq->rid = rid;
3522         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
3523             RF_SHAREABLE | RF_ACTIVE);
3524         if (irq->res == NULL) {
3525                 device_printf(sc->dev,
3526                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
3527                 return (ENOMEM);
3528         }
3529
3530         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
3531             NULL, handler, arg, &irq->tag);
3532         if (rc != 0) {
3533                 device_printf(sc->dev,
3534                     "failed to setup interrupt for rid %d, name %s: %d\n",
3535                     rid, name, rc);
3536         } else if (name)
3537                 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
3538
3539         return (rc);
3540 }
3541
3542 static int
3543 t4_free_irq(struct adapter *sc, struct irq *irq)
3544 {
3545         if (irq->tag)
3546                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
3547         if (irq->res)
3548                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
3549
3550         bzero(irq, sizeof(*irq));
3551
3552         return (0);
3553 }
3554
3555 static void
3556 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
3557     unsigned int end)
3558 {
3559         uint32_t *p = (uint32_t *)(buf + start);
3560
3561         for ( ; start <= end; start += sizeof(uint32_t))
3562                 *p++ = t4_read_reg(sc, start);
3563 }
3564
3565 static void
3566 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
3567 {
3568         int i, n;
3569         const unsigned int *reg_ranges;
3570         static const unsigned int t4_reg_ranges[] = {
3571                 0x1008, 0x1108,
3572                 0x1180, 0x11b4,
3573                 0x11fc, 0x123c,
3574                 0x1300, 0x173c,
3575                 0x1800, 0x18fc,
3576                 0x3000, 0x30d8,
3577                 0x30e0, 0x5924,
3578                 0x5960, 0x59d4,
3579                 0x5a00, 0x5af8,
3580                 0x6000, 0x6098,
3581                 0x6100, 0x6150,
3582                 0x6200, 0x6208,
3583                 0x6240, 0x6248,
3584                 0x6280, 0x6338,
3585                 0x6370, 0x638c,
3586                 0x6400, 0x643c,
3587                 0x6500, 0x6524,
3588                 0x6a00, 0x6a38,
3589                 0x6a60, 0x6a78,
3590                 0x6b00, 0x6b84,
3591                 0x6bf0, 0x6c84,
3592                 0x6cf0, 0x6d84,
3593                 0x6df0, 0x6e84,
3594                 0x6ef0, 0x6f84,
3595                 0x6ff0, 0x7084,
3596                 0x70f0, 0x7184,
3597                 0x71f0, 0x7284,
3598                 0x72f0, 0x7384,
3599                 0x73f0, 0x7450,
3600                 0x7500, 0x7530,
3601                 0x7600, 0x761c,
3602                 0x7680, 0x76cc,
3603                 0x7700, 0x7798,
3604                 0x77c0, 0x77fc,
3605                 0x7900, 0x79fc,
3606                 0x7b00, 0x7c38,
3607                 0x7d00, 0x7efc,
3608                 0x8dc0, 0x8e1c,
3609                 0x8e30, 0x8e78,
3610                 0x8ea0, 0x8f6c,
3611                 0x8fc0, 0x9074,
3612                 0x90fc, 0x90fc,
3613                 0x9400, 0x9458,
3614                 0x9600, 0x96bc,
3615                 0x9800, 0x9808,
3616                 0x9820, 0x983c,
3617                 0x9850, 0x9864,
3618                 0x9c00, 0x9c6c,
3619                 0x9c80, 0x9cec,
3620                 0x9d00, 0x9d6c,
3621                 0x9d80, 0x9dec,
3622                 0x9e00, 0x9e6c,
3623                 0x9e80, 0x9eec,
3624                 0x9f00, 0x9f6c,
3625                 0x9f80, 0x9fec,
3626                 0xd004, 0xd03c,
3627                 0xdfc0, 0xdfe0,
3628                 0xe000, 0xea7c,
3629                 0xf000, 0x11110,
3630                 0x11118, 0x11190,
3631                 0x19040, 0x1906c,
3632                 0x19078, 0x19080,
3633                 0x1908c, 0x19124,
3634                 0x19150, 0x191b0,
3635                 0x191d0, 0x191e8,
3636                 0x19238, 0x1924c,
3637                 0x193f8, 0x19474,
3638                 0x19490, 0x194f8,
3639                 0x19800, 0x19f30,
3640                 0x1a000, 0x1a06c,
3641                 0x1a0b0, 0x1a120,
3642                 0x1a128, 0x1a138,
3643                 0x1a190, 0x1a1c4,
3644                 0x1a1fc, 0x1a1fc,
3645                 0x1e040, 0x1e04c,
3646                 0x1e284, 0x1e28c,
3647                 0x1e2c0, 0x1e2c0,
3648                 0x1e2e0, 0x1e2e0,
3649                 0x1e300, 0x1e384,
3650                 0x1e3c0, 0x1e3c8,
3651                 0x1e440, 0x1e44c,
3652                 0x1e684, 0x1e68c,
3653                 0x1e6c0, 0x1e6c0,
3654                 0x1e6e0, 0x1e6e0,
3655                 0x1e700, 0x1e784,
3656                 0x1e7c0, 0x1e7c8,
3657                 0x1e840, 0x1e84c,
3658                 0x1ea84, 0x1ea8c,
3659                 0x1eac0, 0x1eac0,
3660                 0x1eae0, 0x1eae0,
3661                 0x1eb00, 0x1eb84,
3662                 0x1ebc0, 0x1ebc8,
3663                 0x1ec40, 0x1ec4c,
3664                 0x1ee84, 0x1ee8c,
3665                 0x1eec0, 0x1eec0,
3666                 0x1eee0, 0x1eee0,
3667                 0x1ef00, 0x1ef84,
3668                 0x1efc0, 0x1efc8,
3669                 0x1f040, 0x1f04c,
3670                 0x1f284, 0x1f28c,
3671                 0x1f2c0, 0x1f2c0,
3672                 0x1f2e0, 0x1f2e0,
3673                 0x1f300, 0x1f384,
3674                 0x1f3c0, 0x1f3c8,
3675                 0x1f440, 0x1f44c,
3676                 0x1f684, 0x1f68c,
3677                 0x1f6c0, 0x1f6c0,
3678                 0x1f6e0, 0x1f6e0,
3679                 0x1f700, 0x1f784,
3680                 0x1f7c0, 0x1f7c8,
3681                 0x1f840, 0x1f84c,
3682                 0x1fa84, 0x1fa8c,
3683                 0x1fac0, 0x1fac0,
3684                 0x1fae0, 0x1fae0,
3685                 0x1fb00, 0x1fb84,
3686                 0x1fbc0, 0x1fbc8,
3687                 0x1fc40, 0x1fc4c,
3688                 0x1fe84, 0x1fe8c,
3689                 0x1fec0, 0x1fec0,
3690                 0x1fee0, 0x1fee0,
3691                 0x1ff00, 0x1ff84,
3692                 0x1ffc0, 0x1ffc8,
3693                 0x20000, 0x2002c,
3694                 0x20100, 0x2013c,
3695                 0x20190, 0x201c8,
3696                 0x20200, 0x20318,
3697                 0x20400, 0x20528,
3698                 0x20540, 0x20614,
3699                 0x21000, 0x21040,
3700                 0x2104c, 0x21060,
3701                 0x210c0, 0x210ec,
3702                 0x21200, 0x21268,
3703                 0x21270, 0x21284,
3704                 0x212fc, 0x21388,
3705                 0x21400, 0x21404,
3706                 0x21500, 0x21518,
3707                 0x2152c, 0x2153c,
3708                 0x21550, 0x21554,
3709                 0x21600, 0x21600,
3710                 0x21608, 0x21628,
3711                 0x21630, 0x2163c,
3712                 0x21700, 0x2171c,
3713                 0x21780, 0x2178c,
3714                 0x21800, 0x21c38,
3715                 0x21c80, 0x21d7c,
3716                 0x21e00, 0x21e04,
3717                 0x22000, 0x2202c,
3718                 0x22100, 0x2213c,
3719                 0x22190, 0x221c8,
3720                 0x22200, 0x22318,
3721                 0x22400, 0x22528,
3722                 0x22540, 0x22614,
3723                 0x23000, 0x23040,
3724                 0x2304c, 0x23060,
3725                 0x230c0, 0x230ec,
3726                 0x23200, 0x23268,
3727                 0x23270, 0x23284,
3728                 0x232fc, 0x23388,
3729                 0x23400, 0x23404,
3730                 0x23500, 0x23518,
3731                 0x2352c, 0x2353c,
3732                 0x23550, 0x23554,
3733                 0x23600, 0x23600,
3734                 0x23608, 0x23628,
3735                 0x23630, 0x2363c,
3736                 0x23700, 0x2371c,
3737                 0x23780, 0x2378c,
3738                 0x23800, 0x23c38,
3739                 0x23c80, 0x23d7c,
3740                 0x23e00, 0x23e04,
3741                 0x24000, 0x2402c,
3742                 0x24100, 0x2413c,
3743                 0x24190, 0x241c8,
3744                 0x24200, 0x24318,
3745                 0x24400, 0x24528,
3746                 0x24540, 0x24614,
3747                 0x25000, 0x25040,
3748                 0x2504c, 0x25060,
3749                 0x250c0, 0x250ec,
3750                 0x25200, 0x25268,
3751                 0x25270, 0x25284,
3752                 0x252fc, 0x25388,
3753                 0x25400, 0x25404,
3754                 0x25500, 0x25518,
3755                 0x2552c, 0x2553c,
3756                 0x25550, 0x25554,
3757                 0x25600, 0x25600,
3758                 0x25608, 0x25628,
3759                 0x25630, 0x2563c,
3760                 0x25700, 0x2571c,
3761                 0x25780, 0x2578c,
3762                 0x25800, 0x25c38,
3763                 0x25c80, 0x25d7c,
3764                 0x25e00, 0x25e04,
3765                 0x26000, 0x2602c,
3766                 0x26100, 0x2613c,
3767                 0x26190, 0x261c8,
3768                 0x26200, 0x26318,
3769                 0x26400, 0x26528,
3770                 0x26540, 0x26614,
3771                 0x27000, 0x27040,
3772                 0x2704c, 0x27060,
3773                 0x270c0, 0x270ec,
3774                 0x27200, 0x27268,
3775                 0x27270, 0x27284,
3776                 0x272fc, 0x27388,
3777                 0x27400, 0x27404,
3778                 0x27500, 0x27518,
3779                 0x2752c, 0x2753c,
3780                 0x27550, 0x27554,
3781                 0x27600, 0x27600,
3782                 0x27608, 0x27628,
3783                 0x27630, 0x2763c,
3784                 0x27700, 0x2771c,
3785                 0x27780, 0x2778c,
3786                 0x27800, 0x27c38,
3787                 0x27c80, 0x27d7c,
3788                 0x27e00, 0x27e04
3789         };
3790         static const unsigned int t5_reg_ranges[] = {
3791                 0x1008, 0x1148,
3792                 0x1180, 0x11b4,
3793                 0x11fc, 0x123c,
3794                 0x1280, 0x173c,
3795                 0x1800, 0x18fc,
3796                 0x3000, 0x3028,
3797                 0x3060, 0x30d8,
3798                 0x30e0, 0x30fc,
3799                 0x3140, 0x357c,
3800                 0x35a8, 0x35cc,
3801                 0x35ec, 0x35ec,
3802                 0x3600, 0x5624,
3803                 0x56cc, 0x575c,
3804                 0x580c, 0x5814,
3805                 0x5890, 0x58bc,
3806                 0x5940, 0x59dc,
3807                 0x59fc, 0x5a18,
3808                 0x5a60, 0x5a9c,
3809                 0x5b94, 0x5bfc,
3810                 0x6000, 0x6040,
3811                 0x6058, 0x614c,
3812                 0x7700, 0x7798,
3813                 0x77c0, 0x78fc,
3814                 0x7b00, 0x7c54,
3815                 0x7d00, 0x7efc,
3816                 0x8dc0, 0x8de0,
3817                 0x8df8, 0x8e84,
3818                 0x8ea0, 0x8f84,
3819                 0x8fc0, 0x90f8,
3820                 0x9400, 0x9470,
3821                 0x9600, 0x96f4,
3822                 0x9800, 0x9808,
3823                 0x9820, 0x983c,
3824                 0x9850, 0x9864,
3825                 0x9c00, 0x9c6c,
3826                 0x9c80, 0x9cec,
3827                 0x9d00, 0x9d6c,
3828                 0x9d80, 0x9dec,
3829                 0x9e00, 0x9e6c,
3830                 0x9e80, 0x9eec,
3831                 0x9f00, 0x9f6c,
3832                 0x9f80, 0xa020,
3833                 0xd004, 0xd03c,
3834                 0xdfc0, 0xdfe0,
3835                 0xe000, 0x11088,
3836                 0x1109c, 0x11110,
3837                 0x11118, 0x1117c,
3838                 0x11190, 0x11204,
3839                 0x19040, 0x1906c,
3840                 0x19078, 0x19080,
3841                 0x1908c, 0x19124,
3842                 0x19150, 0x191b0,
3843                 0x191d0, 0x191e8,
3844                 0x19238, 0x19290,
3845                 0x193f8, 0x19474,
3846                 0x19490, 0x194cc,
3847                 0x194f0, 0x194f8,
3848                 0x19c00, 0x19c60,
3849                 0x19c94, 0x19e10,
3850                 0x19e50, 0x19f34,
3851                 0x19f40, 0x19f50,
3852                 0x19f90, 0x19fe4,
3853                 0x1a000, 0x1a06c,
3854                 0x1a0b0, 0x1a120,
3855                 0x1a128, 0x1a138,
3856                 0x1a190, 0x1a1c4,
3857                 0x1a1fc, 0x1a1fc,
3858                 0x1e008, 0x1e00c,
3859                 0x1e040, 0x1e04c,
3860                 0x1e284, 0x1e290,
3861                 0x1e2c0, 0x1e2c0,
3862                 0x1e2e0, 0x1e2e0,
3863                 0x1e300, 0x1e384,
3864                 0x1e3c0, 0x1e3c8,
3865                 0x1e408, 0x1e40c,
3866                 0x1e440, 0x1e44c,
3867                 0x1e684, 0x1e690,
3868                 0x1e6c0, 0x1e6c0,
3869                 0x1e6e0, 0x1e6e0,
3870                 0x1e700, 0x1e784,
3871                 0x1e7c0, 0x1e7c8,
3872                 0x1e808, 0x1e80c,
3873                 0x1e840, 0x1e84c,
3874                 0x1ea84, 0x1ea90,
3875                 0x1eac0, 0x1eac0,
3876                 0x1eae0, 0x1eae0,
3877                 0x1eb00, 0x1eb84,
3878                 0x1ebc0, 0x1ebc8,
3879                 0x1ec08, 0x1ec0c,
3880                 0x1ec40, 0x1ec4c,
3881                 0x1ee84, 0x1ee90,
3882                 0x1eec0, 0x1eec0,
3883                 0x1eee0, 0x1eee0,
3884                 0x1ef00, 0x1ef84,
3885                 0x1efc0, 0x1efc8,
3886                 0x1f008, 0x1f00c,
3887                 0x1f040, 0x1f04c,
3888                 0x1f284, 0x1f290,
3889                 0x1f2c0, 0x1f2c0,
3890                 0x1f2e0, 0x1f2e0,
3891                 0x1f300, 0x1f384,
3892                 0x1f3c0, 0x1f3c8,
3893                 0x1f408, 0x1f40c,
3894                 0x1f440, 0x1f44c,
3895                 0x1f684, 0x1f690,
3896                 0x1f6c0, 0x1f6c0,
3897                 0x1f6e0, 0x1f6e0,
3898                 0x1f700, 0x1f784,
3899                 0x1f7c0, 0x1f7c8,
3900                 0x1f808, 0x1f80c,
3901                 0x1f840, 0x1f84c,
3902                 0x1fa84, 0x1fa90,
3903                 0x1fac0, 0x1fac0,
3904                 0x1fae0, 0x1fae0,
3905                 0x1fb00, 0x1fb84,
3906                 0x1fbc0, 0x1fbc8,
3907                 0x1fc08, 0x1fc0c,
3908                 0x1fc40, 0x1fc4c,
3909                 0x1fe84, 0x1fe90,
3910                 0x1fec0, 0x1fec0,
3911                 0x1fee0, 0x1fee0,
3912                 0x1ff00, 0x1ff84,
3913                 0x1ffc0, 0x1ffc8,
3914                 0x30000, 0x30030,
3915                 0x30100, 0x30144,
3916                 0x30190, 0x301d0,
3917                 0x30200, 0x30318,
3918                 0x30400, 0x3052c,
3919                 0x30540, 0x3061c,
3920                 0x30800, 0x30834,
3921                 0x308c0, 0x30908,
3922                 0x30910, 0x309ac,
3923                 0x30a00, 0x30a2c,
3924                 0x30a44, 0x30a50,
3925                 0x30a74, 0x30c24,
3926                 0x30d00, 0x30d00,
3927                 0x30d08, 0x30d14,
3928                 0x30d1c, 0x30d20,
3929                 0x30d3c, 0x30d50,
3930                 0x31200, 0x3120c,
3931                 0x31220, 0x31220,
3932                 0x31240, 0x31240,
3933                 0x31600, 0x3160c,
3934                 0x31a00, 0x31a1c,
3935                 0x31e00, 0x31e20,
3936                 0x31e38, 0x31e3c,
3937                 0x31e80, 0x31e80,
3938                 0x31e88, 0x31ea8,
3939                 0x31eb0, 0x31eb4,
3940                 0x31ec8, 0x31ed4,
3941                 0x31fb8, 0x32004,
3942                 0x32200, 0x32200,
3943                 0x32208, 0x32240,
3944                 0x32248, 0x32280,
3945                 0x32288, 0x322c0,
3946                 0x322c8, 0x322fc,
3947                 0x32600, 0x32630,
3948                 0x32a00, 0x32abc,
3949                 0x32b00, 0x32b70,
3950                 0x33000, 0x33048,
3951                 0x33060, 0x3309c,
3952                 0x330f0, 0x33148,
3953                 0x33160, 0x3319c,
3954                 0x331f0, 0x332e4,
3955                 0x332f8, 0x333e4,
3956                 0x333f8, 0x33448,
3957                 0x33460, 0x3349c,
3958                 0x334f0, 0x33548,
3959                 0x33560, 0x3359c,
3960                 0x335f0, 0x336e4,
3961                 0x336f8, 0x337e4,
3962                 0x337f8, 0x337fc,
3963                 0x33814, 0x33814,
3964                 0x3382c, 0x3382c,
3965                 0x33880, 0x3388c,
3966                 0x338e8, 0x338ec,
3967                 0x33900, 0x33948,
3968                 0x33960, 0x3399c,
3969                 0x339f0, 0x33ae4,
3970                 0x33af8, 0x33b10,
3971                 0x33b28, 0x33b28,
3972                 0x33b3c, 0x33b50,
3973                 0x33bf0, 0x33c10,
3974                 0x33c28, 0x33c28,
3975                 0x33c3c, 0x33c50,
3976                 0x33cf0, 0x33cfc,
3977                 0x34000, 0x34030,
3978                 0x34100, 0x34144,
3979                 0x34190, 0x341d0,
3980                 0x34200, 0x34318,
3981                 0x34400, 0x3452c,
3982                 0x34540, 0x3461c,
3983                 0x34800, 0x34834,
3984                 0x348c0, 0x34908,
3985                 0x34910, 0x349ac,
3986                 0x34a00, 0x34a2c,
3987                 0x34a44, 0x34a50,
3988                 0x34a74, 0x34c24,
3989                 0x34d00, 0x34d00,
3990                 0x34d08, 0x34d14,
3991                 0x34d1c, 0x34d20,
3992                 0x34d3c, 0x34d50,
3993                 0x35200, 0x3520c,
3994                 0x35220, 0x35220,
3995                 0x35240, 0x35240,
3996                 0x35600, 0x3560c,
3997                 0x35a00, 0x35a1c,
3998                 0x35e00, 0x35e20,
3999                 0x35e38, 0x35e3c,
4000                 0x35e80, 0x35e80,
4001                 0x35e88, 0x35ea8,
4002                 0x35eb0, 0x35eb4,
4003                 0x35ec8, 0x35ed4,
4004                 0x35fb8, 0x36004,
4005                 0x36200, 0x36200,
4006                 0x36208, 0x36240,
4007                 0x36248, 0x36280,
4008                 0x36288, 0x362c0,
4009                 0x362c8, 0x362fc,
4010                 0x36600, 0x36630,
4011                 0x36a00, 0x36abc,
4012                 0x36b00, 0x36b70,
4013                 0x37000, 0x37048,
4014                 0x37060, 0x3709c,
4015                 0x370f0, 0x37148,
4016                 0x37160, 0x3719c,
4017                 0x371f0, 0x372e4,
4018                 0x372f8, 0x373e4,
4019                 0x373f8, 0x37448,
4020                 0x37460, 0x3749c,
4021                 0x374f0, 0x37548,
4022                 0x37560, 0x3759c,
4023                 0x375f0, 0x376e4,
4024                 0x376f8, 0x377e4,
4025                 0x377f8, 0x377fc,
4026                 0x37814, 0x37814,
4027                 0x3782c, 0x3782c,
4028                 0x37880, 0x3788c,
4029                 0x378e8, 0x378ec,
4030                 0x37900, 0x37948,
4031                 0x37960, 0x3799c,
4032                 0x379f0, 0x37ae4,
4033                 0x37af8, 0x37b10,
4034                 0x37b28, 0x37b28,
4035                 0x37b3c, 0x37b50,
4036                 0x37bf0, 0x37c10,
4037                 0x37c28, 0x37c28,
4038                 0x37c3c, 0x37c50,
4039                 0x37cf0, 0x37cfc,
4040                 0x38000, 0x38030,
4041                 0x38100, 0x38144,
4042                 0x38190, 0x381d0,
4043                 0x38200, 0x38318,
4044                 0x38400, 0x3852c,
4045                 0x38540, 0x3861c,
4046                 0x38800, 0x38834,
4047                 0x388c0, 0x38908,
4048                 0x38910, 0x389ac,
4049                 0x38a00, 0x38a2c,
4050                 0x38a44, 0x38a50,
4051                 0x38a74, 0x38c24,
4052                 0x38d00, 0x38d00,
4053                 0x38d08, 0x38d14,
4054                 0x38d1c, 0x38d20,
4055                 0x38d3c, 0x38d50,
4056                 0x39200, 0x3920c,
4057                 0x39220, 0x39220,
4058                 0x39240, 0x39240,
4059                 0x39600, 0x3960c,
4060                 0x39a00, 0x39a1c,
4061                 0x39e00, 0x39e20,
4062                 0x39e38, 0x39e3c,
4063                 0x39e80, 0x39e80,
4064                 0x39e88, 0x39ea8,
4065                 0x39eb0, 0x39eb4,
4066                 0x39ec8, 0x39ed4,
4067                 0x39fb8, 0x3a004,
4068                 0x3a200, 0x3a200,
4069                 0x3a208, 0x3a240,
4070                 0x3a248, 0x3a280,
4071                 0x3a288, 0x3a2c0,
4072                 0x3a2c8, 0x3a2fc,
4073                 0x3a600, 0x3a630,
4074                 0x3aa00, 0x3aabc,
4075                 0x3ab00, 0x3ab70,
4076                 0x3b000, 0x3b048,
4077                 0x3b060, 0x3b09c,
4078                 0x3b0f0, 0x3b148,
4079                 0x3b160, 0x3b19c,
4080                 0x3b1f0, 0x3b2e4,
4081                 0x3b2f8, 0x3b3e4,
4082                 0x3b3f8, 0x3b448,
4083                 0x3b460, 0x3b49c,
4084                 0x3b4f0, 0x3b548,
4085                 0x3b560, 0x3b59c,
4086                 0x3b5f0, 0x3b6e4,
4087                 0x3b6f8, 0x3b7e4,
4088                 0x3b7f8, 0x3b7fc,
4089                 0x3b814, 0x3b814,
4090                 0x3b82c, 0x3b82c,
4091                 0x3b880, 0x3b88c,
4092                 0x3b8e8, 0x3b8ec,
4093                 0x3b900, 0x3b948,
4094                 0x3b960, 0x3b99c,
4095                 0x3b9f0, 0x3bae4,
4096                 0x3baf8, 0x3bb10,
4097                 0x3bb28, 0x3bb28,
4098                 0x3bb3c, 0x3bb50,
4099                 0x3bbf0, 0x3bc10,
4100                 0x3bc28, 0x3bc28,
4101                 0x3bc3c, 0x3bc50,
4102                 0x3bcf0, 0x3bcfc,
4103                 0x3c000, 0x3c030,
4104                 0x3c100, 0x3c144,
4105                 0x3c190, 0x3c1d0,
4106                 0x3c200, 0x3c318,
4107                 0x3c400, 0x3c52c,
4108                 0x3c540, 0x3c61c,
4109                 0x3c800, 0x3c834,
4110                 0x3c8c0, 0x3c908,
4111                 0x3c910, 0x3c9ac,
4112                 0x3ca00, 0x3ca2c,
4113                 0x3ca44, 0x3ca50,
4114                 0x3ca74, 0x3cc24,
4115                 0x3cd00, 0x3cd00,
4116                 0x3cd08, 0x3cd14,
4117                 0x3cd1c, 0x3cd20,
4118                 0x3cd3c, 0x3cd50,
4119                 0x3d200, 0x3d20c,
4120                 0x3d220, 0x3d220,
4121                 0x3d240, 0x3d240,
4122                 0x3d600, 0x3d60c,
4123                 0x3da00, 0x3da1c,
4124                 0x3de00, 0x3de20,
4125                 0x3de38, 0x3de3c,
4126                 0x3de80, 0x3de80,
4127                 0x3de88, 0x3dea8,
4128                 0x3deb0, 0x3deb4,
4129                 0x3dec8, 0x3ded4,
4130                 0x3dfb8, 0x3e004,
4131                 0x3e200, 0x3e200,
4132                 0x3e208, 0x3e240,
4133                 0x3e248, 0x3e280,
4134                 0x3e288, 0x3e2c0,
4135                 0x3e2c8, 0x3e2fc,
4136                 0x3e600, 0x3e630,
4137                 0x3ea00, 0x3eabc,
4138                 0x3eb00, 0x3eb70,
4139                 0x3f000, 0x3f048,
4140                 0x3f060, 0x3f09c,
4141                 0x3f0f0, 0x3f148,
4142                 0x3f160, 0x3f19c,
4143                 0x3f1f0, 0x3f2e4,
4144                 0x3f2f8, 0x3f3e4,
4145                 0x3f3f8, 0x3f448,
4146                 0x3f460, 0x3f49c,
4147                 0x3f4f0, 0x3f548,
4148                 0x3f560, 0x3f59c,
4149                 0x3f5f0, 0x3f6e4,
4150                 0x3f6f8, 0x3f7e4,
4151                 0x3f7f8, 0x3f7fc,
4152                 0x3f814, 0x3f814,
4153                 0x3f82c, 0x3f82c,
4154                 0x3f880, 0x3f88c,
4155                 0x3f8e8, 0x3f8ec,
4156                 0x3f900, 0x3f948,
4157                 0x3f960, 0x3f99c,
4158                 0x3f9f0, 0x3fae4,
4159                 0x3faf8, 0x3fb10,
4160                 0x3fb28, 0x3fb28,
4161                 0x3fb3c, 0x3fb50,
4162                 0x3fbf0, 0x3fc10,
4163                 0x3fc28, 0x3fc28,
4164                 0x3fc3c, 0x3fc50,
4165                 0x3fcf0, 0x3fcfc,
4166                 0x40000, 0x4000c,
4167                 0x40040, 0x40068,
4168                 0x4007c, 0x40144,
4169                 0x40180, 0x4018c,
4170                 0x40200, 0x40298,
4171                 0x402ac, 0x4033c,
4172                 0x403f8, 0x403fc,
4173                 0x41304, 0x413c4,
4174                 0x41400, 0x4141c,
4175                 0x41480, 0x414d0,
4176                 0x44000, 0x44078,
4177                 0x440c0, 0x44278,
4178                 0x442c0, 0x44478,
4179                 0x444c0, 0x44678,
4180                 0x446c0, 0x44878,
4181                 0x448c0, 0x449fc,
4182                 0x45000, 0x45068,
4183                 0x45080, 0x45084,
4184                 0x450a0, 0x450b0,
4185                 0x45200, 0x45268,
4186                 0x45280, 0x45284,
4187                 0x452a0, 0x452b0,
4188                 0x460c0, 0x460e4,
4189                 0x47000, 0x4708c,
4190                 0x47200, 0x47250,
4191                 0x47400, 0x47420,
4192                 0x47600, 0x47618,
4193                 0x47800, 0x47814,
4194                 0x48000, 0x4800c,
4195                 0x48040, 0x48068,
4196                 0x4807c, 0x48144,
4197                 0x48180, 0x4818c,
4198                 0x48200, 0x48298,
4199                 0x482ac, 0x4833c,
4200                 0x483f8, 0x483fc,
4201                 0x49304, 0x493c4,
4202                 0x49400, 0x4941c,
4203                 0x49480, 0x494d0,
4204                 0x4c000, 0x4c078,
4205                 0x4c0c0, 0x4c278,
4206                 0x4c2c0, 0x4c478,
4207                 0x4c4c0, 0x4c678,
4208                 0x4c6c0, 0x4c878,
4209                 0x4c8c0, 0x4c9fc,
4210                 0x4d000, 0x4d068,
4211                 0x4d080, 0x4d084,
4212                 0x4d0a0, 0x4d0b0,
4213                 0x4d200, 0x4d268,
4214                 0x4d280, 0x4d284,
4215                 0x4d2a0, 0x4d2b0,
4216                 0x4e0c0, 0x4e0e4,
4217                 0x4f000, 0x4f08c,
4218                 0x4f200, 0x4f250,
4219                 0x4f400, 0x4f420,
4220                 0x4f600, 0x4f618,
4221                 0x4f800, 0x4f814,
4222                 0x50000, 0x500cc,
4223                 0x50400, 0x50400,
4224                 0x50800, 0x508cc,
4225                 0x50c00, 0x50c00,
4226                 0x51000, 0x5101c,
4227                 0x51300, 0x51308,
4228         };
4229
4230         if (is_t4(sc)) {
4231                 reg_ranges = &t4_reg_ranges[0];
4232                 n = nitems(t4_reg_ranges);
4233         } else {
4234                 reg_ranges = &t5_reg_ranges[0];
4235                 n = nitems(t5_reg_ranges);
4236         }
4237
4238         regs->version = chip_id(sc) | chip_rev(sc) << 10;
4239         for (i = 0; i < n; i += 2)
4240                 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
4241 }
4242
4243 static void
4244 cxgbe_tick(void *arg)
4245 {
4246         struct port_info *pi = arg;
4247         struct adapter *sc = pi->adapter;
4248         struct ifnet *ifp = pi->ifp;
4249         struct sge_txq *txq;
4250         int i, drops;
4251         struct port_stats *s = &pi->stats;
4252
4253         PORT_LOCK(pi);
4254         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4255                 PORT_UNLOCK(pi);
4256                 return; /* without scheduling another callout */
4257         }
4258
4259         t4_get_port_stats(sc, pi->tx_chan, s);
4260
4261         ifp->if_opackets = s->tx_frames - s->tx_pause;
4262         ifp->if_ipackets = s->rx_frames - s->rx_pause;
4263         ifp->if_obytes = s->tx_octets - s->tx_pause * 64;
4264         ifp->if_ibytes = s->rx_octets - s->rx_pause * 64;
4265         ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause;
4266         ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause;
4267         ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
4268             s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
4269             s->rx_trunc3;
4270         for (i = 0; i < 4; i++) {
4271                 if (pi->rx_chan_map & (1 << i)) {
4272                         uint32_t v;
4273
4274                         /*
4275                          * XXX: indirect reads from the same ADDR/DATA pair can
4276                          * race with each other.
4277                          */
4278                         t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
4279                             1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4280                         ifp->if_iqdrops += v;
4281                 }
4282         }
4283
4284         drops = s->tx_drop;
4285         for_each_txq(pi, i, txq)
4286                 drops += txq->br->br_drops;
4287         ifp->if_snd.ifq_drops = drops;
4288
4289         ifp->if_oerrors = s->tx_error_frames;
4290         ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long +
4291             s->rx_fcs_err + s->rx_len_err;
4292
4293         callout_schedule(&pi->tick, hz);
4294         PORT_UNLOCK(pi);
4295 }
4296
4297 static void
4298 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4299 {
4300         struct ifnet *vlan;
4301
4302         if (arg != ifp || ifp->if_type != IFT_ETHER)
4303                 return;
4304
4305         vlan = VLAN_DEVAT(ifp, vid);
4306         VLAN_SETCOOKIE(vlan, ifp);
4307 }
4308
4309 static int
4310 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4311 {
4312
4313 #ifdef INVARIANTS
4314         panic("%s: opcode 0x%02x on iq %p with payload %p",
4315             __func__, rss->opcode, iq, m);
4316 #else
4317         log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
4318             __func__, rss->opcode, iq, m);
4319         m_freem(m);
4320 #endif
4321         return (EDOOFUS);
4322 }
4323
4324 int
4325 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
4326 {
4327         uintptr_t *loc, new;
4328
4329         if (opcode >= nitems(sc->cpl_handler))
4330                 return (EINVAL);
4331
4332         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
4333         loc = (uintptr_t *) &sc->cpl_handler[opcode];
4334         atomic_store_rel_ptr(loc, new);
4335
4336         return (0);
4337 }
4338
4339 static int
4340 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
4341 {
4342
4343 #ifdef INVARIANTS
4344         panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
4345 #else
4346         log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
4347             __func__, iq, ctrl);
4348 #endif
4349         return (EDOOFUS);
4350 }
4351
4352 int
4353 t4_register_an_handler(struct adapter *sc, an_handler_t h)
4354 {
4355         uintptr_t *loc, new;
4356
4357         new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
4358         loc = (uintptr_t *) &sc->an_handler;
4359         atomic_store_rel_ptr(loc, new);
4360
4361         return (0);
4362 }
4363
4364 static int
4365 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
4366 {
4367         const struct cpl_fw6_msg *cpl =
4368             __containerof(rpl, struct cpl_fw6_msg, data[0]);
4369
4370 #ifdef INVARIANTS
4371         panic("%s: fw_msg type %d", __func__, cpl->type);
4372 #else
4373         log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
4374 #endif
4375         return (EDOOFUS);
4376 }
4377
4378 int
4379 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
4380 {
4381         uintptr_t *loc, new;
4382
4383         if (type >= nitems(sc->fw_msg_handler))
4384                 return (EINVAL);
4385
4386         /*
4387          * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
4388          * handler dispatch table.  Reject any attempt to install a handler for
4389          * this subtype.
4390          */
4391         if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
4392                 return (EINVAL);
4393
4394         new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
4395         loc = (uintptr_t *) &sc->fw_msg_handler[type];
4396         atomic_store_rel_ptr(loc, new);
4397
4398         return (0);
4399 }
4400
4401 static int
4402 t4_sysctls(struct adapter *sc)
4403 {
4404         struct sysctl_ctx_list *ctx;
4405         struct sysctl_oid *oid;
4406         struct sysctl_oid_list *children, *c0;
4407         static char *caps[] = {
4408                 "\20\1PPP\2QFC\3DCBX",                  /* caps[0] linkcaps */
4409                 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL"        /* caps[1] niccaps */
4410                     "\6HASHFILTER\7ETHOFLD",
4411                 "\20\1TOE",                             /* caps[2] toecaps */
4412                 "\20\1RDDP\2RDMAC",                     /* caps[3] rdmacaps */
4413                 "\20\1INITIATOR_PDU\2TARGET_PDU"        /* caps[4] iscsicaps */
4414                     "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
4415                     "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
4416                 "\20\1INITIATOR\2TARGET\3CTRL_OFLD"     /* caps[5] fcoecaps */
4417                     "\4PO_INITIAOR\5PO_TARGET"
4418         };
4419         static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
4420
4421         ctx = device_get_sysctl_ctx(sc->dev);
4422
4423         /*
4424          * dev.t4nex.X.
4425          */
4426         oid = device_get_sysctl_tree(sc->dev);
4427         c0 = children = SYSCTL_CHILDREN(oid);
4428
4429         sc->sc_do_rxcopy = 1;
4430         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
4431             &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
4432
4433         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
4434             sc->params.nports, "# of ports");
4435
4436         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
4437             NULL, chip_rev(sc), "chip hardware revision");
4438
4439         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
4440             CTLFLAG_RD, sc->fw_version, 0, "firmware version");
4441
4442         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
4443             CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
4444
4445         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
4446             sc->cfcsum, "config file checksum");
4447
4448         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
4449             CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
4450             sysctl_bitfield, "A", "available doorbells");
4451
4452         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
4453             CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
4454             sysctl_bitfield, "A", "available link capabilities");
4455
4456         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
4457             CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
4458             sysctl_bitfield, "A", "available NIC capabilities");
4459
4460         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
4461             CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
4462             sysctl_bitfield, "A", "available TCP offload capabilities");
4463
4464         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
4465             CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
4466             sysctl_bitfield, "A", "available RDMA capabilities");
4467
4468         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
4469             CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
4470             sysctl_bitfield, "A", "available iSCSI capabilities");
4471
4472         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
4473             CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
4474             sysctl_bitfield, "A", "available FCoE capabilities");
4475
4476         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
4477             sc->params.vpd.cclk, "core clock frequency (in KHz)");
4478
4479         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
4480             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
4481             sizeof(sc->sge.timer_val), sysctl_int_array, "A",
4482             "interrupt holdoff timer values (us)");
4483
4484         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
4485             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
4486             sizeof(sc->sge.counter_val), sysctl_int_array, "A",
4487             "interrupt holdoff packet counter values");
4488
4489         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
4490             NULL, sc->tids.nftids, "number of filters");
4491
4492         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
4493             CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
4494             "chip temperature (in Celsius)");
4495
4496         t4_sge_sysctls(sc, ctx, children);
4497
4498         sc->lro_timeout = 100;
4499         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
4500             &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
4501
4502 #ifdef SBUF_DRAIN
4503         /*
4504          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
4505          */
4506         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
4507             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
4508             "logs and miscellaneous information");
4509         children = SYSCTL_CHILDREN(oid);
4510
4511         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
4512             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4513             sysctl_cctrl, "A", "congestion control");
4514
4515         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
4516             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4517             sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
4518
4519         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
4520             CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
4521             sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
4522
4523         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
4524             CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
4525             sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
4526
4527         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
4528             CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
4529             sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
4530
4531         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
4532             CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
4533             sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
4534
4535         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
4536             CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
4537             sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
4538
4539         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
4540             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4541             sysctl_cim_la, "A", "CIM logic analyzer");
4542
4543         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
4544             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4545             sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
4546
4547         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
4548             CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
4549             sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
4550
4551         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
4552             CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
4553             sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
4554
4555         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
4556             CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
4557             sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
4558
4559         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
4560             CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
4561             sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
4562
4563         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
4564             CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
4565             sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
4566
4567         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
4568             CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
4569             sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
4570
4571         if (is_t5(sc)) {
4572                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
4573                     CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
4574                     sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
4575
4576                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
4577                     CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
4578                     sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
4579         }
4580
4581         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
4582             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4583             sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
4584
4585         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
4586             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4587             sysctl_cim_qcfg, "A", "CIM queue configuration");
4588
4589         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
4590             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4591             sysctl_cpl_stats, "A", "CPL statistics");
4592
4593         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
4594             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4595             sysctl_ddp_stats, "A", "non-TCP DDP statistics");
4596
4597         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
4598             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4599             sysctl_devlog, "A", "firmware's device log");
4600
4601         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
4602             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4603             sysctl_fcoe_stats, "A", "FCoE statistics");
4604
4605         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
4606             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4607             sysctl_hw_sched, "A", "hardware scheduler ");
4608
4609         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
4610             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4611             sysctl_l2t, "A", "hardware L2 table");
4612
4613         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
4614             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4615             sysctl_lb_stats, "A", "loopback statistics");
4616
4617         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
4618             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4619             sysctl_meminfo, "A", "memory regions");
4620
4621         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
4622             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4623             sysctl_mps_tcam, "A", "MPS TCAM entries");
4624
4625         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
4626             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4627             sysctl_path_mtus, "A", "path MTUs");
4628
4629         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
4630             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4631             sysctl_pm_stats, "A", "PM statistics");
4632
4633         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
4634             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4635             sysctl_rdma_stats, "A", "RDMA statistics");
4636
4637         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
4638             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4639             sysctl_tcp_stats, "A", "TCP statistics");
4640
4641         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
4642             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4643             sysctl_tids, "A", "TID information");
4644
4645         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
4646             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4647             sysctl_tp_err_stats, "A", "TP error statistics");
4648
4649         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
4650             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4651             sysctl_tp_la, "A", "TP logic analyzer");
4652
4653         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
4654             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4655             sysctl_tx_rate, "A", "Tx rate");
4656
4657         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
4658             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4659             sysctl_ulprx_la, "A", "ULPRX logic analyzer");
4660
4661         if (is_t5(sc)) {
4662                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
4663                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4664                     sysctl_wcwr_stats, "A", "write combined work requests");
4665         }
4666 #endif
4667
4668 #ifdef TCP_OFFLOAD
4669         if (is_offload(sc)) {
4670                 /*
4671                  * dev.t4nex.X.toe.
4672                  */
4673                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
4674                     NULL, "TOE parameters");
4675                 children = SYSCTL_CHILDREN(oid);
4676
4677                 sc->tt.sndbuf = 256 * 1024;
4678                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
4679                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
4680
4681                 sc->tt.ddp = 0;
4682                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
4683                     &sc->tt.ddp, 0, "DDP allowed");
4684
4685                 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
4686                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
4687                     &sc->tt.indsz, 0, "DDP max indicate size allowed");
4688
4689                 sc->tt.ddp_thres =
4690                     G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
4691                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
4692                     &sc->tt.ddp_thres, 0, "DDP threshold");
4693
4694                 sc->tt.rx_coalesce = 1;
4695                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
4696                     CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
4697
4698                 sc->tt.tx_align = 1;
4699                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align",
4700                     CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload");
4701         }
4702 #endif
4703
4704
4705         return (0);
4706 }
4707
4708 static int
4709 cxgbe_sysctls(struct port_info *pi)
4710 {
4711         struct sysctl_ctx_list *ctx;
4712         struct sysctl_oid *oid;
4713         struct sysctl_oid_list *children;
4714         struct adapter *sc = pi->adapter;
4715
4716         ctx = device_get_sysctl_ctx(pi->dev);
4717
4718         /*
4719          * dev.cxgbe.X.
4720          */
4721         oid = device_get_sysctl_tree(pi->dev);
4722         children = SYSCTL_CHILDREN(oid);
4723
4724         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
4725            CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
4726         if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
4727                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
4728                     CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
4729                     "PHY temperature (in Celsius)");
4730                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
4731                     CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
4732                     "PHY firmware version");
4733         }
4734         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
4735             &pi->nrxq, 0, "# of rx queues");
4736         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
4737             &pi->ntxq, 0, "# of tx queues");
4738         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
4739             &pi->first_rxq, 0, "index of first rx queue");
4740         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
4741             &pi->first_txq, 0, "index of first tx queue");
4742         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
4743             CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU",
4744             "Reserve queue 0 for non-flowid packets");
4745
4746 #ifdef TCP_OFFLOAD
4747         if (is_offload(sc)) {
4748                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
4749                     &pi->nofldrxq, 0,
4750                     "# of rx queues for offloaded TCP connections");
4751                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
4752                     &pi->nofldtxq, 0,
4753                     "# of tx queues for offloaded TCP connections");
4754                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
4755                     CTLFLAG_RD, &pi->first_ofld_rxq, 0,
4756                     "index of first TOE rx queue");
4757                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
4758                     CTLFLAG_RD, &pi->first_ofld_txq, 0,
4759                     "index of first TOE tx queue");
4760         }
4761 #endif
4762 #ifdef DEV_NETMAP
4763         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
4764             &pi->nnmrxq, 0, "# of rx queues for netmap");
4765         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
4766             &pi->nnmtxq, 0, "# of tx queues for netmap");
4767         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
4768             CTLFLAG_RD, &pi->first_nm_rxq, 0,
4769             "index of first netmap rx queue");
4770         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
4771             CTLFLAG_RD, &pi->first_nm_txq, 0,
4772             "index of first netmap tx queue");
4773 #endif
4774
4775         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
4776             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
4777             "holdoff timer index");
4778         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
4779             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
4780             "holdoff packet counter index");
4781
4782         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
4783             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
4784             "rx queue size");
4785         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
4786             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
4787             "tx queue size");
4788
4789         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
4790             CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings,
4791             "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
4792
4793         /*
4794          * dev.cxgbe.X.stats.
4795          */
4796         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
4797             NULL, "port statistics");
4798         children = SYSCTL_CHILDREN(oid);
4799
4800 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
4801         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
4802             CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
4803             sysctl_handle_t4_reg64, "QU", desc)
4804
4805         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
4806             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
4807         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
4808             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
4809         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
4810             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
4811         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
4812             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
4813         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
4814             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
4815         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
4816             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
4817         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
4818             "# of tx frames in this range",
4819             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
4820         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
4821             "# of tx frames in this range",
4822             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
4823         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
4824             "# of tx frames in this range",
4825             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
4826         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
4827             "# of tx frames in this range",
4828             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
4829         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
4830             "# of tx frames in this range",
4831             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
4832         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
4833             "# of tx frames in this range",
4834             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
4835         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
4836             "# of tx frames in this range",
4837             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
4838         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
4839             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
4840         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
4841             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
4842         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
4843             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
4844         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
4845             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
4846         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
4847             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
4848         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
4849             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
4850         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
4851             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
4852         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
4853             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
4854         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
4855             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
4856         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
4857             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
4858
4859         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
4860             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
4861         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
4862             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
4863         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
4864             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
4865         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
4866             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
4867         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
4868             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
4869         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
4870             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
4871         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
4872             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
4873         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
4874             "# of frames received with bad FCS",
4875             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
4876         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
4877             "# of frames received with length error",
4878             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
4879         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
4880             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
4881         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
4882             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
4883         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
4884             "# of rx frames in this range",
4885             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
4886         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
4887             "# of rx frames in this range",
4888             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
4889         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
4890             "# of rx frames in this range",
4891             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
4892         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
4893             "# of rx frames in this range",
4894             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
4895         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
4896             "# of rx frames in this range",
4897             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
4898         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
4899             "# of rx frames in this range",
4900             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
4901         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
4902             "# of rx frames in this range",
4903             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
4904         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
4905             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
4906         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
4907             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
4908         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
4909             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
4910         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
4911             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
4912         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
4913             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
4914         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
4915             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
4916         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
4917             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
4918         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
4919             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
4920         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
4921             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
4922
4923 #undef SYSCTL_ADD_T4_REG64
4924
4925 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
4926         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
4927             &pi->stats.name, desc)
4928
4929         /* We get these from port_stats and they may be stale by upto 1s */
4930         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
4931             "# drops due to buffer-group 0 overflows");
4932         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
4933             "# drops due to buffer-group 1 overflows");
4934         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
4935             "# drops due to buffer-group 2 overflows");
4936         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
4937             "# drops due to buffer-group 3 overflows");
4938         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
4939             "# of buffer-group 0 truncated packets");
4940         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
4941             "# of buffer-group 1 truncated packets");
4942         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
4943             "# of buffer-group 2 truncated packets");
4944         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
4945             "# of buffer-group 3 truncated packets");
4946
4947 #undef SYSCTL_ADD_T4_PORTSTAT
4948
4949         return (0);
4950 }
4951
4952 static int
4953 sysctl_int_array(SYSCTL_HANDLER_ARGS)
4954 {
4955         int rc, *i, space = 0;
4956         struct sbuf sb;
4957
4958         sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
4959         for (i = arg1; arg2; arg2 -= sizeof(int), i++) {
4960                 if (space)
4961                         sbuf_printf(&sb, " ");
4962                 sbuf_printf(&sb, "%d", *i);
4963                 space = 1;
4964         }
4965         sbuf_finish(&sb);
4966         rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
4967         sbuf_delete(&sb);
4968         return (rc);
4969 }
4970
4971 static int
4972 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
4973 {
4974         int rc;
4975         struct sbuf *sb;
4976
4977         rc = sysctl_wire_old_buffer(req, 0);
4978         if (rc != 0)
4979                 return(rc);
4980
4981         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
4982         if (sb == NULL)
4983                 return (ENOMEM);
4984
4985         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
4986         rc = sbuf_finish(sb);
4987         sbuf_delete(sb);
4988
4989         return (rc);
4990 }
4991
4992 static int
4993 sysctl_btphy(SYSCTL_HANDLER_ARGS)
4994 {
4995         struct port_info *pi = arg1;
4996         int op = arg2;
4997         struct adapter *sc = pi->adapter;
4998         u_int v;
4999         int rc;
5000
5001         rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt");
5002         if (rc)
5003                 return (rc);
5004         /* XXX: magic numbers */
5005         rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
5006             &v);
5007         end_synchronized_op(sc, 0);
5008         if (rc)
5009                 return (rc);
5010         if (op == 0)
5011                 v /= 256;
5012
5013         rc = sysctl_handle_int(oidp, &v, 0, req);
5014         return (rc);
5015 }
5016
5017 static int
5018 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
5019 {
5020         struct port_info *pi = arg1;
5021         int rc, val;
5022
5023         val = pi->rsrv_noflowq;
5024         rc = sysctl_handle_int(oidp, &val, 0, req);
5025         if (rc != 0 || req->newptr == NULL)
5026                 return (rc);
5027
5028         if ((val >= 1) && (pi->ntxq > 1))
5029                 pi->rsrv_noflowq = 1;
5030         else
5031                 pi->rsrv_noflowq = 0;
5032
5033         return (rc);
5034 }
5035
5036 static int
5037 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
5038 {
5039         struct port_info *pi = arg1;
5040         struct adapter *sc = pi->adapter;
5041         int idx, rc, i;
5042         struct sge_rxq *rxq;
5043 #ifdef TCP_OFFLOAD
5044         struct sge_ofld_rxq *ofld_rxq;
5045 #endif
5046         uint8_t v;
5047
5048         idx = pi->tmr_idx;
5049
5050         rc = sysctl_handle_int(oidp, &idx, 0, req);
5051         if (rc != 0 || req->newptr == NULL)
5052                 return (rc);
5053
5054         if (idx < 0 || idx >= SGE_NTIMERS)
5055                 return (EINVAL);
5056
5057         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5058             "t4tmr");
5059         if (rc)
5060                 return (rc);
5061
5062         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
5063         for_each_rxq(pi, i, rxq) {
5064 #ifdef atomic_store_rel_8
5065                 atomic_store_rel_8(&rxq->iq.intr_params, v);
5066 #else
5067                 rxq->iq.intr_params = v;
5068 #endif
5069         }
5070 #ifdef TCP_OFFLOAD
5071         for_each_ofld_rxq(pi, i, ofld_rxq) {
5072 #ifdef atomic_store_rel_8
5073                 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
5074 #else
5075                 ofld_rxq->iq.intr_params = v;
5076 #endif
5077         }
5078 #endif
5079         pi->tmr_idx = idx;
5080
5081         end_synchronized_op(sc, LOCK_HELD);
5082         return (0);
5083 }
5084
5085 static int
5086 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
5087 {
5088         struct port_info *pi = arg1;
5089         struct adapter *sc = pi->adapter;
5090         int idx, rc;
5091
5092         idx = pi->pktc_idx;
5093
5094         rc = sysctl_handle_int(oidp, &idx, 0, req);
5095         if (rc != 0 || req->newptr == NULL)
5096                 return (rc);
5097
5098         if (idx < -1 || idx >= SGE_NCOUNTERS)
5099                 return (EINVAL);
5100
5101         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5102             "t4pktc");
5103         if (rc)
5104                 return (rc);
5105
5106         if (pi->flags & PORT_INIT_DONE)
5107                 rc = EBUSY; /* cannot be changed once the queues are created */
5108         else
5109                 pi->pktc_idx = idx;
5110
5111         end_synchronized_op(sc, LOCK_HELD);
5112         return (rc);
5113 }
5114
5115 static int
5116 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
5117 {
5118         struct port_info *pi = arg1;
5119         struct adapter *sc = pi->adapter;
5120         int qsize, rc;
5121
5122         qsize = pi->qsize_rxq;
5123
5124         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5125         if (rc != 0 || req->newptr == NULL)
5126                 return (rc);
5127
5128         if (qsize < 128 || (qsize & 7))
5129                 return (EINVAL);
5130
5131         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5132             "t4rxqs");
5133         if (rc)
5134                 return (rc);
5135
5136         if (pi->flags & PORT_INIT_DONE)
5137                 rc = EBUSY; /* cannot be changed once the queues are created */
5138         else
5139                 pi->qsize_rxq = qsize;
5140
5141         end_synchronized_op(sc, LOCK_HELD);
5142         return (rc);
5143 }
5144
5145 static int
5146 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
5147 {
5148         struct port_info *pi = arg1;
5149         struct adapter *sc = pi->adapter;
5150         int qsize, rc;
5151
5152         qsize = pi->qsize_txq;
5153
5154         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5155         if (rc != 0 || req->newptr == NULL)
5156                 return (rc);
5157
5158         /* bufring size must be powerof2 */
5159         if (qsize < 128 || !powerof2(qsize))
5160                 return (EINVAL);
5161
5162         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5163             "t4txqs");
5164         if (rc)
5165                 return (rc);
5166
5167         if (pi->flags & PORT_INIT_DONE)
5168                 rc = EBUSY; /* cannot be changed once the queues are created */
5169         else
5170                 pi->qsize_txq = qsize;
5171
5172         end_synchronized_op(sc, LOCK_HELD);
5173         return (rc);
5174 }
5175
5176 static int
5177 sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
5178 {
5179         struct port_info *pi = arg1;
5180         struct adapter *sc = pi->adapter;
5181         struct link_config *lc = &pi->link_cfg;
5182         int rc;
5183
5184         if (req->newptr == NULL) {
5185                 struct sbuf *sb;
5186                 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX";
5187
5188                 rc = sysctl_wire_old_buffer(req, 0);
5189                 if (rc != 0)
5190                         return(rc);
5191
5192                 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5193                 if (sb == NULL)
5194                         return (ENOMEM);
5195
5196                 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits);
5197                 rc = sbuf_finish(sb);
5198                 sbuf_delete(sb);
5199         } else {
5200                 char s[2];
5201                 int n;
5202
5203                 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX));
5204                 s[1] = 0;
5205
5206                 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
5207                 if (rc != 0)
5208                         return(rc);
5209
5210                 if (s[1] != 0)
5211                         return (EINVAL);
5212                 if (s[0] < '0' || s[0] > '9')
5213                         return (EINVAL);        /* not a number */
5214                 n = s[0] - '0';
5215                 if (n & ~(PAUSE_TX | PAUSE_RX))
5216                         return (EINVAL);        /* some other bit is set too */
5217
5218                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE");
5219                 if (rc)
5220                         return (rc);
5221                 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) {
5222                         int link_ok = lc->link_ok;
5223
5224                         lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX);
5225                         lc->requested_fc |= n;
5226                         rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc);
5227                         lc->link_ok = link_ok;  /* restore */
5228                 }
5229                 end_synchronized_op(sc, 0);
5230         }
5231
5232         return (rc);
5233 }
5234
5235 static int
5236 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
5237 {
5238         struct adapter *sc = arg1;
5239         int reg = arg2;
5240         uint64_t val;
5241
5242         val = t4_read_reg64(sc, reg);
5243
5244         return (sysctl_handle_64(oidp, &val, 0, req));
5245 }
5246
5247 static int
5248 sysctl_temperature(SYSCTL_HANDLER_ARGS)
5249 {
5250         struct adapter *sc = arg1;
5251         int rc, t;
5252         uint32_t param, val;
5253
5254         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
5255         if (rc)
5256                 return (rc);
5257         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5258             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5259             V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
5260         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
5261         end_synchronized_op(sc, 0);
5262         if (rc)
5263                 return (rc);
5264
5265         /* unknown is returned as 0 but we display -1 in that case */
5266         t = val == 0 ? -1 : val;
5267
5268         rc = sysctl_handle_int(oidp, &t, 0, req);
5269         return (rc);
5270 }
5271
5272 #ifdef SBUF_DRAIN
5273 static int
5274 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
5275 {
5276         struct adapter *sc = arg1;
5277         struct sbuf *sb;
5278         int rc, i;
5279         uint16_t incr[NMTUS][NCCTRL_WIN];
5280         static const char *dec_fac[] = {
5281                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
5282                 "0.9375"
5283         };
5284
5285         rc = sysctl_wire_old_buffer(req, 0);
5286         if (rc != 0)
5287                 return (rc);
5288
5289         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5290         if (sb == NULL)
5291                 return (ENOMEM);
5292
5293         t4_read_cong_tbl(sc, incr);
5294
5295         for (i = 0; i < NCCTRL_WIN; ++i) {
5296                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
5297                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
5298                     incr[5][i], incr[6][i], incr[7][i]);
5299                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
5300                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
5301                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
5302                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
5303         }
5304
5305         rc = sbuf_finish(sb);
5306         sbuf_delete(sb);
5307
5308         return (rc);
5309 }
5310
5311 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
5312         "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",   /* ibq's */
5313         "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
5314         "SGE0-RX", "SGE1-RX"    /* additional obq's (T5 onwards) */
5315 };
5316
5317 static int
5318 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
5319 {
5320         struct adapter *sc = arg1;
5321         struct sbuf *sb;
5322         int rc, i, n, qid = arg2;
5323         uint32_t *buf, *p;
5324         char *qtype;
5325         u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
5326
5327         KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
5328             ("%s: bad qid %d\n", __func__, qid));
5329
5330         if (qid < CIM_NUM_IBQ) {
5331                 /* inbound queue */
5332                 qtype = "IBQ";
5333                 n = 4 * CIM_IBQ_SIZE;
5334                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5335                 rc = t4_read_cim_ibq(sc, qid, buf, n);
5336         } else {
5337                 /* outbound queue */
5338                 qtype = "OBQ";
5339                 qid -= CIM_NUM_IBQ;
5340                 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
5341                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5342                 rc = t4_read_cim_obq(sc, qid, buf, n);
5343         }
5344
5345         if (rc < 0) {
5346                 rc = -rc;
5347                 goto done;
5348         }
5349         n = rc * sizeof(uint32_t);      /* rc has # of words actually read */
5350
5351         rc = sysctl_wire_old_buffer(req, 0);
5352         if (rc != 0)
5353                 goto done;
5354
5355         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5356         if (sb == NULL) {
5357                 rc = ENOMEM;
5358                 goto done;
5359         }
5360
5361         sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
5362         for (i = 0, p = buf; i < n; i += 16, p += 4)
5363                 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
5364                     p[2], p[3]);
5365
5366         rc = sbuf_finish(sb);
5367         sbuf_delete(sb);
5368 done:
5369         free(buf, M_CXGBE);
5370         return (rc);
5371 }
5372
5373 static int
5374 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
5375 {
5376         struct adapter *sc = arg1;
5377         u_int cfg;
5378         struct sbuf *sb;
5379         uint32_t *buf, *p;
5380         int rc;
5381
5382         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
5383         if (rc != 0)
5384                 return (rc);
5385
5386         rc = sysctl_wire_old_buffer(req, 0);
5387         if (rc != 0)
5388                 return (rc);
5389
5390         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5391         if (sb == NULL)
5392                 return (ENOMEM);
5393
5394         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
5395             M_ZERO | M_WAITOK);
5396
5397         rc = -t4_cim_read_la(sc, buf, NULL);
5398         if (rc != 0)
5399                 goto done;
5400
5401         sbuf_printf(sb, "Status   Data      PC%s",
5402             cfg & F_UPDBGLACAPTPCONLY ? "" :
5403             "     LS0Stat  LS0Addr             LS0Data");
5404
5405         KASSERT((sc->params.cim_la_size & 7) == 0,
5406             ("%s: p will walk off the end of buf", __func__));
5407
5408         for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
5409                 if (cfg & F_UPDBGLACAPTPCONLY) {
5410                         sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
5411                             p[6], p[7]);
5412                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
5413                             (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
5414                             p[4] & 0xff, p[5] >> 8);
5415                         sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
5416                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5417                             p[1] & 0xf, p[2] >> 4);
5418                 } else {
5419                         sbuf_printf(sb,
5420                             "\n  %02x   %x%07x %x%07x %08x %08x "
5421                             "%08x%08x%08x%08x",
5422                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5423                             p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
5424                             p[6], p[7]);
5425                 }
5426         }
5427
5428         rc = sbuf_finish(sb);
5429         sbuf_delete(sb);
5430 done:
5431         free(buf, M_CXGBE);
5432         return (rc);
5433 }
5434
5435 static int
5436 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
5437 {
5438         struct adapter *sc = arg1;
5439         u_int i;
5440         struct sbuf *sb;
5441         uint32_t *buf, *p;
5442         int rc;
5443
5444         rc = sysctl_wire_old_buffer(req, 0);
5445         if (rc != 0)
5446                 return (rc);
5447
5448         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5449         if (sb == NULL)
5450                 return (ENOMEM);
5451
5452         buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
5453             M_ZERO | M_WAITOK);
5454
5455         t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
5456         p = buf;
5457
5458         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5459                 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
5460                     p[1], p[0]);
5461         }
5462
5463         sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
5464         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5465                 sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
5466                     (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
5467                     (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
5468                     (p[1] >> 2) | ((p[2] & 3) << 30),
5469                     (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
5470                     p[0] & 1);
5471         }
5472
5473         rc = sbuf_finish(sb);
5474         sbuf_delete(sb);
5475         free(buf, M_CXGBE);
5476         return (rc);
5477 }
5478
5479 static int
5480 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
5481 {
5482         struct adapter *sc = arg1;
5483         u_int i;
5484         struct sbuf *sb;
5485         uint32_t *buf, *p;
5486         int rc;
5487
5488         rc = sysctl_wire_old_buffer(req, 0);
5489         if (rc != 0)
5490                 return (rc);
5491
5492         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5493         if (sb == NULL)
5494                 return (ENOMEM);
5495
5496         buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
5497             M_ZERO | M_WAITOK);
5498
5499         t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
5500         p = buf;
5501
5502         sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
5503         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5504                 sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
5505                     (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
5506                     p[4], p[3], p[2], p[1], p[0]);
5507         }
5508
5509         sbuf_printf(sb, "\n\nCntl ID               Data");
5510         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5511                 sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
5512                     (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
5513         }
5514
5515         rc = sbuf_finish(sb);
5516         sbuf_delete(sb);
5517         free(buf, M_CXGBE);
5518         return (rc);
5519 }
5520
5521 static int
5522 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
5523 {
5524         struct adapter *sc = arg1;
5525         struct sbuf *sb;
5526         int rc, i;
5527         uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5528         uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5529         uint16_t thres[CIM_NUM_IBQ];
5530         uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
5531         uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
5532         u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
5533
5534         if (is_t4(sc)) {
5535                 cim_num_obq = CIM_NUM_OBQ;
5536                 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
5537                 obq_rdaddr = A_UP_OBQ_0_REALADDR;
5538         } else {
5539                 cim_num_obq = CIM_NUM_OBQ_T5;
5540                 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
5541                 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
5542         }
5543         nq = CIM_NUM_IBQ + cim_num_obq;
5544
5545         rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
5546         if (rc == 0)
5547                 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
5548         if (rc != 0)
5549                 return (rc);
5550
5551         t4_read_cimq_cfg(sc, base, size, thres);
5552
5553         rc = sysctl_wire_old_buffer(req, 0);
5554         if (rc != 0)
5555                 return (rc);
5556
5557         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5558         if (sb == NULL)
5559                 return (ENOMEM);
5560
5561         sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
5562
5563         for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
5564                 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
5565                     qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
5566                     G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5567                     G_QUEREMFLITS(p[2]) * 16);
5568         for ( ; i < nq; i++, p += 4, wr += 2)
5569                 sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
5570                     base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
5571                     wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5572                     G_QUEREMFLITS(p[2]) * 16);
5573
5574         rc = sbuf_finish(sb);
5575         sbuf_delete(sb);
5576
5577         return (rc);
5578 }
5579
5580 static int
5581 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
5582 {
5583         struct adapter *sc = arg1;
5584         struct sbuf *sb;
5585         int rc;
5586         struct tp_cpl_stats stats;
5587
5588         rc = sysctl_wire_old_buffer(req, 0);
5589         if (rc != 0)
5590                 return (rc);
5591
5592         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5593         if (sb == NULL)
5594                 return (ENOMEM);
5595
5596         t4_tp_get_cpl_stats(sc, &stats);
5597
5598         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
5599             "channel 3\n");
5600         sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
5601                    stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
5602         sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
5603                    stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
5604
5605         rc = sbuf_finish(sb);
5606         sbuf_delete(sb);
5607
5608         return (rc);
5609 }
5610
5611 static int
5612 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
5613 {
5614         struct adapter *sc = arg1;
5615         struct sbuf *sb;
5616         int rc;
5617         struct tp_usm_stats stats;
5618
5619         rc = sysctl_wire_old_buffer(req, 0);
5620         if (rc != 0)
5621                 return(rc);
5622
5623         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5624         if (sb == NULL)
5625                 return (ENOMEM);
5626
5627         t4_get_usm_stats(sc, &stats);
5628
5629         sbuf_printf(sb, "Frames: %u\n", stats.frames);
5630         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
5631         sbuf_printf(sb, "Drops:  %u", stats.drops);
5632
5633         rc = sbuf_finish(sb);
5634         sbuf_delete(sb);
5635
5636         return (rc);
5637 }
5638
5639 const char *devlog_level_strings[] = {
5640         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
5641         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
5642         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
5643         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
5644         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
5645         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
5646 };
5647
5648 const char *devlog_facility_strings[] = {
5649         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
5650         [FW_DEVLOG_FACILITY_CF]         = "CF",
5651         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
5652         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
5653         [FW_DEVLOG_FACILITY_RES]        = "RES",
5654         [FW_DEVLOG_FACILITY_HW]         = "HW",
5655         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
5656         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
5657         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
5658         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
5659         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
5660         [FW_DEVLOG_FACILITY_VI]         = "VI",
5661         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
5662         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
5663         [FW_DEVLOG_FACILITY_TM]         = "TM",
5664         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
5665         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
5666         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
5667         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
5668         [FW_DEVLOG_FACILITY_RI]         = "RI",
5669         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
5670         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
5671         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
5672         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE"
5673 };
5674
5675 static int
5676 sysctl_devlog(SYSCTL_HANDLER_ARGS)
5677 {
5678         struct adapter *sc = arg1;
5679         struct devlog_params *dparams = &sc->params.devlog;
5680         struct fw_devlog_e *buf, *e;
5681         int i, j, rc, nentries, first = 0, m;
5682         struct sbuf *sb;
5683         uint64_t ftstamp = UINT64_MAX;
5684
5685         if (dparams->start == 0) {
5686                 dparams->memtype = FW_MEMTYPE_EDC0;
5687                 dparams->start = 0x84000;
5688                 dparams->size = 32768;
5689         }
5690
5691         nentries = dparams->size / sizeof(struct fw_devlog_e);
5692
5693         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
5694         if (buf == NULL)
5695                 return (ENOMEM);
5696
5697         m = fwmtype_to_hwmtype(dparams->memtype);
5698         rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
5699         if (rc != 0)
5700                 goto done;
5701
5702         for (i = 0; i < nentries; i++) {
5703                 e = &buf[i];
5704
5705                 if (e->timestamp == 0)
5706                         break;  /* end */
5707
5708                 e->timestamp = be64toh(e->timestamp);
5709                 e->seqno = be32toh(e->seqno);
5710                 for (j = 0; j < 8; j++)
5711                         e->params[j] = be32toh(e->params[j]);
5712
5713                 if (e->timestamp < ftstamp) {
5714                         ftstamp = e->timestamp;
5715                         first = i;
5716                 }
5717         }
5718
5719         if (buf[first].timestamp == 0)
5720                 goto done;      /* nothing in the log */
5721
5722         rc = sysctl_wire_old_buffer(req, 0);
5723         if (rc != 0)
5724                 goto done;
5725
5726         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5727         if (sb == NULL) {
5728                 rc = ENOMEM;
5729                 goto done;
5730         }
5731         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
5732             "Seq#", "Tstamp", "Level", "Facility", "Message");
5733
5734         i = first;
5735         do {
5736                 e = &buf[i];
5737                 if (e->timestamp == 0)
5738                         break;  /* end */
5739
5740                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
5741                     e->seqno, e->timestamp,
5742                     (e->level < nitems(devlog_level_strings) ?
5743                         devlog_level_strings[e->level] : "UNKNOWN"),
5744                     (e->facility < nitems(devlog_facility_strings) ?
5745                         devlog_facility_strings[e->facility] : "UNKNOWN"));
5746                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
5747                     e->params[2], e->params[3], e->params[4],
5748                     e->params[5], e->params[6], e->params[7]);
5749
5750                 if (++i == nentries)
5751                         i = 0;
5752         } while (i != first);
5753
5754         rc = sbuf_finish(sb);
5755         sbuf_delete(sb);
5756 done:
5757         free(buf, M_CXGBE);
5758         return (rc);
5759 }
5760
5761 static int
5762 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
5763 {
5764         struct adapter *sc = arg1;
5765         struct sbuf *sb;
5766         int rc;
5767         struct tp_fcoe_stats stats[4];
5768
5769         rc = sysctl_wire_old_buffer(req, 0);
5770         if (rc != 0)
5771                 return (rc);
5772
5773         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5774         if (sb == NULL)
5775                 return (ENOMEM);
5776
5777         t4_get_fcoe_stats(sc, 0, &stats[0]);
5778         t4_get_fcoe_stats(sc, 1, &stats[1]);
5779         t4_get_fcoe_stats(sc, 2, &stats[2]);
5780         t4_get_fcoe_stats(sc, 3, &stats[3]);
5781
5782         sbuf_printf(sb, "                   channel 0        channel 1        "
5783             "channel 2        channel 3\n");
5784         sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
5785             stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
5786             stats[3].octetsDDP);
5787         sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
5788             stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
5789         sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
5790             stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
5791             stats[3].framesDrop);
5792
5793         rc = sbuf_finish(sb);
5794         sbuf_delete(sb);
5795
5796         return (rc);
5797 }
5798
5799 static int
5800 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
5801 {
5802         struct adapter *sc = arg1;
5803         struct sbuf *sb;
5804         int rc, i;
5805         unsigned int map, kbps, ipg, mode;
5806         unsigned int pace_tab[NTX_SCHED];
5807
5808         rc = sysctl_wire_old_buffer(req, 0);
5809         if (rc != 0)
5810                 return (rc);
5811
5812         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5813         if (sb == NULL)
5814                 return (ENOMEM);
5815
5816         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
5817         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
5818         t4_read_pace_tbl(sc, pace_tab);
5819
5820         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
5821             "Class IPG (0.1 ns)   Flow IPG (us)");
5822
5823         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
5824                 t4_get_tx_sched(sc, i, &kbps, &ipg);
5825                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
5826                     (mode & (1 << i)) ? "flow" : "class", map & 3);
5827                 if (kbps)
5828                         sbuf_printf(sb, "%9u     ", kbps);
5829                 else
5830                         sbuf_printf(sb, " disabled     ");
5831
5832                 if (ipg)
5833                         sbuf_printf(sb, "%13u        ", ipg);
5834                 else
5835                         sbuf_printf(sb, "     disabled        ");
5836
5837                 if (pace_tab[i])
5838                         sbuf_printf(sb, "%10u", pace_tab[i]);
5839                 else
5840                         sbuf_printf(sb, "  disabled");
5841         }
5842
5843         rc = sbuf_finish(sb);
5844         sbuf_delete(sb);
5845
5846         return (rc);
5847 }
5848
5849 static int
5850 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
5851 {
5852         struct adapter *sc = arg1;
5853         struct sbuf *sb;
5854         int rc, i, j;
5855         uint64_t *p0, *p1;
5856         struct lb_port_stats s[2];
5857         static const char *stat_name[] = {
5858                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
5859                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
5860                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
5861                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
5862                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
5863                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
5864                 "BG2FramesTrunc:", "BG3FramesTrunc:"
5865         };
5866
5867         rc = sysctl_wire_old_buffer(req, 0);
5868         if (rc != 0)
5869                 return (rc);
5870
5871         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5872         if (sb == NULL)
5873                 return (ENOMEM);
5874
5875         memset(s, 0, sizeof(s));
5876
5877         for (i = 0; i < 4; i += 2) {
5878                 t4_get_lb_stats(sc, i, &s[0]);
5879                 t4_get_lb_stats(sc, i + 1, &s[1]);
5880
5881                 p0 = &s[0].octets;
5882                 p1 = &s[1].octets;
5883                 sbuf_printf(sb, "%s                       Loopback %u"
5884                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
5885
5886                 for (j = 0; j < nitems(stat_name); j++)
5887                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
5888                                    *p0++, *p1++);
5889         }
5890
5891         rc = sbuf_finish(sb);
5892         sbuf_delete(sb);
5893
5894         return (rc);
5895 }
5896
5897 static int
5898 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
5899 {
5900         int rc = 0;
5901         struct port_info *pi = arg1;
5902         struct sbuf *sb;
5903         static const char *linkdnreasons[] = {
5904                 "non-specific", "remote fault", "autoneg failed", "reserved3",
5905                 "PHY overheated", "unknown", "rx los", "reserved7"
5906         };
5907
5908         rc = sysctl_wire_old_buffer(req, 0);
5909         if (rc != 0)
5910                 return(rc);
5911         sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
5912         if (sb == NULL)
5913                 return (ENOMEM);
5914
5915         if (pi->linkdnrc < 0)
5916                 sbuf_printf(sb, "n/a");
5917         else if (pi->linkdnrc < nitems(linkdnreasons))
5918                 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
5919         else
5920                 sbuf_printf(sb, "%d", pi->linkdnrc);
5921
5922         rc = sbuf_finish(sb);
5923         sbuf_delete(sb);
5924
5925         return (rc);
5926 }
5927
5928 struct mem_desc {
5929         unsigned int base;
5930         unsigned int limit;
5931         unsigned int idx;
5932 };
5933
5934 static int
5935 mem_desc_cmp(const void *a, const void *b)
5936 {
5937         return ((const struct mem_desc *)a)->base -
5938                ((const struct mem_desc *)b)->base;
5939 }
5940
5941 static void
5942 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
5943     unsigned int to)
5944 {
5945         unsigned int size;
5946
5947         size = to - from + 1;
5948         if (size == 0)
5949                 return;
5950
5951         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
5952         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
5953 }
5954
5955 static int
5956 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
5957 {
5958         struct adapter *sc = arg1;
5959         struct sbuf *sb;
5960         int rc, i, n;
5961         uint32_t lo, hi, used, alloc;
5962         static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
5963         static const char *region[] = {
5964                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
5965                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
5966                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
5967                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
5968                 "RQUDP region:", "PBL region:", "TXPBL region:",
5969                 "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
5970                 "On-chip queues:"
5971         };
5972         struct mem_desc avail[4];
5973         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
5974         struct mem_desc *md = mem;
5975
5976         rc = sysctl_wire_old_buffer(req, 0);
5977         if (rc != 0)
5978                 return (rc);
5979
5980         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5981         if (sb == NULL)
5982                 return (ENOMEM);
5983
5984         for (i = 0; i < nitems(mem); i++) {
5985                 mem[i].limit = 0;
5986                 mem[i].idx = i;
5987         }
5988
5989         /* Find and sort the populated memory ranges */
5990         i = 0;
5991         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
5992         if (lo & F_EDRAM0_ENABLE) {
5993                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
5994                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
5995                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
5996                 avail[i].idx = 0;
5997                 i++;
5998         }
5999         if (lo & F_EDRAM1_ENABLE) {
6000                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
6001                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
6002                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
6003                 avail[i].idx = 1;
6004                 i++;
6005         }
6006         if (lo & F_EXT_MEM_ENABLE) {
6007                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
6008                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
6009                 avail[i].limit = avail[i].base +
6010                     (G_EXT_MEM_SIZE(hi) << 20);
6011                 avail[i].idx = is_t4(sc) ? 2 : 3;       /* Call it MC for T4 */
6012                 i++;
6013         }
6014         if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
6015                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
6016                 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
6017                 avail[i].limit = avail[i].base +
6018                     (G_EXT_MEM1_SIZE(hi) << 20);
6019                 avail[i].idx = 4;
6020                 i++;
6021         }
6022         if (!i)                                    /* no memory available */
6023                 return 0;
6024         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
6025
6026         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
6027         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
6028         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
6029         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
6030         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
6031         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
6032         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
6033         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
6034         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
6035
6036         /* the next few have explicit upper bounds */
6037         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
6038         md->limit = md->base - 1 +
6039                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
6040                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
6041         md++;
6042
6043         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
6044         md->limit = md->base - 1 +
6045                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
6046                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
6047         md++;
6048
6049         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6050                 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
6051                 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
6052                 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
6053         } else {
6054                 md->base = 0;
6055                 md->idx = nitems(region);  /* hide it */
6056         }
6057         md++;
6058
6059 #define ulp_region(reg) \
6060         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
6061         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
6062
6063         ulp_region(RX_ISCSI);
6064         ulp_region(RX_TDDP);
6065         ulp_region(TX_TPT);
6066         ulp_region(RX_STAG);
6067         ulp_region(RX_RQ);
6068         ulp_region(RX_RQUDP);
6069         ulp_region(RX_PBL);
6070         ulp_region(TX_PBL);
6071 #undef ulp_region
6072
6073         md->base = 0;
6074         md->idx = nitems(region);
6075         if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
6076                 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
6077                 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
6078                     A_SGE_DBVFIFO_SIZE))) << 2) - 1;
6079         }
6080         md++;
6081
6082         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
6083         md->limit = md->base + sc->tids.ntids - 1;
6084         md++;
6085         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
6086         md->limit = md->base + sc->tids.ntids - 1;
6087         md++;
6088
6089         md->base = sc->vres.ocq.start;
6090         if (sc->vres.ocq.size)
6091                 md->limit = md->base + sc->vres.ocq.size - 1;
6092         else
6093                 md->idx = nitems(region);  /* hide it */
6094         md++;
6095
6096         /* add any address-space holes, there can be up to 3 */
6097         for (n = 0; n < i - 1; n++)
6098                 if (avail[n].limit < avail[n + 1].base)
6099                         (md++)->base = avail[n].limit;
6100         if (avail[n].limit)
6101                 (md++)->base = avail[n].limit;
6102
6103         n = md - mem;
6104         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
6105
6106         for (lo = 0; lo < i; lo++)
6107                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
6108                                 avail[lo].limit - 1);
6109
6110         sbuf_printf(sb, "\n");
6111         for (i = 0; i < n; i++) {
6112                 if (mem[i].idx >= nitems(region))
6113                         continue;                        /* skip holes */
6114                 if (!mem[i].limit)
6115                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
6116                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
6117                                 mem[i].limit);
6118         }
6119
6120         sbuf_printf(sb, "\n");
6121         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
6122         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
6123         mem_region_show(sb, "uP RAM:", lo, hi);
6124
6125         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
6126         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
6127         mem_region_show(sb, "uP Extmem2:", lo, hi);
6128
6129         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
6130         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
6131                    G_PMRXMAXPAGE(lo),
6132                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
6133                    (lo & F_PMRXNUMCHN) ? 2 : 1);
6134
6135         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
6136         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
6137         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
6138                    G_PMTXMAXPAGE(lo),
6139                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
6140                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
6141         sbuf_printf(sb, "%u p-structs\n",
6142                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
6143
6144         for (i = 0; i < 4; i++) {
6145                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
6146                 if (is_t4(sc)) {
6147                         used = G_USED(lo);
6148                         alloc = G_ALLOC(lo);
6149                 } else {
6150                         used = G_T5_USED(lo);
6151                         alloc = G_T5_ALLOC(lo);
6152                 }
6153                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
6154                            i, used, alloc);
6155         }
6156         for (i = 0; i < 4; i++) {
6157                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
6158                 if (is_t4(sc)) {
6159                         used = G_USED(lo);
6160                         alloc = G_ALLOC(lo);
6161                 } else {
6162                         used = G_T5_USED(lo);
6163                         alloc = G_T5_ALLOC(lo);
6164                 }
6165                 sbuf_printf(sb,
6166                            "\nLoopback %d using %u pages out of %u allocated",
6167                            i, used, alloc);
6168         }
6169
6170         rc = sbuf_finish(sb);
6171         sbuf_delete(sb);
6172
6173         return (rc);
6174 }
6175
6176 static inline void
6177 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
6178 {
6179         *mask = x | y;
6180         y = htobe64(y);
6181         memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
6182 }
6183
6184 static int
6185 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
6186 {
6187         struct adapter *sc = arg1;
6188         struct sbuf *sb;
6189         int rc, i, n;
6190
6191         rc = sysctl_wire_old_buffer(req, 0);
6192         if (rc != 0)
6193                 return (rc);
6194
6195         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6196         if (sb == NULL)
6197                 return (ENOMEM);
6198
6199         sbuf_printf(sb,
6200             "Idx  Ethernet address     Mask     Vld Ports PF"
6201             "  VF              Replication             P0 P1 P2 P3  ML");
6202         n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
6203             NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
6204         for (i = 0; i < n; i++) {
6205                 uint64_t tcamx, tcamy, mask;
6206                 uint32_t cls_lo, cls_hi;
6207                 uint8_t addr[ETHER_ADDR_LEN];
6208
6209                 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
6210                 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
6211                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
6212                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
6213
6214                 if (tcamx & tcamy)
6215                         continue;
6216
6217                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
6218                 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
6219                            "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
6220                            addr[3], addr[4], addr[5], (uintmax_t)mask,
6221                            (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
6222                            G_PORTMAP(cls_hi), G_PF(cls_lo),
6223                            (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
6224
6225                 if (cls_lo & F_REPLICATE) {
6226                         struct fw_ldst_cmd ldst_cmd;
6227
6228                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
6229                         ldst_cmd.op_to_addrspace =
6230                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
6231                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
6232                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
6233                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
6234                         ldst_cmd.u.mps.fid_ctl =
6235                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
6236                                 V_FW_LDST_CMD_CTL(i));
6237
6238                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
6239                             "t4mps");
6240                         if (rc)
6241                                 break;
6242                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
6243                             sizeof(ldst_cmd), &ldst_cmd);
6244                         end_synchronized_op(sc, 0);
6245
6246                         if (rc != 0) {
6247                                 sbuf_printf(sb,
6248                                     " ------------ error %3u ------------", rc);
6249                                 rc = 0;
6250                         } else {
6251                                 sbuf_printf(sb, " %08x %08x %08x %08x",
6252                                     be32toh(ldst_cmd.u.mps.rplc127_96),
6253                                     be32toh(ldst_cmd.u.mps.rplc95_64),
6254                                     be32toh(ldst_cmd.u.mps.rplc63_32),
6255                                     be32toh(ldst_cmd.u.mps.rplc31_0));
6256                         }
6257                 } else
6258                         sbuf_printf(sb, "%36s", "");
6259
6260                 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
6261                     G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
6262                     G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
6263         }
6264
6265         if (rc)
6266                 (void) sbuf_finish(sb);
6267         else
6268                 rc = sbuf_finish(sb);
6269         sbuf_delete(sb);
6270
6271         return (rc);
6272 }
6273
6274 static int
6275 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
6276 {
6277         struct adapter *sc = arg1;
6278         struct sbuf *sb;
6279         int rc;
6280         uint16_t mtus[NMTUS];
6281
6282         rc = sysctl_wire_old_buffer(req, 0);
6283         if (rc != 0)
6284                 return (rc);
6285
6286         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6287         if (sb == NULL)
6288                 return (ENOMEM);
6289
6290         t4_read_mtu_tbl(sc, mtus, NULL);
6291
6292         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
6293             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
6294             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
6295             mtus[14], mtus[15]);
6296
6297         rc = sbuf_finish(sb);
6298         sbuf_delete(sb);
6299
6300         return (rc);
6301 }
6302
6303 static int
6304 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
6305 {
6306         struct adapter *sc = arg1;
6307         struct sbuf *sb;
6308         int rc, i;
6309         uint32_t cnt[PM_NSTATS];
6310         uint64_t cyc[PM_NSTATS];
6311         static const char *rx_stats[] = {
6312                 "Read:", "Write bypass:", "Write mem:", "Flush:"
6313         };
6314         static const char *tx_stats[] = {
6315                 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
6316         };
6317
6318         rc = sysctl_wire_old_buffer(req, 0);
6319         if (rc != 0)
6320                 return (rc);
6321
6322         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6323         if (sb == NULL)
6324                 return (ENOMEM);
6325
6326         t4_pmtx_get_stats(sc, cnt, cyc);
6327         sbuf_printf(sb, "                Tx pcmds             Tx bytes");
6328         for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
6329                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
6330                     cyc[i]);
6331
6332         t4_pmrx_get_stats(sc, cnt, cyc);
6333         sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
6334         for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
6335                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
6336                     cyc[i]);
6337
6338         rc = sbuf_finish(sb);
6339         sbuf_delete(sb);
6340
6341         return (rc);
6342 }
6343
6344 static int
6345 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
6346 {
6347         struct adapter *sc = arg1;
6348         struct sbuf *sb;
6349         int rc;
6350         struct tp_rdma_stats stats;
6351
6352         rc = sysctl_wire_old_buffer(req, 0);
6353         if (rc != 0)
6354                 return (rc);
6355
6356         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6357         if (sb == NULL)
6358                 return (ENOMEM);
6359
6360         t4_tp_get_rdma_stats(sc, &stats);
6361         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
6362         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
6363
6364         rc = sbuf_finish(sb);
6365         sbuf_delete(sb);
6366
6367         return (rc);
6368 }
6369
6370 static int
6371 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
6372 {
6373         struct adapter *sc = arg1;
6374         struct sbuf *sb;
6375         int rc;
6376         struct tp_tcp_stats v4, v6;
6377
6378         rc = sysctl_wire_old_buffer(req, 0);
6379         if (rc != 0)
6380                 return (rc);
6381
6382         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6383         if (sb == NULL)
6384                 return (ENOMEM);
6385
6386         t4_tp_get_tcp_stats(sc, &v4, &v6);
6387         sbuf_printf(sb,
6388             "                                IP                 IPv6\n");
6389         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
6390             v4.tcpOutRsts, v6.tcpOutRsts);
6391         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
6392             v4.tcpInSegs, v6.tcpInSegs);
6393         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
6394             v4.tcpOutSegs, v6.tcpOutSegs);
6395         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
6396             v4.tcpRetransSegs, v6.tcpRetransSegs);
6397
6398         rc = sbuf_finish(sb);
6399         sbuf_delete(sb);
6400
6401         return (rc);
6402 }
6403
6404 static int
6405 sysctl_tids(SYSCTL_HANDLER_ARGS)
6406 {
6407         struct adapter *sc = arg1;
6408         struct sbuf *sb;
6409         int rc;
6410         struct tid_info *t = &sc->tids;
6411
6412         rc = sysctl_wire_old_buffer(req, 0);
6413         if (rc != 0)
6414                 return (rc);
6415
6416         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6417         if (sb == NULL)
6418                 return (ENOMEM);
6419
6420         if (t->natids) {
6421                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
6422                     t->atids_in_use);
6423         }
6424
6425         if (t->ntids) {
6426                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6427                         uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
6428
6429                         if (b) {
6430                                 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
6431                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6432                                     t->ntids - 1);
6433                         } else {
6434                                 sbuf_printf(sb, "TID range: %u-%u",
6435                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6436                                     t->ntids - 1);
6437                         }
6438                 } else
6439                         sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
6440                 sbuf_printf(sb, ", in use: %u\n",
6441                     atomic_load_acq_int(&t->tids_in_use));
6442         }
6443
6444         if (t->nstids) {
6445                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
6446                     t->stid_base + t->nstids - 1, t->stids_in_use);
6447         }
6448
6449         if (t->nftids) {
6450                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
6451                     t->ftid_base + t->nftids - 1);
6452         }
6453
6454         if (t->netids) {
6455                 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
6456                     t->etid_base + t->netids - 1);
6457         }
6458
6459         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
6460             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
6461             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
6462
6463         rc = sbuf_finish(sb);
6464         sbuf_delete(sb);
6465
6466         return (rc);
6467 }
6468
6469 static int
6470 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
6471 {
6472         struct adapter *sc = arg1;
6473         struct sbuf *sb;
6474         int rc;
6475         struct tp_err_stats stats;
6476
6477         rc = sysctl_wire_old_buffer(req, 0);
6478         if (rc != 0)
6479                 return (rc);
6480
6481         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6482         if (sb == NULL)
6483                 return (ENOMEM);
6484
6485         t4_tp_get_err_stats(sc, &stats);
6486
6487         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
6488                       "channel 3\n");
6489         sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
6490             stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
6491             stats.macInErrs[3]);
6492         sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
6493             stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
6494             stats.hdrInErrs[3]);
6495         sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
6496             stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
6497             stats.tcpInErrs[3]);
6498         sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
6499             stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
6500             stats.tcp6InErrs[3]);
6501         sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
6502             stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
6503             stats.tnlCongDrops[3]);
6504         sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
6505             stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
6506             stats.tnlTxDrops[3]);
6507         sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
6508             stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
6509             stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
6510         sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
6511             stats.ofldChanDrops[0], stats.ofldChanDrops[1],
6512             stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
6513         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
6514             stats.ofldNoNeigh, stats.ofldCongDefer);
6515
6516         rc = sbuf_finish(sb);
6517         sbuf_delete(sb);
6518
6519         return (rc);
6520 }
6521
6522 struct field_desc {
6523         const char *name;
6524         u_int start;
6525         u_int width;
6526 };
6527
6528 static void
6529 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
6530 {
6531         char buf[32];
6532         int line_size = 0;
6533
6534         while (f->name) {
6535                 uint64_t mask = (1ULL << f->width) - 1;
6536                 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
6537                     ((uintmax_t)v >> f->start) & mask);
6538
6539                 if (line_size + len >= 79) {
6540                         line_size = 8;
6541                         sbuf_printf(sb, "\n        ");
6542                 }
6543                 sbuf_printf(sb, "%s ", buf);
6544                 line_size += len + 1;
6545                 f++;
6546         }
6547         sbuf_printf(sb, "\n");
6548 }
6549
6550 static struct field_desc tp_la0[] = {
6551         { "RcfOpCodeOut", 60, 4 },
6552         { "State", 56, 4 },
6553         { "WcfState", 52, 4 },
6554         { "RcfOpcSrcOut", 50, 2 },
6555         { "CRxError", 49, 1 },
6556         { "ERxError", 48, 1 },
6557         { "SanityFailed", 47, 1 },
6558         { "SpuriousMsg", 46, 1 },
6559         { "FlushInputMsg", 45, 1 },
6560         { "FlushInputCpl", 44, 1 },
6561         { "RssUpBit", 43, 1 },
6562         { "RssFilterHit", 42, 1 },
6563         { "Tid", 32, 10 },
6564         { "InitTcb", 31, 1 },
6565         { "LineNumber", 24, 7 },
6566         { "Emsg", 23, 1 },
6567         { "EdataOut", 22, 1 },
6568         { "Cmsg", 21, 1 },
6569         { "CdataOut", 20, 1 },
6570         { "EreadPdu", 19, 1 },
6571         { "CreadPdu", 18, 1 },
6572         { "TunnelPkt", 17, 1 },
6573         { "RcfPeerFin", 16, 1 },
6574         { "RcfReasonOut", 12, 4 },
6575         { "TxCchannel", 10, 2 },
6576         { "RcfTxChannel", 8, 2 },
6577         { "RxEchannel", 6, 2 },
6578         { "RcfRxChannel", 5, 1 },
6579         { "RcfDataOutSrdy", 4, 1 },
6580         { "RxDvld", 3, 1 },
6581         { "RxOoDvld", 2, 1 },
6582         { "RxCongestion", 1, 1 },
6583         { "TxCongestion", 0, 1 },
6584         { NULL }
6585 };
6586
6587 static struct field_desc tp_la1[] = {
6588         { "CplCmdIn", 56, 8 },
6589         { "CplCmdOut", 48, 8 },
6590         { "ESynOut", 47, 1 },
6591         { "EAckOut", 46, 1 },
6592         { "EFinOut", 45, 1 },
6593         { "ERstOut", 44, 1 },
6594         { "SynIn", 43, 1 },
6595         { "AckIn", 42, 1 },
6596         { "FinIn", 41, 1 },
6597         { "RstIn", 40, 1 },
6598         { "DataIn", 39, 1 },
6599         { "DataInVld", 38, 1 },
6600         { "PadIn", 37, 1 },
6601         { "RxBufEmpty", 36, 1 },
6602         { "RxDdp", 35, 1 },
6603         { "RxFbCongestion", 34, 1 },
6604         { "TxFbCongestion", 33, 1 },
6605         { "TxPktSumSrdy", 32, 1 },
6606         { "RcfUlpType", 28, 4 },
6607         { "Eread", 27, 1 },
6608         { "Ebypass", 26, 1 },
6609         { "Esave", 25, 1 },
6610         { "Static0", 24, 1 },
6611         { "Cread", 23, 1 },
6612         { "Cbypass", 22, 1 },
6613         { "Csave", 21, 1 },
6614         { "CPktOut", 20, 1 },
6615         { "RxPagePoolFull", 18, 2 },
6616         { "RxLpbkPkt", 17, 1 },
6617         { "TxLpbkPkt", 16, 1 },
6618         { "RxVfValid", 15, 1 },
6619         { "SynLearned", 14, 1 },
6620         { "SetDelEntry", 13, 1 },
6621         { "SetInvEntry", 12, 1 },
6622         { "CpcmdDvld", 11, 1 },
6623         { "CpcmdSave", 10, 1 },
6624         { "RxPstructsFull", 8, 2 },
6625         { "EpcmdDvld", 7, 1 },
6626         { "EpcmdFlush", 6, 1 },
6627         { "EpcmdTrimPrefix", 5, 1 },
6628         { "EpcmdTrimPostfix", 4, 1 },
6629         { "ERssIp4Pkt", 3, 1 },
6630         { "ERssIp6Pkt", 2, 1 },
6631         { "ERssTcpUdpPkt", 1, 1 },
6632         { "ERssFceFipPkt", 0, 1 },
6633         { NULL }
6634 };
6635
6636 static struct field_desc tp_la2[] = {
6637         { "CplCmdIn", 56, 8 },
6638         { "MpsVfVld", 55, 1 },
6639         { "MpsPf", 52, 3 },
6640         { "MpsVf", 44, 8 },
6641         { "SynIn", 43, 1 },
6642         { "AckIn", 42, 1 },
6643         { "FinIn", 41, 1 },
6644         { "RstIn", 40, 1 },
6645         { "DataIn", 39, 1 },
6646         { "DataInVld", 38, 1 },
6647         { "PadIn", 37, 1 },
6648         { "RxBufEmpty", 36, 1 },
6649         { "RxDdp", 35, 1 },
6650         { "RxFbCongestion", 34, 1 },
6651         { "TxFbCongestion", 33, 1 },
6652         { "TxPktSumSrdy", 32, 1 },
6653         { "RcfUlpType", 28, 4 },
6654         { "Eread", 27, 1 },
6655         { "Ebypass", 26, 1 },
6656         { "Esave", 25, 1 },
6657         { "Static0", 24, 1 },
6658         { "Cread", 23, 1 },
6659         { "Cbypass", 22, 1 },
6660         { "Csave", 21, 1 },
6661         { "CPktOut", 20, 1 },
6662         { "RxPagePoolFull", 18, 2 },
6663         { "RxLpbkPkt", 17, 1 },
6664         { "TxLpbkPkt", 16, 1 },
6665         { "RxVfValid", 15, 1 },
6666         { "SynLearned", 14, 1 },
6667         { "SetDelEntry", 13, 1 },
6668         { "SetInvEntry", 12, 1 },
6669         { "CpcmdDvld", 11, 1 },
6670         { "CpcmdSave", 10, 1 },
6671         { "RxPstructsFull", 8, 2 },
6672         { "EpcmdDvld", 7, 1 },
6673         { "EpcmdFlush", 6, 1 },
6674         { "EpcmdTrimPrefix", 5, 1 },
6675         { "EpcmdTrimPostfix", 4, 1 },
6676         { "ERssIp4Pkt", 3, 1 },
6677         { "ERssIp6Pkt", 2, 1 },
6678         { "ERssTcpUdpPkt", 1, 1 },
6679         { "ERssFceFipPkt", 0, 1 },
6680         { NULL }
6681 };
6682
6683 static void
6684 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
6685 {
6686
6687         field_desc_show(sb, *p, tp_la0);
6688 }
6689
6690 static void
6691 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
6692 {
6693
6694         if (idx)
6695                 sbuf_printf(sb, "\n");
6696         field_desc_show(sb, p[0], tp_la0);
6697         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6698                 field_desc_show(sb, p[1], tp_la0);
6699 }
6700
6701 static void
6702 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
6703 {
6704
6705         if (idx)
6706                 sbuf_printf(sb, "\n");
6707         field_desc_show(sb, p[0], tp_la0);
6708         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6709                 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
6710 }
6711
6712 static int
6713 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
6714 {
6715         struct adapter *sc = arg1;
6716         struct sbuf *sb;
6717         uint64_t *buf, *p;
6718         int rc;
6719         u_int i, inc;
6720         void (*show_func)(struct sbuf *, uint64_t *, int);
6721
6722         rc = sysctl_wire_old_buffer(req, 0);
6723         if (rc != 0)
6724                 return (rc);
6725
6726         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6727         if (sb == NULL)
6728                 return (ENOMEM);
6729
6730         buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
6731
6732         t4_tp_read_la(sc, buf, NULL);
6733         p = buf;
6734
6735         switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
6736         case 2:
6737                 inc = 2;
6738                 show_func = tp_la_show2;
6739                 break;
6740         case 3:
6741                 inc = 2;
6742                 show_func = tp_la_show3;
6743                 break;
6744         default:
6745                 inc = 1;
6746                 show_func = tp_la_show;
6747         }
6748
6749         for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
6750                 (*show_func)(sb, p, i);
6751
6752         rc = sbuf_finish(sb);
6753         sbuf_delete(sb);
6754         free(buf, M_CXGBE);
6755         return (rc);
6756 }
6757
6758 static int
6759 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
6760 {
6761         struct adapter *sc = arg1;
6762         struct sbuf *sb;
6763         int rc;
6764         u64 nrate[NCHAN], orate[NCHAN];
6765
6766         rc = sysctl_wire_old_buffer(req, 0);
6767         if (rc != 0)
6768                 return (rc);
6769
6770         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6771         if (sb == NULL)
6772                 return (ENOMEM);
6773
6774         t4_get_chan_txrate(sc, nrate, orate);
6775         sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
6776                  "channel 3\n");
6777         sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
6778             nrate[0], nrate[1], nrate[2], nrate[3]);
6779         sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
6780             orate[0], orate[1], orate[2], orate[3]);
6781
6782         rc = sbuf_finish(sb);
6783         sbuf_delete(sb);
6784
6785         return (rc);
6786 }
6787
6788 static int
6789 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
6790 {
6791         struct adapter *sc = arg1;
6792         struct sbuf *sb;
6793         uint32_t *buf, *p;
6794         int rc, i;
6795
6796         rc = sysctl_wire_old_buffer(req, 0);
6797         if (rc != 0)
6798                 return (rc);
6799
6800         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6801         if (sb == NULL)
6802                 return (ENOMEM);
6803
6804         buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
6805             M_ZERO | M_WAITOK);
6806
6807         t4_ulprx_read_la(sc, buf);
6808         p = buf;
6809
6810         sbuf_printf(sb, "      Pcmd        Type   Message"
6811             "                Data");
6812         for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
6813                 sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
6814                     p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
6815         }
6816
6817         rc = sbuf_finish(sb);
6818         sbuf_delete(sb);
6819         free(buf, M_CXGBE);
6820         return (rc);
6821 }
6822
6823 static int
6824 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
6825 {
6826         struct adapter *sc = arg1;
6827         struct sbuf *sb;
6828         int rc, v;
6829
6830         rc = sysctl_wire_old_buffer(req, 0);
6831         if (rc != 0)
6832                 return (rc);
6833
6834         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6835         if (sb == NULL)
6836                 return (ENOMEM);
6837
6838         v = t4_read_reg(sc, A_SGE_STAT_CFG);
6839         if (G_STATSOURCE_T5(v) == 7) {
6840                 if (G_STATMODE(v) == 0) {
6841                         sbuf_printf(sb, "total %d, incomplete %d",
6842                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6843                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6844                 } else if (G_STATMODE(v) == 1) {
6845                         sbuf_printf(sb, "total %d, data overflow %d",
6846                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6847                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6848                 }
6849         }
6850         rc = sbuf_finish(sb);
6851         sbuf_delete(sb);
6852
6853         return (rc);
6854 }
6855 #endif
6856
6857 static inline void
6858 txq_start(struct ifnet *ifp, struct sge_txq *txq)
6859 {
6860         struct buf_ring *br;
6861         struct mbuf *m;
6862
6863         TXQ_LOCK_ASSERT_OWNED(txq);
6864
6865         br = txq->br;
6866         m = txq->m ? txq->m : drbr_dequeue(ifp, br);
6867         if (m)
6868                 t4_eth_tx(ifp, txq, m);
6869 }
6870
6871 void
6872 t4_tx_callout(void *arg)
6873 {
6874         struct sge_eq *eq = arg;
6875         struct adapter *sc;
6876
6877         if (EQ_TRYLOCK(eq) == 0)
6878                 goto reschedule;
6879
6880         if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) {
6881                 EQ_UNLOCK(eq);
6882 reschedule:
6883                 if (__predict_true(!(eq->flags && EQ_DOOMED)))
6884                         callout_schedule(&eq->tx_callout, 1);
6885                 return;
6886         }
6887
6888         EQ_LOCK_ASSERT_OWNED(eq);
6889
6890         if (__predict_true((eq->flags & EQ_DOOMED) == 0)) {
6891
6892                 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
6893                         struct sge_txq *txq = arg;
6894                         struct port_info *pi = txq->ifp->if_softc;
6895
6896                         sc = pi->adapter;
6897                 } else {
6898                         struct sge_wrq *wrq = arg;
6899
6900                         sc = wrq->adapter;
6901                 }
6902
6903                 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task);
6904         }
6905
6906         EQ_UNLOCK(eq);
6907 }
6908
6909 void
6910 t4_tx_task(void *arg, int count)
6911 {
6912         struct sge_eq *eq = arg;
6913
6914         EQ_LOCK(eq);
6915         if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
6916                 struct sge_txq *txq = arg;
6917                 txq_start(txq->ifp, txq);
6918         } else {
6919                 struct sge_wrq *wrq = arg;
6920                 t4_wrq_tx_locked(wrq->adapter, wrq, NULL);
6921         }
6922         EQ_UNLOCK(eq);
6923 }
6924
6925 static uint32_t
6926 fconf_to_mode(uint32_t fconf)
6927 {
6928         uint32_t mode;
6929
6930         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
6931             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
6932
6933         if (fconf & F_FRAGMENTATION)
6934                 mode |= T4_FILTER_IP_FRAGMENT;
6935
6936         if (fconf & F_MPSHITTYPE)
6937                 mode |= T4_FILTER_MPS_HIT_TYPE;
6938
6939         if (fconf & F_MACMATCH)
6940                 mode |= T4_FILTER_MAC_IDX;
6941
6942         if (fconf & F_ETHERTYPE)
6943                 mode |= T4_FILTER_ETH_TYPE;
6944
6945         if (fconf & F_PROTOCOL)
6946                 mode |= T4_FILTER_IP_PROTO;
6947
6948         if (fconf & F_TOS)
6949                 mode |= T4_FILTER_IP_TOS;
6950
6951         if (fconf & F_VLAN)
6952                 mode |= T4_FILTER_VLAN;
6953
6954         if (fconf & F_VNIC_ID)
6955                 mode |= T4_FILTER_VNIC;
6956
6957         if (fconf & F_PORT)
6958                 mode |= T4_FILTER_PORT;
6959
6960         if (fconf & F_FCOE)
6961                 mode |= T4_FILTER_FCoE;
6962
6963         return (mode);
6964 }
6965
6966 static uint32_t
6967 mode_to_fconf(uint32_t mode)
6968 {
6969         uint32_t fconf = 0;
6970
6971         if (mode & T4_FILTER_IP_FRAGMENT)
6972                 fconf |= F_FRAGMENTATION;
6973
6974         if (mode & T4_FILTER_MPS_HIT_TYPE)
6975                 fconf |= F_MPSHITTYPE;
6976
6977         if (mode & T4_FILTER_MAC_IDX)
6978                 fconf |= F_MACMATCH;
6979
6980         if (mode & T4_FILTER_ETH_TYPE)
6981                 fconf |= F_ETHERTYPE;
6982
6983         if (mode & T4_FILTER_IP_PROTO)
6984                 fconf |= F_PROTOCOL;
6985
6986         if (mode & T4_FILTER_IP_TOS)
6987                 fconf |= F_TOS;
6988
6989         if (mode & T4_FILTER_VLAN)
6990                 fconf |= F_VLAN;
6991
6992         if (mode & T4_FILTER_VNIC)
6993                 fconf |= F_VNIC_ID;
6994
6995         if (mode & T4_FILTER_PORT)
6996                 fconf |= F_PORT;
6997
6998         if (mode & T4_FILTER_FCoE)
6999                 fconf |= F_FCOE;
7000
7001         return (fconf);
7002 }
7003
7004 static uint32_t
7005 fspec_to_fconf(struct t4_filter_specification *fs)
7006 {
7007         uint32_t fconf = 0;
7008
7009         if (fs->val.frag || fs->mask.frag)
7010                 fconf |= F_FRAGMENTATION;
7011
7012         if (fs->val.matchtype || fs->mask.matchtype)
7013                 fconf |= F_MPSHITTYPE;
7014
7015         if (fs->val.macidx || fs->mask.macidx)
7016                 fconf |= F_MACMATCH;
7017
7018         if (fs->val.ethtype || fs->mask.ethtype)
7019                 fconf |= F_ETHERTYPE;
7020
7021         if (fs->val.proto || fs->mask.proto)
7022                 fconf |= F_PROTOCOL;
7023
7024         if (fs->val.tos || fs->mask.tos)
7025                 fconf |= F_TOS;
7026
7027         if (fs->val.vlan_vld || fs->mask.vlan_vld)
7028                 fconf |= F_VLAN;
7029
7030         if (fs->val.vnic_vld || fs->mask.vnic_vld)
7031                 fconf |= F_VNIC_ID;
7032
7033         if (fs->val.iport || fs->mask.iport)
7034                 fconf |= F_PORT;
7035
7036         if (fs->val.fcoe || fs->mask.fcoe)
7037                 fconf |= F_FCOE;
7038
7039         return (fconf);
7040 }
7041
7042 static int
7043 get_filter_mode(struct adapter *sc, uint32_t *mode)
7044 {
7045         int rc;
7046         uint32_t fconf;
7047
7048         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7049             "t4getfm");
7050         if (rc)
7051                 return (rc);
7052
7053         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
7054             A_TP_VLAN_PRI_MAP);
7055
7056         if (sc->params.tp.vlan_pri_map != fconf) {
7057                 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
7058                     device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
7059                     fconf);
7060         }
7061
7062         *mode = fconf_to_mode(fconf);
7063
7064         end_synchronized_op(sc, LOCK_HELD);
7065         return (0);
7066 }
7067
7068 static int
7069 set_filter_mode(struct adapter *sc, uint32_t mode)
7070 {
7071         uint32_t fconf;
7072         int rc;
7073
7074         fconf = mode_to_fconf(mode);
7075
7076         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7077             "t4setfm");
7078         if (rc)
7079                 return (rc);
7080
7081         if (sc->tids.ftids_in_use > 0) {
7082                 rc = EBUSY;
7083                 goto done;
7084         }
7085
7086 #ifdef TCP_OFFLOAD
7087         if (sc->offload_map) {
7088                 rc = EBUSY;
7089                 goto done;
7090         }
7091 #endif
7092
7093         rc = -t4_set_filter_mode(sc, fconf);
7094 done:
7095         end_synchronized_op(sc, LOCK_HELD);
7096         return (rc);
7097 }
7098
7099 static inline uint64_t
7100 get_filter_hits(struct adapter *sc, uint32_t fid)
7101 {
7102         uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
7103         uint64_t hits;
7104
7105         memwin_info(sc, 0, &mw_base, NULL);
7106         off = position_memwin(sc, 0,
7107             tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
7108         if (is_t4(sc)) {
7109                 hits = t4_read_reg64(sc, mw_base + off + 16);
7110                 hits = be64toh(hits);
7111         } else {
7112                 hits = t4_read_reg(sc, mw_base + off + 24);
7113                 hits = be32toh(hits);
7114         }
7115
7116         return (hits);
7117 }
7118
7119 static int
7120 get_filter(struct adapter *sc, struct t4_filter *t)
7121 {
7122         int i, rc, nfilters = sc->tids.nftids;
7123         struct filter_entry *f;
7124
7125         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7126             "t4getf");
7127         if (rc)
7128                 return (rc);
7129
7130         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
7131             t->idx >= nfilters) {
7132                 t->idx = 0xffffffff;
7133                 goto done;
7134         }
7135
7136         f = &sc->tids.ftid_tab[t->idx];
7137         for (i = t->idx; i < nfilters; i++, f++) {
7138                 if (f->valid) {
7139                         t->idx = i;
7140                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
7141                         t->smtidx = f->smtidx;
7142                         if (f->fs.hitcnts)
7143                                 t->hits = get_filter_hits(sc, t->idx);
7144                         else
7145                                 t->hits = UINT64_MAX;
7146                         t->fs = f->fs;
7147
7148                         goto done;
7149                 }
7150         }
7151
7152         t->idx = 0xffffffff;
7153 done:
7154         end_synchronized_op(sc, LOCK_HELD);
7155         return (0);
7156 }
7157
7158 static int
7159 set_filter(struct adapter *sc, struct t4_filter *t)
7160 {
7161         unsigned int nfilters, nports;
7162         struct filter_entry *f;
7163         int i, rc;
7164
7165         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
7166         if (rc)
7167                 return (rc);
7168
7169         nfilters = sc->tids.nftids;
7170         nports = sc->params.nports;
7171
7172         if (nfilters == 0) {
7173                 rc = ENOTSUP;
7174                 goto done;
7175         }
7176
7177         if (!(sc->flags & FULL_INIT_DONE)) {
7178                 rc = EAGAIN;
7179                 goto done;
7180         }
7181
7182         if (t->idx >= nfilters) {
7183                 rc = EINVAL;
7184                 goto done;
7185         }
7186
7187         /* Validate against the global filter mode */
7188         if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
7189             sc->params.tp.vlan_pri_map) {
7190                 rc = E2BIG;
7191                 goto done;
7192         }
7193
7194         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
7195                 rc = EINVAL;
7196                 goto done;
7197         }
7198
7199         if (t->fs.val.iport >= nports) {
7200                 rc = EINVAL;
7201                 goto done;
7202         }
7203
7204         /* Can't specify an iq if not steering to it */
7205         if (!t->fs.dirsteer && t->fs.iq) {
7206                 rc = EINVAL;
7207                 goto done;
7208         }
7209
7210         /* IPv6 filter idx must be 4 aligned */
7211         if (t->fs.type == 1 &&
7212             ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
7213                 rc = EINVAL;
7214                 goto done;
7215         }
7216
7217         if (sc->tids.ftid_tab == NULL) {
7218                 KASSERT(sc->tids.ftids_in_use == 0,
7219                     ("%s: no memory allocated but filters_in_use > 0",
7220                     __func__));
7221
7222                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
7223                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
7224                 if (sc->tids.ftid_tab == NULL) {
7225                         rc = ENOMEM;
7226                         goto done;
7227                 }
7228                 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
7229         }
7230
7231         for (i = 0; i < 4; i++) {
7232                 f = &sc->tids.ftid_tab[t->idx + i];
7233
7234                 if (f->pending || f->valid) {
7235                         rc = EBUSY;
7236                         goto done;
7237                 }
7238                 if (f->locked) {
7239                         rc = EPERM;
7240                         goto done;
7241                 }
7242
7243                 if (t->fs.type == 0)
7244                         break;
7245         }
7246
7247         f = &sc->tids.ftid_tab[t->idx];
7248         f->fs = t->fs;
7249
7250         rc = set_filter_wr(sc, t->idx);
7251 done:
7252         end_synchronized_op(sc, 0);
7253
7254         if (rc == 0) {
7255                 mtx_lock(&sc->tids.ftid_lock);
7256                 for (;;) {
7257                         if (f->pending == 0) {
7258                                 rc = f->valid ? 0 : EIO;
7259                                 break;
7260                         }
7261
7262                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7263                             PCATCH, "t4setfw", 0)) {
7264                                 rc = EINPROGRESS;
7265                                 break;
7266                         }
7267                 }
7268                 mtx_unlock(&sc->tids.ftid_lock);
7269         }
7270         return (rc);
7271 }
7272
7273 static int
7274 del_filter(struct adapter *sc, struct t4_filter *t)
7275 {
7276         unsigned int nfilters;
7277         struct filter_entry *f;
7278         int rc;
7279
7280         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
7281         if (rc)
7282                 return (rc);
7283
7284         nfilters = sc->tids.nftids;
7285
7286         if (nfilters == 0) {
7287                 rc = ENOTSUP;
7288                 goto done;
7289         }
7290
7291         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
7292             t->idx >= nfilters) {
7293                 rc = EINVAL;
7294                 goto done;
7295         }
7296
7297         if (!(sc->flags & FULL_INIT_DONE)) {
7298                 rc = EAGAIN;
7299                 goto done;
7300         }
7301
7302         f = &sc->tids.ftid_tab[t->idx];
7303
7304         if (f->pending) {
7305                 rc = EBUSY;
7306                 goto done;
7307         }
7308         if (f->locked) {
7309                 rc = EPERM;
7310                 goto done;
7311         }
7312
7313         if (f->valid) {
7314                 t->fs = f->fs;  /* extra info for the caller */
7315                 rc = del_filter_wr(sc, t->idx);
7316         }
7317
7318 done:
7319         end_synchronized_op(sc, 0);
7320
7321         if (rc == 0) {
7322                 mtx_lock(&sc->tids.ftid_lock);
7323                 for (;;) {
7324                         if (f->pending == 0) {
7325                                 rc = f->valid ? EIO : 0;
7326                                 break;
7327                         }
7328
7329                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7330                             PCATCH, "t4delfw", 0)) {
7331                                 rc = EINPROGRESS;
7332                                 break;
7333                         }
7334                 }
7335                 mtx_unlock(&sc->tids.ftid_lock);
7336         }
7337
7338         return (rc);
7339 }
7340
7341 static void
7342 clear_filter(struct filter_entry *f)
7343 {
7344         if (f->l2t)
7345                 t4_l2t_release(f->l2t);
7346
7347         bzero(f, sizeof (*f));
7348 }
7349
7350 static int
7351 set_filter_wr(struct adapter *sc, int fidx)
7352 {
7353         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7354         struct wrqe *wr;
7355         struct fw_filter_wr *fwr;
7356         unsigned int ftid;
7357
7358         ASSERT_SYNCHRONIZED_OP(sc);
7359
7360         if (f->fs.newdmac || f->fs.newvlan) {
7361                 /* This filter needs an L2T entry; allocate one. */
7362                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
7363                 if (f->l2t == NULL)
7364                         return (EAGAIN);
7365                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
7366                     f->fs.dmac)) {
7367                         t4_l2t_release(f->l2t);
7368                         f->l2t = NULL;
7369                         return (ENOMEM);
7370                 }
7371         }
7372
7373         ftid = sc->tids.ftid_base + fidx;
7374
7375         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7376         if (wr == NULL)
7377                 return (ENOMEM);
7378
7379         fwr = wrtod(wr);
7380         bzero(fwr, sizeof (*fwr));
7381
7382         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
7383         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
7384         fwr->tid_to_iq =
7385             htobe32(V_FW_FILTER_WR_TID(ftid) |
7386                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
7387                 V_FW_FILTER_WR_NOREPLY(0) |
7388                 V_FW_FILTER_WR_IQ(f->fs.iq));
7389         fwr->del_filter_to_l2tix =
7390             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
7391                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
7392                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
7393                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
7394                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
7395                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
7396                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
7397                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
7398                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
7399                     f->fs.newvlan == VLAN_REWRITE) |
7400                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
7401                     f->fs.newvlan == VLAN_REWRITE) |
7402                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
7403                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
7404                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
7405                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
7406         fwr->ethtype = htobe16(f->fs.val.ethtype);
7407         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
7408         fwr->frag_to_ovlan_vldm =
7409             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
7410                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
7411                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
7412                 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
7413                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
7414                 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
7415         fwr->smac_sel = 0;
7416         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
7417             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
7418         fwr->maci_to_matchtypem =
7419             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
7420                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
7421                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
7422                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
7423                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
7424                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
7425                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
7426                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
7427         fwr->ptcl = f->fs.val.proto;
7428         fwr->ptclm = f->fs.mask.proto;
7429         fwr->ttyp = f->fs.val.tos;
7430         fwr->ttypm = f->fs.mask.tos;
7431         fwr->ivlan = htobe16(f->fs.val.vlan);
7432         fwr->ivlanm = htobe16(f->fs.mask.vlan);
7433         fwr->ovlan = htobe16(f->fs.val.vnic);
7434         fwr->ovlanm = htobe16(f->fs.mask.vnic);
7435         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
7436         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
7437         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
7438         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
7439         fwr->lp = htobe16(f->fs.val.dport);
7440         fwr->lpm = htobe16(f->fs.mask.dport);
7441         fwr->fp = htobe16(f->fs.val.sport);
7442         fwr->fpm = htobe16(f->fs.mask.sport);
7443         if (f->fs.newsmac)
7444                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
7445
7446         f->pending = 1;
7447         sc->tids.ftids_in_use++;
7448
7449         t4_wrq_tx(sc, wr);
7450         return (0);
7451 }
7452
7453 static int
7454 del_filter_wr(struct adapter *sc, int fidx)
7455 {
7456         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7457         struct wrqe *wr;
7458         struct fw_filter_wr *fwr;
7459         unsigned int ftid;
7460
7461         ftid = sc->tids.ftid_base + fidx;
7462
7463         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7464         if (wr == NULL)
7465                 return (ENOMEM);
7466         fwr = wrtod(wr);
7467         bzero(fwr, sizeof (*fwr));
7468
7469         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
7470
7471         f->pending = 1;
7472         t4_wrq_tx(sc, wr);
7473         return (0);
7474 }
7475
7476 int
7477 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
7478 {
7479         struct adapter *sc = iq->adapter;
7480         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
7481         unsigned int idx = GET_TID(rpl);
7482         unsigned int rc;
7483         struct filter_entry *f;
7484
7485         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
7486             rss->opcode));
7487
7488         if (is_ftid(sc, idx)) {
7489
7490                 idx -= sc->tids.ftid_base;
7491                 f = &sc->tids.ftid_tab[idx];
7492                 rc = G_COOKIE(rpl->cookie);
7493
7494                 mtx_lock(&sc->tids.ftid_lock);
7495                 if (rc == FW_FILTER_WR_FLT_ADDED) {
7496                         KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
7497                             __func__, idx));
7498                         f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
7499                         f->pending = 0;  /* asynchronous setup completed */
7500                         f->valid = 1;
7501                 } else {
7502                         if (rc != FW_FILTER_WR_FLT_DELETED) {
7503                                 /* Add or delete failed, display an error */
7504                                 log(LOG_ERR,
7505                                     "filter %u setup failed with error %u\n",
7506                                     idx, rc);
7507                         }
7508
7509                         clear_filter(f);
7510                         sc->tids.ftids_in_use--;
7511                 }
7512                 wakeup(&sc->tids.ftid_tab);
7513                 mtx_unlock(&sc->tids.ftid_lock);
7514         }
7515
7516         return (0);
7517 }
7518
7519 static int
7520 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
7521 {
7522         int rc;
7523
7524         if (cntxt->cid > M_CTXTQID)
7525                 return (EINVAL);
7526
7527         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
7528             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
7529                 return (EINVAL);
7530
7531         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
7532         if (rc)
7533                 return (rc);
7534
7535         if (sc->flags & FW_OK) {
7536                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
7537                     &cntxt->data[0]);
7538                 if (rc == 0)
7539                         goto done;
7540         }
7541
7542         /*
7543          * Read via firmware failed or wasn't even attempted.  Read directly via
7544          * the backdoor.
7545          */
7546         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
7547 done:
7548         end_synchronized_op(sc, 0);
7549         return (rc);
7550 }
7551
7552 static int
7553 load_fw(struct adapter *sc, struct t4_data *fw)
7554 {
7555         int rc;
7556         uint8_t *fw_data;
7557
7558         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
7559         if (rc)
7560                 return (rc);
7561
7562         if (sc->flags & FULL_INIT_DONE) {
7563                 rc = EBUSY;
7564                 goto done;
7565         }
7566
7567         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
7568         if (fw_data == NULL) {
7569                 rc = ENOMEM;
7570                 goto done;
7571         }
7572
7573         rc = copyin(fw->data, fw_data, fw->len);
7574         if (rc == 0)
7575                 rc = -t4_load_fw(sc, fw_data, fw->len);
7576
7577         free(fw_data, M_CXGBE);
7578 done:
7579         end_synchronized_op(sc, 0);
7580         return (rc);
7581 }
7582
7583 static int
7584 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
7585 {
7586         uint32_t addr, off, remaining, i, n;
7587         uint32_t *buf, *b;
7588         uint32_t mw_base, mw_aperture;
7589         int rc;
7590         uint8_t *dst;
7591
7592         rc = validate_mem_range(sc, mr->addr, mr->len);
7593         if (rc != 0)
7594                 return (rc);
7595
7596         memwin_info(sc, win, &mw_base, &mw_aperture);
7597         buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
7598         addr = mr->addr;
7599         remaining = mr->len;
7600         dst = (void *)mr->data;
7601
7602         while (remaining) {
7603                 off = position_memwin(sc, win, addr);
7604
7605                 /* number of bytes that we'll copy in the inner loop */
7606                 n = min(remaining, mw_aperture - off);
7607                 for (i = 0; i < n; i += 4)
7608                         *b++ = t4_read_reg(sc, mw_base + off + i);
7609
7610                 rc = copyout(buf, dst, n);
7611                 if (rc != 0)
7612                         break;
7613
7614                 b = buf;
7615                 dst += n;
7616                 remaining -= n;
7617                 addr += n;
7618         }
7619
7620         free(buf, M_CXGBE);
7621         return (rc);
7622 }
7623
7624 static int
7625 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
7626 {
7627         int rc;
7628
7629         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
7630                 return (EINVAL);
7631
7632         if (i2cd->len > sizeof(i2cd->data))
7633                 return (EFBIG);
7634
7635         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
7636         if (rc)
7637                 return (rc);
7638         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
7639             i2cd->offset, i2cd->len, &i2cd->data[0]);
7640         end_synchronized_op(sc, 0);
7641
7642         return (rc);
7643 }
7644
7645 static int
7646 in_range(int val, int lo, int hi)
7647 {
7648
7649         return (val < 0 || (val <= hi && val >= lo));
7650 }
7651
7652 static int
7653 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
7654 {
7655         int fw_subcmd, fw_type, rc;
7656
7657         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
7658         if (rc)
7659                 return (rc);
7660
7661         if (!(sc->flags & FULL_INIT_DONE)) {
7662                 rc = EAGAIN;
7663                 goto done;
7664         }
7665
7666         /*
7667          * Translate the cxgbetool parameters into T4 firmware parameters.  (The
7668          * sub-command and type are in common locations.)
7669          */
7670         if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
7671                 fw_subcmd = FW_SCHED_SC_CONFIG;
7672         else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
7673                 fw_subcmd = FW_SCHED_SC_PARAMS;
7674         else {
7675                 rc = EINVAL;
7676                 goto done;
7677         }
7678         if (p->type == SCHED_CLASS_TYPE_PACKET)
7679                 fw_type = FW_SCHED_TYPE_PKTSCHED;
7680         else {
7681                 rc = EINVAL;
7682                 goto done;
7683         }
7684
7685         if (fw_subcmd == FW_SCHED_SC_CONFIG) {
7686                 /* Vet our parameters ..*/
7687                 if (p->u.config.minmax < 0) {
7688                         rc = EINVAL;
7689                         goto done;
7690                 }
7691
7692                 /* And pass the request to the firmware ...*/
7693                 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1);
7694                 goto done;
7695         }
7696
7697         if (fw_subcmd == FW_SCHED_SC_PARAMS) {
7698                 int fw_level;
7699                 int fw_mode;
7700                 int fw_rateunit;
7701                 int fw_ratemode;
7702
7703                 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
7704                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
7705                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
7706                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
7707                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
7708                         fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
7709                 else {
7710                         rc = EINVAL;
7711                         goto done;
7712                 }
7713
7714                 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
7715                         fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
7716                 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
7717                         fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
7718                 else {
7719                         rc = EINVAL;
7720                         goto done;
7721                 }
7722
7723                 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
7724                         fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
7725                 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
7726                         fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
7727                 else {
7728                         rc = EINVAL;
7729                         goto done;
7730                 }
7731
7732                 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
7733                         fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
7734                 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
7735                         fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
7736                 else {
7737                         rc = EINVAL;
7738                         goto done;
7739                 }
7740
7741                 /* Vet our parameters ... */
7742                 if (!in_range(p->u.params.channel, 0, 3) ||
7743                     !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
7744                     !in_range(p->u.params.minrate, 0, 10000000) ||
7745                     !in_range(p->u.params.maxrate, 0, 10000000) ||
7746                     !in_range(p->u.params.weight, 0, 100)) {
7747                         rc = ERANGE;
7748                         goto done;
7749                 }
7750
7751                 /*
7752                  * Translate any unset parameters into the firmware's
7753                  * nomenclature and/or fail the call if the parameters
7754                  * are required ...
7755                  */
7756                 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
7757                     p->u.params.channel < 0 || p->u.params.cl < 0) {
7758                         rc = EINVAL;
7759                         goto done;
7760                 }
7761                 if (p->u.params.minrate < 0)
7762                         p->u.params.minrate = 0;
7763                 if (p->u.params.maxrate < 0) {
7764                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7765                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7766                                 rc = EINVAL;
7767                                 goto done;
7768                         } else
7769                                 p->u.params.maxrate = 0;
7770                 }
7771                 if (p->u.params.weight < 0) {
7772                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
7773                                 rc = EINVAL;
7774                                 goto done;
7775                         } else
7776                                 p->u.params.weight = 0;
7777                 }
7778                 if (p->u.params.pktsize < 0) {
7779                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7780                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7781                                 rc = EINVAL;
7782                                 goto done;
7783                         } else
7784                                 p->u.params.pktsize = 0;
7785                 }
7786
7787                 /* See what the firmware thinks of the request ... */
7788                 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
7789                     fw_rateunit, fw_ratemode, p->u.params.channel,
7790                     p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
7791                     p->u.params.weight, p->u.params.pktsize, 1);
7792                 goto done;
7793         }
7794
7795         rc = EINVAL;
7796 done:
7797         end_synchronized_op(sc, 0);
7798         return (rc);
7799 }
7800
7801 static int
7802 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
7803 {
7804         struct port_info *pi = NULL;
7805         struct sge_txq *txq;
7806         uint32_t fw_mnem, fw_queue, fw_class;
7807         int i, rc;
7808
7809         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
7810         if (rc)
7811                 return (rc);
7812
7813         if (!(sc->flags & FULL_INIT_DONE)) {
7814                 rc = EAGAIN;
7815                 goto done;
7816         }
7817
7818         if (p->port >= sc->params.nports) {
7819                 rc = EINVAL;
7820                 goto done;
7821         }
7822
7823         pi = sc->port[p->port];
7824         if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
7825                 rc = EINVAL;
7826                 goto done;
7827         }
7828
7829         /*
7830          * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
7831          * Scheduling Class in this case).
7832          */
7833         fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
7834             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
7835         fw_class = p->cl < 0 ? 0xffffffff : p->cl;
7836
7837         /*
7838          * If op.queue is non-negative, then we're only changing the scheduling
7839          * on a single specified TX queue.
7840          */
7841         if (p->queue >= 0) {
7842                 txq = &sc->sge.txq[pi->first_txq + p->queue];
7843                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7844                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7845                     &fw_class);
7846                 goto done;
7847         }
7848
7849         /*
7850          * Change the scheduling on all the TX queues for the
7851          * interface.
7852          */
7853         for_each_txq(pi, i, txq) {
7854                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7855                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7856                     &fw_class);
7857                 if (rc)
7858                         goto done;
7859         }
7860
7861         rc = 0;
7862 done:
7863         end_synchronized_op(sc, 0);
7864         return (rc);
7865 }
7866
7867 int
7868 t4_os_find_pci_capability(struct adapter *sc, int cap)
7869 {
7870         int i;
7871
7872         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
7873 }
7874
7875 int
7876 t4_os_pci_save_state(struct adapter *sc)
7877 {
7878         device_t dev;
7879         struct pci_devinfo *dinfo;
7880
7881         dev = sc->dev;
7882         dinfo = device_get_ivars(dev);
7883
7884         pci_cfg_save(dev, dinfo, 0);
7885         return (0);
7886 }
7887
7888 int
7889 t4_os_pci_restore_state(struct adapter *sc)
7890 {
7891         device_t dev;
7892         struct pci_devinfo *dinfo;
7893
7894         dev = sc->dev;
7895         dinfo = device_get_ivars(dev);
7896
7897         pci_cfg_restore(dev, dinfo);
7898         return (0);
7899 }
7900
7901 void
7902 t4_os_portmod_changed(const struct adapter *sc, int idx)
7903 {
7904         struct port_info *pi = sc->port[idx];
7905         static const char *mod_str[] = {
7906                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
7907         };
7908
7909         build_medialist(pi, &pi->media);
7910 #ifdef DEV_NETMAP
7911         build_medialist(pi, &pi->nm_media);
7912 #endif
7913
7914         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
7915                 if_printf(pi->ifp, "transceiver unplugged.\n");
7916         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
7917                 if_printf(pi->ifp, "unknown transceiver inserted.\n");
7918         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
7919                 if_printf(pi->ifp, "unsupported transceiver inserted.\n");
7920         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
7921                 if_printf(pi->ifp, "%s transceiver inserted.\n",
7922                     mod_str[pi->mod_type]);
7923         } else {
7924                 if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
7925                     pi->mod_type);
7926         }
7927 }
7928
7929 void
7930 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
7931 {
7932         struct port_info *pi = sc->port[idx];
7933         struct ifnet *ifp = pi->ifp;
7934
7935         if (link_stat) {
7936                 pi->linkdnrc = -1;
7937                 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
7938                 if_link_state_change(ifp, LINK_STATE_UP);
7939         } else {
7940                 if (reason >= 0)
7941                         pi->linkdnrc = reason;
7942                 if_link_state_change(ifp, LINK_STATE_DOWN);
7943         }
7944 }
7945
7946 void
7947 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
7948 {
7949         struct adapter *sc;
7950
7951         sx_slock(&t4_list_lock);
7952         SLIST_FOREACH(sc, &t4_list, link) {
7953                 /*
7954                  * func should not make any assumptions about what state sc is
7955                  * in - the only guarantee is that sc->sc_lock is a valid lock.
7956                  */
7957                 func(sc, arg);
7958         }
7959         sx_sunlock(&t4_list_lock);
7960 }
7961
7962 static int
7963 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
7964 {
7965        return (0);
7966 }
7967
7968 static int
7969 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
7970 {
7971        return (0);
7972 }
7973
7974 static int
7975 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
7976     struct thread *td)
7977 {
7978         int rc;
7979         struct adapter *sc = dev->si_drv1;
7980
7981         rc = priv_check(td, PRIV_DRIVER);
7982         if (rc != 0)
7983                 return (rc);
7984
7985         switch (cmd) {
7986         case CHELSIO_T4_GETREG: {
7987                 struct t4_reg *edata = (struct t4_reg *)data;
7988
7989                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
7990                         return (EFAULT);
7991
7992                 if (edata->size == 4)
7993                         edata->val = t4_read_reg(sc, edata->addr);
7994                 else if (edata->size == 8)
7995                         edata->val = t4_read_reg64(sc, edata->addr);
7996                 else
7997                         return (EINVAL);
7998
7999                 break;
8000         }
8001         case CHELSIO_T4_SETREG: {
8002                 struct t4_reg *edata = (struct t4_reg *)data;
8003
8004                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8005                         return (EFAULT);
8006
8007                 if (edata->size == 4) {
8008                         if (edata->val & 0xffffffff00000000)
8009                                 return (EINVAL);
8010                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
8011                 } else if (edata->size == 8)
8012                         t4_write_reg64(sc, edata->addr, edata->val);
8013                 else
8014                         return (EINVAL);
8015                 break;
8016         }
8017         case CHELSIO_T4_REGDUMP: {
8018                 struct t4_regdump *regs = (struct t4_regdump *)data;
8019                 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
8020                 uint8_t *buf;
8021
8022                 if (regs->len < reglen) {
8023                         regs->len = reglen; /* hint to the caller */
8024                         return (ENOBUFS);
8025                 }
8026
8027                 regs->len = reglen;
8028                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
8029                 t4_get_regs(sc, regs, buf);
8030                 rc = copyout(buf, regs->data, reglen);
8031                 free(buf, M_CXGBE);
8032                 break;
8033         }
8034         case CHELSIO_T4_GET_FILTER_MODE:
8035                 rc = get_filter_mode(sc, (uint32_t *)data);
8036                 break;
8037         case CHELSIO_T4_SET_FILTER_MODE:
8038                 rc = set_filter_mode(sc, *(uint32_t *)data);
8039                 break;
8040         case CHELSIO_T4_GET_FILTER:
8041                 rc = get_filter(sc, (struct t4_filter *)data);
8042                 break;
8043         case CHELSIO_T4_SET_FILTER:
8044                 rc = set_filter(sc, (struct t4_filter *)data);
8045                 break;
8046         case CHELSIO_T4_DEL_FILTER:
8047                 rc = del_filter(sc, (struct t4_filter *)data);
8048                 break;
8049         case CHELSIO_T4_GET_SGE_CONTEXT:
8050                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
8051                 break;
8052         case CHELSIO_T4_LOAD_FW:
8053                 rc = load_fw(sc, (struct t4_data *)data);
8054                 break;
8055         case CHELSIO_T4_GET_MEM:
8056                 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
8057                 break;
8058         case CHELSIO_T4_GET_I2C:
8059                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
8060                 break;
8061         case CHELSIO_T4_CLEAR_STATS: {
8062                 int i;
8063                 u_int port_id = *(uint32_t *)data;
8064                 struct port_info *pi;
8065
8066                 if (port_id >= sc->params.nports)
8067                         return (EINVAL);
8068                 pi = sc->port[port_id];
8069
8070                 /* MAC stats */
8071                 t4_clr_port_stats(sc, pi->tx_chan);
8072
8073                 if (pi->flags & PORT_INIT_DONE) {
8074                         struct sge_rxq *rxq;
8075                         struct sge_txq *txq;
8076                         struct sge_wrq *wrq;
8077
8078                         for_each_rxq(pi, i, rxq) {
8079 #if defined(INET) || defined(INET6)
8080                                 rxq->lro.lro_queued = 0;
8081                                 rxq->lro.lro_flushed = 0;
8082 #endif
8083                                 rxq->rxcsum = 0;
8084                                 rxq->vlan_extraction = 0;
8085                         }
8086
8087                         for_each_txq(pi, i, txq) {
8088                                 txq->txcsum = 0;
8089                                 txq->tso_wrs = 0;
8090                                 txq->vlan_insertion = 0;
8091                                 txq->imm_wrs = 0;
8092                                 txq->sgl_wrs = 0;
8093                                 txq->txpkt_wrs = 0;
8094                                 txq->txpkts_wrs = 0;
8095                                 txq->txpkts_pkts = 0;
8096                                 txq->br->br_drops = 0;
8097                                 txq->no_dmamap = 0;
8098                                 txq->no_desc = 0;
8099                         }
8100
8101 #ifdef TCP_OFFLOAD
8102                         /* nothing to clear for each ofld_rxq */
8103
8104                         for_each_ofld_txq(pi, i, wrq) {
8105                                 wrq->tx_wrs = 0;
8106                                 wrq->no_desc = 0;
8107                         }
8108 #endif
8109                         wrq = &sc->sge.ctrlq[pi->port_id];
8110                         wrq->tx_wrs = 0;
8111                         wrq->no_desc = 0;
8112                 }
8113                 break;
8114         }
8115         case CHELSIO_T4_SCHED_CLASS:
8116                 rc = set_sched_class(sc, (struct t4_sched_params *)data);
8117                 break;
8118         case CHELSIO_T4_SCHED_QUEUE:
8119                 rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
8120                 break;
8121         case CHELSIO_T4_GET_TRACER:
8122                 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
8123                 break;
8124         case CHELSIO_T4_SET_TRACER:
8125                 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
8126                 break;
8127         default:
8128                 rc = EINVAL;
8129         }
8130
8131         return (rc);
8132 }
8133
8134 #ifdef TCP_OFFLOAD
8135 void
8136 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask,
8137     const unsigned int *pgsz_order)
8138 {
8139         struct port_info *pi = ifp->if_softc;
8140         struct adapter *sc = pi->adapter;
8141
8142         t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
8143         t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |
8144                 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) |
8145                 V_HPZ3(pgsz_order[3]));
8146 }
8147
8148 static int
8149 toe_capability(struct port_info *pi, int enable)
8150 {
8151         int rc;
8152         struct adapter *sc = pi->adapter;
8153
8154         ASSERT_SYNCHRONIZED_OP(sc);
8155
8156         if (!is_offload(sc))
8157                 return (ENODEV);
8158
8159         if (enable) {
8160                 /*
8161                  * We need the port's queues around so that we're able to send
8162                  * and receive CPLs to/from the TOE even if the ifnet for this
8163                  * port has never been UP'd administratively.
8164                  */
8165                 if (!(pi->flags & PORT_INIT_DONE)) {
8166                         rc = cxgbe_init_synchronized(pi);
8167                         if (rc)
8168                                 return (rc);
8169                 }
8170
8171                 if (isset(&sc->offload_map, pi->port_id))
8172                         return (0);
8173
8174                 if (!(sc->flags & TOM_INIT_DONE)) {
8175                         rc = t4_activate_uld(sc, ULD_TOM);
8176                         if (rc == EAGAIN) {
8177                                 log(LOG_WARNING,
8178                                     "You must kldload t4_tom.ko before trying "
8179                                     "to enable TOE on a cxgbe interface.\n");
8180                         }
8181                         if (rc != 0)
8182                                 return (rc);
8183                         KASSERT(sc->tom_softc != NULL,
8184                             ("%s: TOM activated but softc NULL", __func__));
8185                         KASSERT(sc->flags & TOM_INIT_DONE,
8186                             ("%s: TOM activated but flag not set", __func__));
8187                 }
8188
8189                 setbit(&sc->offload_map, pi->port_id);
8190         } else {
8191                 if (!isset(&sc->offload_map, pi->port_id))
8192                         return (0);
8193
8194                 KASSERT(sc->flags & TOM_INIT_DONE,
8195                     ("%s: TOM never initialized?", __func__));
8196                 clrbit(&sc->offload_map, pi->port_id);
8197         }
8198
8199         return (0);
8200 }
8201
8202 /*
8203  * Add an upper layer driver to the global list.
8204  */
8205 int
8206 t4_register_uld(struct uld_info *ui)
8207 {
8208         int rc = 0;
8209         struct uld_info *u;
8210
8211         sx_xlock(&t4_uld_list_lock);
8212         SLIST_FOREACH(u, &t4_uld_list, link) {
8213             if (u->uld_id == ui->uld_id) {
8214                     rc = EEXIST;
8215                     goto done;
8216             }
8217         }
8218
8219         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
8220         ui->refcount = 0;
8221 done:
8222         sx_xunlock(&t4_uld_list_lock);
8223         return (rc);
8224 }
8225
8226 int
8227 t4_unregister_uld(struct uld_info *ui)
8228 {
8229         int rc = EINVAL;
8230         struct uld_info *u;
8231
8232         sx_xlock(&t4_uld_list_lock);
8233
8234         SLIST_FOREACH(u, &t4_uld_list, link) {
8235             if (u == ui) {
8236                     if (ui->refcount > 0) {
8237                             rc = EBUSY;
8238                             goto done;
8239                     }
8240
8241                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
8242                     rc = 0;
8243                     goto done;
8244             }
8245         }
8246 done:
8247         sx_xunlock(&t4_uld_list_lock);
8248         return (rc);
8249 }
8250
8251 int
8252 t4_activate_uld(struct adapter *sc, int id)
8253 {
8254         int rc = EAGAIN;
8255         struct uld_info *ui;
8256
8257         ASSERT_SYNCHRONIZED_OP(sc);
8258
8259         sx_slock(&t4_uld_list_lock);
8260
8261         SLIST_FOREACH(ui, &t4_uld_list, link) {
8262                 if (ui->uld_id == id) {
8263                         rc = ui->activate(sc);
8264                         if (rc == 0)
8265                                 ui->refcount++;
8266                         goto done;
8267                 }
8268         }
8269 done:
8270         sx_sunlock(&t4_uld_list_lock);
8271
8272         return (rc);
8273 }
8274
8275 int
8276 t4_deactivate_uld(struct adapter *sc, int id)
8277 {
8278         int rc = EINVAL;
8279         struct uld_info *ui;
8280
8281         ASSERT_SYNCHRONIZED_OP(sc);
8282
8283         sx_slock(&t4_uld_list_lock);
8284
8285         SLIST_FOREACH(ui, &t4_uld_list, link) {
8286                 if (ui->uld_id == id) {
8287                         rc = ui->deactivate(sc);
8288                         if (rc == 0)
8289                                 ui->refcount--;
8290                         goto done;
8291                 }
8292         }
8293 done:
8294         sx_sunlock(&t4_uld_list_lock);
8295
8296         return (rc);
8297 }
8298 #endif
8299
8300 /*
8301  * Come up with reasonable defaults for some of the tunables, provided they're
8302  * not set by the user (in which case we'll use the values as is).
8303  */
8304 static void
8305 tweak_tunables(void)
8306 {
8307         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
8308
8309         if (t4_ntxq10g < 1)
8310                 t4_ntxq10g = min(nc, NTXQ_10G);
8311
8312         if (t4_ntxq1g < 1)
8313                 t4_ntxq1g = min(nc, NTXQ_1G);
8314
8315         if (t4_nrxq10g < 1)
8316                 t4_nrxq10g = min(nc, NRXQ_10G);
8317
8318         if (t4_nrxq1g < 1)
8319                 t4_nrxq1g = min(nc, NRXQ_1G);
8320
8321 #ifdef TCP_OFFLOAD
8322         if (t4_nofldtxq10g < 1)
8323                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
8324
8325         if (t4_nofldtxq1g < 1)
8326                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
8327
8328         if (t4_nofldrxq10g < 1)
8329                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
8330
8331         if (t4_nofldrxq1g < 1)
8332                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
8333
8334         if (t4_toecaps_allowed == -1)
8335                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
8336 #else
8337         if (t4_toecaps_allowed == -1)
8338                 t4_toecaps_allowed = 0;
8339 #endif
8340
8341 #ifdef DEV_NETMAP
8342         if (t4_nnmtxq10g < 1)
8343                 t4_nnmtxq10g = min(nc, NNMTXQ_10G);
8344
8345         if (t4_nnmtxq1g < 1)
8346                 t4_nnmtxq1g = min(nc, NNMTXQ_1G);
8347
8348         if (t4_nnmrxq10g < 1)
8349                 t4_nnmrxq10g = min(nc, NNMRXQ_10G);
8350
8351         if (t4_nnmrxq1g < 1)
8352                 t4_nnmrxq1g = min(nc, NNMRXQ_1G);
8353 #endif
8354
8355         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
8356                 t4_tmr_idx_10g = TMR_IDX_10G;
8357
8358         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
8359                 t4_pktc_idx_10g = PKTC_IDX_10G;
8360
8361         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
8362                 t4_tmr_idx_1g = TMR_IDX_1G;
8363
8364         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
8365                 t4_pktc_idx_1g = PKTC_IDX_1G;
8366
8367         if (t4_qsize_txq < 128)
8368                 t4_qsize_txq = 128;
8369
8370         if (t4_qsize_rxq < 128)
8371                 t4_qsize_rxq = 128;
8372         while (t4_qsize_rxq & 7)
8373                 t4_qsize_rxq++;
8374
8375         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
8376 }
8377
8378 static struct sx mlu;   /* mod load unload */
8379 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
8380
8381 static int
8382 mod_event(module_t mod, int cmd, void *arg)
8383 {
8384         int rc = 0;
8385         static int loaded = 0;
8386
8387         switch (cmd) {
8388         case MOD_LOAD:
8389                 sx_xlock(&mlu);
8390                 if (loaded++ == 0) {
8391                         t4_sge_modload();
8392                         sx_init(&t4_list_lock, "T4/T5 adapters");
8393                         SLIST_INIT(&t4_list);
8394 #ifdef TCP_OFFLOAD
8395                         sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
8396                         SLIST_INIT(&t4_uld_list);
8397 #endif
8398                         t4_tracer_modload();
8399                         tweak_tunables();
8400                 }
8401                 sx_xunlock(&mlu);
8402                 break;
8403
8404         case MOD_UNLOAD:
8405                 sx_xlock(&mlu);
8406                 if (--loaded == 0) {
8407                         int tries;
8408
8409                         sx_slock(&t4_list_lock);
8410                         if (!SLIST_EMPTY(&t4_list)) {
8411                                 rc = EBUSY;
8412                                 sx_sunlock(&t4_list_lock);
8413                                 goto done_unload;
8414                         }
8415 #ifdef TCP_OFFLOAD
8416                         sx_slock(&t4_uld_list_lock);
8417                         if (!SLIST_EMPTY(&t4_uld_list)) {
8418                                 rc = EBUSY;
8419                                 sx_sunlock(&t4_uld_list_lock);
8420                                 sx_sunlock(&t4_list_lock);
8421                                 goto done_unload;
8422                         }
8423 #endif
8424                         tries = 0;
8425                         while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
8426                                 uprintf("%ju clusters with custom free routine "
8427                                     "still is use.\n", t4_sge_extfree_refs());
8428                                 pause("t4unload", 2 * hz);
8429                         }
8430 #ifdef TCP_OFFLOAD
8431                         sx_sunlock(&t4_uld_list_lock);
8432 #endif
8433                         sx_sunlock(&t4_list_lock);
8434
8435                         if (t4_sge_extfree_refs() == 0) {
8436                                 t4_tracer_modunload();
8437 #ifdef TCP_OFFLOAD
8438                                 sx_destroy(&t4_uld_list_lock);
8439 #endif
8440                                 sx_destroy(&t4_list_lock);
8441                                 t4_sge_modunload();
8442                                 loaded = 0;
8443                         } else {
8444                                 rc = EBUSY;
8445                                 loaded++;       /* undo earlier decrement */
8446                         }
8447                 }
8448 done_unload:
8449                 sx_xunlock(&mlu);
8450                 break;
8451         }
8452
8453         return (rc);
8454 }
8455
8456 static devclass_t t4_devclass, t5_devclass;
8457 static devclass_t cxgbe_devclass, cxl_devclass;
8458
8459 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
8460 MODULE_VERSION(t4nex, 1);
8461 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
8462
8463 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
8464 MODULE_VERSION(t5nex, 1);
8465 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
8466
8467 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
8468 MODULE_VERSION(cxgbe, 1);
8469
8470 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
8471 MODULE_VERSION(cxl, 1);