sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/conf.h>
  36 #include <sys/priv.h>
  37 #include <sys/kernel.h>
  38 #include <sys/bus.h>
  39 #include <sys/module.h>
  40 #include <sys/malloc.h>
  41 #include <sys/queue.h>
  42 #include <sys/taskqueue.h>
  43 #include <sys/pciio.h>
  44 #include <dev/pci/pcireg.h>
  45 #include <dev/pci/pcivar.h>
  46 #include <dev/pci/pci_private.h>
  47 #include <sys/firmware.h>
  48 #include <sys/sbuf.h>
  49 #include <sys/smp.h>
  50 #include <sys/socket.h>
  51 #include <sys/sockio.h>
  52 #include <sys/sysctl.h>
  53 #include <net/ethernet.h>
  54 #include <net/if.h>
  55 #include <net/if_types.h>
  56 #include <net/if_dl.h>
  57 #include <net/if_vlan_var.h>
  58 #if defined(__i386__) || defined(__amd64__)
  59 #include <vm/vm.h>
  60 #include <vm/pmap.h>
  61 #endif
  62
  63 #include "common/common.h"
  64 #include "common/t4_msg.h"
  65 #include "common/t4_regs.h"
  66 #include "common/t4_regs_values.h"
  67 #include "t4_ioctl.h"
  68 #include "t4_l2t.h"
  69
  70 /* T4 bus driver interface */
  71 static int t4_probe(device_t);
  72 static int t4_attach(device_t);
  73 static int t4_detach(device_t);
  74 static device_method_t t4_methods[] = {
  75         DEVMETHOD(device_probe,         t4_probe),
  76         DEVMETHOD(device_attach,        t4_attach),
  77         DEVMETHOD(device_detach,        t4_detach),
  78
  79         DEVMETHOD_END
  80 };
  81 static driver_t t4_driver = {
  82         "t4nex",
  83         t4_methods,
  84         sizeof(struct adapter)
  85 };
  86
  87
  88 /* T4 port (cxgbe) interface */
  89 static int cxgbe_probe(device_t);
  90 static int cxgbe_attach(device_t);
  91 static int cxgbe_detach(device_t);
  92 static device_method_t cxgbe_methods[] = {
  93         DEVMETHOD(device_probe,         cxgbe_probe),
  94         DEVMETHOD(device_attach,        cxgbe_attach),
  95         DEVMETHOD(device_detach,        cxgbe_detach),
  96         { 0, 0 }
  97 };
  98 static driver_t cxgbe_driver = {
  99         "cxgbe",
 100         cxgbe_methods,
 101         sizeof(struct port_info)
 102 };
 103
 104 static d_ioctl_t t4_ioctl;
 105 static d_open_t t4_open;
 106 static d_close_t t4_close;
 107
 108 static struct cdevsw t4_cdevsw = {
 109        .d_version = D_VERSION,
 110        .d_flags = 0,
 111        .d_open = t4_open,
 112        .d_close = t4_close,
 113        .d_ioctl = t4_ioctl,
 114        .d_name = "t4nex",
 115 };
 116
 117 /* T5 bus driver interface */
 118 static int t5_probe(device_t);
 119 static device_method_t t5_methods[] = {
 120         DEVMETHOD(device_probe,         t5_probe),
 121         DEVMETHOD(device_attach,        t4_attach),
 122         DEVMETHOD(device_detach,        t4_detach),
 123
 124         DEVMETHOD_END
 125 };
 126 static driver_t t5_driver = {
 127         "t5nex",
 128         t5_methods,
 129         sizeof(struct adapter)
 130 };
 131
 132
 133 /* T5 port (cxl) interface */
 134 static driver_t cxl_driver = {
 135         "cxl",
 136         cxgbe_methods,
 137         sizeof(struct port_info)
 138 };
 139
 140 static struct cdevsw t5_cdevsw = {
 141        .d_version = D_VERSION,
 142        .d_flags = 0,
 143        .d_open = t4_open,
 144        .d_close = t4_close,
 145        .d_ioctl = t4_ioctl,
 146        .d_name = "t5nex",
 147 };
 148
 149 /* ifnet + media interface */
 150 static void cxgbe_init(void *);
 151 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 152 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 153 static void cxgbe_qflush(struct ifnet *);
 154 static int cxgbe_media_change(struct ifnet *);
 155 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 156
 157 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
 158
 159 /*
 160  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 161  * then ADAPTER_LOCK, then t4_uld_list_lock.
 162  */
 163 static struct sx t4_list_lock;
 164 SLIST_HEAD(, adapter) t4_list;
 165 #ifdef TCP_OFFLOAD
 166 static struct sx t4_uld_list_lock;
 167 SLIST_HEAD(, uld_info) t4_uld_list;
 168 #endif
 169
 170 /*
 171  * Tunables.  See tweak_tunables() too.
 172  *
 173  * Each tunable is set to a default value here if it's known at compile-time.
 174  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
 175  * provide a reasonable default when the driver is loaded.
 176  *
 177  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
 178  * T5 are under hw.cxl.
 179  */
 180
 181 /*
 182  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 183  */
 184 #define NTXQ_10G 16
 185 static int t4_ntxq10g = -1;
 186 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 187
 188 #define NRXQ_10G 8
 189 static int t4_nrxq10g = -1;
 190 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 191
 192 #define NTXQ_1G 4
 193 static int t4_ntxq1g = -1;
 194 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 195
 196 #define NRXQ_1G 2
 197 static int t4_nrxq1g = -1;
 198 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 199
 200 static int t4_rsrv_noflowq = 0;
 201 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
 202
 203 #ifdef TCP_OFFLOAD
 204 #define NOFLDTXQ_10G 8
 205 static int t4_nofldtxq10g = -1;
 206 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 207
 208 #define NOFLDRXQ_10G 2
 209 static int t4_nofldrxq10g = -1;
 210 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 211
 212 #define NOFLDTXQ_1G 2
 213 static int t4_nofldtxq1g = -1;
 214 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 215
 216 #define NOFLDRXQ_1G 1
 217 static int t4_nofldrxq1g = -1;
 218 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 219 #endif
 220
 221 #ifdef DEV_NETMAP
 222 #define NNMTXQ_10G 2
 223 static int t4_nnmtxq10g = -1;
 224 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g);
 225
 226 #define NNMRXQ_10G 2
 227 static int t4_nnmrxq10g = -1;
 228 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g);
 229
 230 #define NNMTXQ_1G 1
 231 static int t4_nnmtxq1g = -1;
 232 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g);
 233
 234 #define NNMRXQ_1G 1
 235 static int t4_nnmrxq1g = -1;
 236 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g);
 237 #endif
 238
 239 /*
 240  * Holdoff parameters for 10G and 1G ports.
 241  */
 242 #define TMR_IDX_10G 1
 243 static int t4_tmr_idx_10g = TMR_IDX_10G;
 244 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 245
 246 #define PKTC_IDX_10G (-1)
 247 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 248 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 249
 250 #define TMR_IDX_1G 1
 251 static int t4_tmr_idx_1g = TMR_IDX_1G;
 252 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 253
 254 #define PKTC_IDX_1G (-1)
 255 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 256 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 257
 258 /*
 259  * Size (# of entries) of each tx and rx queue.
 260  */
 261 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 262 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 263
 264 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 265 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 266
 267 /*
 268  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 269  */
 270 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 271 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 272
 273 /*
 274  * Configuration file.
 275  */
 276 #define DEFAULT_CF      "default"
 277 #define FLASH_CF        "flash"
 278 #define UWIRE_CF        "uwire"
 279 #define FPGA_CF         "fpga"
 280 static char t4_cfg_file[32] = DEFAULT_CF;
 281 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 282
 283 /*
 284  * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively).
 285  * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
 286  * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
 287  *            mark or when signalled to do so, 0 to never emit PAUSE.
 288  */
 289 static int t4_pause_settings = PAUSE_TX | PAUSE_RX;
 290 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings);
 291
 292 /*
 293  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
 294  * encouraged respectively).
 295  */
 296 static unsigned int t4_fw_install = 1;
 297 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 298
 299 /*
 300  * ASIC features that will be used.  Disable the ones you don't want so that the
 301  * chip resources aren't wasted on features that will not be used.
 302  */
 303 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 304 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 305
 306 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 307 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 308
 309 static int t4_toecaps_allowed = -1;
 310 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 311
 312 static int t4_rdmacaps_allowed = 0;
 313 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 314
 315 static int t4_iscsicaps_allowed = 0;
 316 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 317
 318 static int t4_fcoecaps_allowed = 0;
 319 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 320
 321 static int t5_write_combine = 0;
 322 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 323
 324 struct intrs_and_queues {
 325         uint16_t intr_type;     /* INTx, MSI, or MSI-X */
 326         uint16_t nirq;          /* Total # of vectors */
 327         uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
 328         uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */
 329         uint16_t ntxq10g;       /* # of NIC txq's for each 10G port */
 330         uint16_t nrxq10g;       /* # of NIC rxq's for each 10G port */
 331         uint16_t ntxq1g;        /* # of NIC txq's for each 1G port */
 332         uint16_t nrxq1g;        /* # of NIC rxq's for each 1G port */
 333         uint16_t rsrv_noflowq;  /* Flag whether to reserve queue 0 */
 334 #ifdef TCP_OFFLOAD
 335         uint16_t nofldtxq10g;   /* # of TOE txq's for each 10G port */
 336         uint16_t nofldrxq10g;   /* # of TOE rxq's for each 10G port */
 337         uint16_t nofldtxq1g;    /* # of TOE txq's for each 1G port */
 338         uint16_t nofldrxq1g;    /* # of TOE rxq's for each 1G port */
 339 #endif
 340 #ifdef DEV_NETMAP
 341         uint16_t nnmtxq10g;     /* # of netmap txq's for each 10G port */
 342         uint16_t nnmrxq10g;     /* # of netmap rxq's for each 10G port */
 343         uint16_t nnmtxq1g;      /* # of netmap txq's for each 1G port */
 344         uint16_t nnmrxq1g;      /* # of netmap rxq's for each 1G port */
 345 #endif
 346 };
 347
 348 struct filter_entry {
 349         uint32_t valid:1;       /* filter allocated and valid */
 350         uint32_t locked:1;      /* filter is administratively locked */
 351         uint32_t pending:1;     /* filter action is pending firmware reply */
 352         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 353         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 354
 355         struct t4_filter_specification fs;
 356 };
 357
 358 static int map_bars_0_and_4(struct adapter *);
 359 static int map_bar_2(struct adapter *);
 360 static void setup_memwin(struct adapter *);
 361 static int validate_mem_range(struct adapter *, uint32_t, int);
 362 static int fwmtype_to_hwmtype(int);
 363 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
 364     uint32_t *);
 365 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
 366 static uint32_t position_memwin(struct adapter *, int, uint32_t);
 367 static int cfg_itype_and_nqueues(struct adapter *, int, int,
 368     struct intrs_and_queues *);
 369 static int prep_firmware(struct adapter *);
 370 static int partition_resources(struct adapter *, const struct firmware *,
 371     const char *);
 372 static int get_params__pre_init(struct adapter *);
 373 static int get_params__post_init(struct adapter *);
 374 static int set_params__post_init(struct adapter *);
 375 static void t4_set_desc(struct adapter *);
 376 static void build_medialist(struct port_info *, struct ifmedia *);
 377 static int cxgbe_init_synchronized(struct port_info *);
 378 static int cxgbe_uninit_synchronized(struct port_info *);
 379 static int setup_intr_handlers(struct adapter *);
 380 static void quiesce_eq(struct adapter *, struct sge_eq *);
 381 static void quiesce_iq(struct adapter *, struct sge_iq *);
 382 static void quiesce_fl(struct adapter *, struct sge_fl *);
 383 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 384     driver_intr_t *, void *, char *);
 385 static int t4_free_irq(struct adapter *, struct irq *);
 386 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
 387     unsigned int);
 388 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 389 static void cxgbe_tick(void *);
 390 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 391 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
 392     struct mbuf *);
 393 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 394 static int fw_msg_not_handled(struct adapter *, const __be64 *);
 395 static int t4_sysctls(struct adapter *);
 396 static int cxgbe_sysctls(struct port_info *);
 397 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 398 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 399 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
 400 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 401 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 402 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 403 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 404 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 405 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
 406 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 407 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 408 #ifdef SBUF_DRAIN
 409 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 410 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 411 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 412 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
 413 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
 414 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 415 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 416 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 417 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 418 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 419 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 420 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 421 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
 422 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 423 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
 424 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 425 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 426 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 427 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 428 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 429 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 430 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
 431 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 432 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 433 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 434 #endif
 435 static inline void txq_start(struct ifnet *, struct sge_txq *);
 436 static uint32_t fconf_to_mode(uint32_t);
 437 static uint32_t mode_to_fconf(uint32_t);
 438 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 439 static int get_filter_mode(struct adapter *, uint32_t *);
 440 static int set_filter_mode(struct adapter *, uint32_t);
 441 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 442 static int get_filter(struct adapter *, struct t4_filter *);
 443 static int set_filter(struct adapter *, struct t4_filter *);
 444 static int del_filter(struct adapter *, struct t4_filter *);
 445 static void clear_filter(struct filter_entry *);
 446 static int set_filter_wr(struct adapter *, int);
 447 static int del_filter_wr(struct adapter *, int);
 448 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 449 static int load_fw(struct adapter *, struct t4_data *);
 450 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 451 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 452 static int set_sched_class(struct adapter *, struct t4_sched_params *);
 453 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
 454 #ifdef TCP_OFFLOAD
 455 static int toe_capability(struct port_info *, int);
 456 #endif
 457 static int mod_event(module_t, int, void *);
 458
 459 struct {
 460         uint16_t device;
 461         char *desc;
 462 } t4_pciids[] = {
 463         {0xa000, "Chelsio Terminator 4 FPGA"},
 464         {0x4400, "Chelsio T440-dbg"},
 465         {0x4401, "Chelsio T420-CR"},
 466         {0x4402, "Chelsio T422-CR"},
 467         {0x4403, "Chelsio T440-CR"},
 468         {0x4404, "Chelsio T420-BCH"},
 469         {0x4405, "Chelsio T440-BCH"},
 470         {0x4406, "Chelsio T440-CH"},
 471         {0x4407, "Chelsio T420-SO"},
 472         {0x4408, "Chelsio T420-CX"},
 473         {0x4409, "Chelsio T420-BT"},
 474         {0x440a, "Chelsio T404-BT"},
 475         {0x440e, "Chelsio T440-LP-CR"},
 476 }, t5_pciids[] = {
 477         {0xb000, "Chelsio Terminator 5 FPGA"},
 478         {0x5400, "Chelsio T580-dbg"},
 479         {0x5401,  "Chelsio T520-CR"},           /* 2 x 10G */
 480         {0x5402,  "Chelsio T522-CR"},           /* 2 x 10G, 2 X 1G */
 481         {0x5403,  "Chelsio T540-CR"},           /* 4 x 10G */
 482         {0x5407,  "Chelsio T520-SO"},           /* 2 x 10G, nomem */
 483         {0x5409,  "Chelsio T520-BT"},           /* 2 x 10GBaseT */
 484         {0x540a,  "Chelsio T504-BT"},           /* 4 x 1G */
 485         {0x540d,  "Chelsio T580-CR"},           /* 2 x 40G */
 486         {0x540e,  "Chelsio T540-LP-CR"},        /* 4 x 10G */
 487         {0x5410,  "Chelsio T580-LP-CR"},        /* 2 x 40G */
 488         {0x5411,  "Chelsio T520-LL-CR"},        /* 2 x 10G */
 489         {0x5412,  "Chelsio T560-CR"},           /* 1 x 40G, 2 x 10G */
 490         {0x5414,  "Chelsio T580-LP-SO-CR"},     /* 2 x 40G, nomem */
 491 #ifdef notyet
 492         {0x5404,  "Chelsio T520-BCH"},
 493         {0x5405,  "Chelsio T540-BCH"},
 494         {0x5406,  "Chelsio T540-CH"},
 495         {0x5408,  "Chelsio T520-CX"},
 496         {0x540b,  "Chelsio B520-SR"},
 497         {0x540c,  "Chelsio B504-BT"},
 498         {0x540f,  "Chelsio Amsterdam"},
 499         {0x5413,  "Chelsio T580-CHR"},
 500 #endif
 501 };
 502
 503 #ifdef TCP_OFFLOAD
 504 /*
 505  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 506  * exactly the same for both rxq and ofld_rxq.
 507  */
 508 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 509 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 510 #endif
 511
 512 /* No easy way to include t4_msg.h before adapter.h so we check this way */
 513 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
 514 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
 515
 516 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
 517
 518 static int
 519 t4_probe(device_t dev)
 520 {
 521         int i;
 522         uint16_t v = pci_get_vendor(dev);
 523         uint16_t d = pci_get_device(dev);
 524         uint8_t f = pci_get_function(dev);
 525
 526         if (v != PCI_VENDOR_ID_CHELSIO)
 527                 return (ENXIO);
 528
 529         /* Attach only to PF0 of the FPGA */
 530         if (d == 0xa000 && f != 0)
 531                 return (ENXIO);
 532
 533         for (i = 0; i < nitems(t4_pciids); i++) {
 534                 if (d == t4_pciids[i].device) {
 535                         device_set_desc(dev, t4_pciids[i].desc);
 536                         return (BUS_PROBE_DEFAULT);
 537                 }
 538         }
 539
 540         return (ENXIO);
 541 }
 542
 543 static int
 544 t5_probe(device_t dev)
 545 {
 546         int i;
 547         uint16_t v = pci_get_vendor(dev);
 548         uint16_t d = pci_get_device(dev);
 549         uint8_t f = pci_get_function(dev);
 550
 551         if (v != PCI_VENDOR_ID_CHELSIO)
 552                 return (ENXIO);
 553
 554         /* Attach only to PF0 of the FPGA */
 555         if (d == 0xb000 && f != 0)
 556                 return (ENXIO);
 557
 558         for (i = 0; i < nitems(t5_pciids); i++) {
 559                 if (d == t5_pciids[i].device) {
 560                         device_set_desc(dev, t5_pciids[i].desc);
 561                         return (BUS_PROBE_DEFAULT);
 562                 }
 563         }
 564
 565         return (ENXIO);
 566 }
 567
 568 static int
 569 t4_attach(device_t dev)
 570 {
 571         struct adapter *sc;
 572         int rc = 0, i, n10g, n1g, rqidx, tqidx;
 573         struct intrs_and_queues iaq;
 574         struct sge *s;
 575 #ifdef TCP_OFFLOAD
 576         int ofld_rqidx, ofld_tqidx;
 577 #endif
 578 #ifdef DEV_NETMAP
 579         int nm_rqidx, nm_tqidx;
 580 #endif
 581         const char *pcie_ts;
 582
 583         sc = device_get_softc(dev);
 584         sc->dev = dev;
 585
 586         pci_enable_busmaster(dev);
 587         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 588                 uint32_t v;
 589
 590                 pci_set_max_read_req(dev, 4096);
 591                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 592                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 593                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 594         }
 595
 596         sc->traceq = -1;
 597         mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
 598         snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
 599             device_get_nameunit(dev));
 600
 601         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 602             device_get_nameunit(dev));
 603         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 604         sx_xlock(&t4_list_lock);
 605         SLIST_INSERT_HEAD(&t4_list, sc, link);
 606         sx_xunlock(&t4_list_lock);
 607
 608         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 609         TAILQ_INIT(&sc->sfl);
 610         callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
 611
 612         rc = map_bars_0_and_4(sc);
 613         if (rc != 0)
 614                 goto done; /* error message displayed already */
 615
 616         /*
 617          * This is the real PF# to which we're attaching.  Works from within PCI
 618          * passthrough environments too, where pci_get_function() could return a
 619          * different PF# depending on the passthrough configuration.  We need to
 620          * use the real PF# in all our communication with the firmware.
 621          */
 622         sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 623         sc->mbox = sc->pf;
 624
 625         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 626         sc->an_handler = an_not_handled;
 627         for (i = 0; i < nitems(sc->cpl_handler); i++)
 628                 sc->cpl_handler[i] = cpl_not_handled;
 629         for (i = 0; i < nitems(sc->fw_msg_handler); i++)
 630                 sc->fw_msg_handler[i] = fw_msg_not_handled;
 631         t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 632         t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
 633         t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt);
 634         t4_init_sge_cpl_handlers(sc);
 635
 636         /* Prepare the adapter for operation */
 637         rc = -t4_prep_adapter(sc);
 638         if (rc != 0) {
 639                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 640                 goto done;
 641         }
 642
 643         /*
 644          * Do this really early, with the memory windows set up even before the
 645          * character device.  The userland tool's register i/o and mem read
 646          * will work even in "recovery mode".
 647          */
 648         setup_memwin(sc);
 649         sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
 650             device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
 651             device_get_nameunit(dev));
 652         if (sc->cdev == NULL)
 653                 device_printf(dev, "failed to create nexus char device.\n");
 654         else
 655                 sc->cdev->si_drv1 = sc;
 656
 657         /* Go no further if recovery mode has been requested. */
 658         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 659                 device_printf(dev, "recovery mode.\n");
 660                 goto done;
 661         }
 662
 663         /* Prepare the firmware for operation */
 664         rc = prep_firmware(sc);
 665         if (rc != 0)
 666                 goto done; /* error message displayed already */
 667
 668         rc = get_params__post_init(sc);
 669         if (rc != 0)
 670                 goto done; /* error message displayed already */
 671
 672         rc = set_params__post_init(sc);
 673         if (rc != 0)
 674                 goto done; /* error message displayed already */
 675
 676         rc = map_bar_2(sc);
 677         if (rc != 0)
 678                 goto done; /* error message displayed already */
 679
 680         rc = t4_create_dma_tag(sc);
 681         if (rc != 0)
 682                 goto done; /* error message displayed already */
 683
 684         /*
 685          * First pass over all the ports - allocate VIs and initialize some
 686          * basic parameters like mac address, port type, etc.  We also figure
 687          * out whether a port is 10G or 1G and use that information when
 688          * calculating how many interrupts to attempt to allocate.
 689          */
 690         n10g = n1g = 0;
 691         for_each_port(sc, i) {
 692                 struct port_info *pi;
 693
 694                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 695                 sc->port[i] = pi;
 696
 697                 /* These must be set before t4_port_init */
 698                 pi->adapter = sc;
 699                 pi->port_id = i;
 700
 701                 /* Allocate the vi and initialize parameters like mac addr */
 702                 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 703                 if (rc != 0) {
 704                         device_printf(dev, "unable to initialize port %d: %d\n",
 705                             i, rc);
 706                         free(pi, M_CXGBE);
 707                         sc->port[i] = NULL;
 708                         goto done;
 709                 }
 710
 711                 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX);
 712                 pi->link_cfg.requested_fc |= t4_pause_settings;
 713                 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX);
 714                 pi->link_cfg.fc |= t4_pause_settings;
 715
 716                 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
 717                 if (rc != 0) {
 718                         device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
 719                         free(pi, M_CXGBE);
 720                         sc->port[i] = NULL;
 721                         goto done;
 722                 }
 723
 724                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 725                     device_get_nameunit(dev), i);
 726                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 727                 sc->chan_map[pi->tx_chan] = i;
 728
 729                 if (is_10G_port(pi) || is_40G_port(pi)) {
 730                         n10g++;
 731                         pi->tmr_idx = t4_tmr_idx_10g;
 732                         pi->pktc_idx = t4_pktc_idx_10g;
 733                 } else {
 734                         n1g++;
 735                         pi->tmr_idx = t4_tmr_idx_1g;
 736                         pi->pktc_idx = t4_pktc_idx_1g;
 737                 }
 738
 739                 pi->xact_addr_filt = -1;
 740                 pi->linkdnrc = -1;
 741
 742                 pi->qsize_rxq = t4_qsize_rxq;
 743                 pi->qsize_txq = t4_qsize_txq;
 744
 745                 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
 746                 if (pi->dev == NULL) {
 747                         device_printf(dev,
 748                             "failed to add device for port %d.\n", i);
 749                         rc = ENXIO;
 750                         goto done;
 751                 }
 752                 device_set_softc(pi->dev, pi);
 753         }
 754
 755         /*
 756          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 757          */
 758         rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
 759         if (rc != 0)
 760                 goto done; /* error message displayed already */
 761
 762         sc->intr_type = iaq.intr_type;
 763         sc->intr_count = iaq.nirq;
 764
 765         s = &sc->sge;
 766         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 767         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 768         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
 769         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 770         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
 771 #ifdef TCP_OFFLOAD
 772         if (is_offload(sc)) {
 773                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 774                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 775                 s->neq += s->nofldtxq + s->nofldrxq;
 776                 s->niq += s->nofldrxq;
 777
 778                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 779                     M_CXGBE, M_ZERO | M_WAITOK);
 780                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 781                     M_CXGBE, M_ZERO | M_WAITOK);
 782         }
 783 #endif
 784 #ifdef DEV_NETMAP
 785         s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g;
 786         s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g;
 787         s->neq += s->nnmtxq + s->nnmrxq;
 788         s->niq += s->nnmrxq;
 789
 790         s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
 791             M_CXGBE, M_ZERO | M_WAITOK);
 792         s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
 793             M_CXGBE, M_ZERO | M_WAITOK);
 794 #endif
 795
 796         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 797             M_ZERO | M_WAITOK);
 798         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 799             M_ZERO | M_WAITOK);
 800         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 801             M_ZERO | M_WAITOK);
 802         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 803             M_ZERO | M_WAITOK);
 804         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 805             M_ZERO | M_WAITOK);
 806
 807         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 808             M_ZERO | M_WAITOK);
 809
 810         t4_init_l2t(sc, M_WAITOK);
 811
 812         /*
 813          * Second pass over the ports.  This time we know the number of rx and
 814          * tx queues that each port should get.
 815          */
 816         rqidx = tqidx = 0;
 817 #ifdef TCP_OFFLOAD
 818         ofld_rqidx = ofld_tqidx = 0;
 819 #endif
 820 #ifdef DEV_NETMAP
 821         nm_rqidx = nm_tqidx = 0;
 822 #endif
 823         for_each_port(sc, i) {
 824                 struct port_info *pi = sc->port[i];
 825
 826                 if (pi == NULL)
 827                         continue;
 828
 829                 pi->first_rxq = rqidx;
 830                 pi->first_txq = tqidx;
 831                 if (is_10G_port(pi) || is_40G_port(pi)) {
 832                         pi->flags |= iaq.intr_flags_10g;
 833                         pi->nrxq = iaq.nrxq10g;
 834                         pi->ntxq = iaq.ntxq10g;
 835                 } else {
 836                         pi->flags |= iaq.intr_flags_1g;
 837                         pi->nrxq = iaq.nrxq1g;
 838                         pi->ntxq = iaq.ntxq1g;
 839                 }
 840
 841                 if (pi->ntxq > 1)
 842                         pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
 843                 else
 844                         pi->rsrv_noflowq = 0;
 845
 846                 rqidx += pi->nrxq;
 847                 tqidx += pi->ntxq;
 848 #ifdef TCP_OFFLOAD
 849                 if (is_offload(sc)) {
 850                         pi->first_ofld_rxq = ofld_rqidx;
 851                         pi->first_ofld_txq = ofld_tqidx;
 852                         if (is_10G_port(pi) || is_40G_port(pi)) {
 853                                 pi->nofldrxq = iaq.nofldrxq10g;
 854                                 pi->nofldtxq = iaq.nofldtxq10g;
 855                         } else {
 856                                 pi->nofldrxq = iaq.nofldrxq1g;
 857                                 pi->nofldtxq = iaq.nofldtxq1g;
 858                         }
 859                         ofld_rqidx += pi->nofldrxq;
 860                         ofld_tqidx += pi->nofldtxq;
 861                 }
 862 #endif
 863 #ifdef DEV_NETMAP
 864                 pi->first_nm_rxq = nm_rqidx;
 865                 pi->first_nm_txq = nm_tqidx;
 866                 if (is_10G_port(pi) || is_40G_port(pi)) {
 867                         pi->nnmrxq = iaq.nnmrxq10g;
 868                         pi->nnmtxq = iaq.nnmtxq10g;
 869                 } else {
 870                         pi->nnmrxq = iaq.nnmrxq1g;
 871                         pi->nnmtxq = iaq.nnmtxq1g;
 872                 }
 873                 nm_rqidx += pi->nnmrxq;
 874                 nm_tqidx += pi->nnmtxq;
 875 #endif
 876         }
 877
 878         rc = setup_intr_handlers(sc);
 879         if (rc != 0) {
 880                 device_printf(dev,
 881                     "failed to setup interrupt handlers: %d\n", rc);
 882                 goto done;
 883         }
 884
 885         rc = bus_generic_attach(dev);
 886         if (rc != 0) {
 887                 device_printf(dev,
 888                     "failed to attach all child ports: %d\n", rc);
 889                 goto done;
 890         }
 891
 892         switch (sc->params.pci.speed) {
 893                 case 0x1:
 894                         pcie_ts = "2.5";
 895                         break;
 896                 case 0x2:
 897                         pcie_ts = "5.0";
 898                         break;
 899                 case 0x3:
 900                         pcie_ts = "8.0";
 901                         break;
 902                 default:
 903                         pcie_ts = "??";
 904                         break;
 905         }
 906         device_printf(dev,
 907             "PCIe x%d (%s GTS/s) (%d), %d ports, %d %s interrupt%s, %d eq, %d iq\n",
 908             sc->params.pci.width, pcie_ts, sc->params.pci.speed,
 909             sc->params.nports, sc->intr_count,
 910             sc->intr_type == INTR_MSIX ? "MSI-X" :
 911             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
 912             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
 913
 914         t4_set_desc(sc);
 915
 916 done:
 917         if (rc != 0 && sc->cdev) {
 918                 /* cdev was created and so cxgbetool works; recover that way. */
 919                 device_printf(dev,
 920                     "error during attach, adapter is now in recovery mode.\n");
 921                 rc = 0;
 922         }
 923
 924         if (rc != 0)
 925                 t4_detach(dev);
 926         else
 927                 t4_sysctls(sc);
 928
 929         return (rc);
 930 }
 931
 932 /*
 933  * Idempotent
 934  */
 935 static int
 936 t4_detach(device_t dev)
 937 {
 938         struct adapter *sc;
 939         struct port_info *pi;
 940         int i, rc;
 941
 942         sc = device_get_softc(dev);
 943
 944         if (sc->flags & FULL_INIT_DONE)
 945                 t4_intr_disable(sc);
 946
 947         if (sc->cdev) {
 948                 destroy_dev(sc->cdev);
 949                 sc->cdev = NULL;
 950         }
 951
 952         rc = bus_generic_detach(dev);
 953         if (rc) {
 954                 device_printf(dev,
 955                     "failed to detach child devices: %d\n", rc);
 956                 return (rc);
 957         }
 958
 959         for (i = 0; i < sc->intr_count; i++)
 960                 t4_free_irq(sc, &sc->irq[i]);
 961
 962         for (i = 0; i < MAX_NPORTS; i++) {
 963                 pi = sc->port[i];
 964                 if (pi) {
 965                         t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid);
 966                         if (pi->dev)
 967                                 device_delete_child(dev, pi->dev);
 968
 969                         mtx_destroy(&pi->pi_lock);
 970                         free(pi, M_CXGBE);
 971                 }
 972         }
 973
 974         if (sc->flags & FULL_INIT_DONE)
 975                 adapter_full_uninit(sc);
 976
 977         if (sc->flags & FW_OK)
 978                 t4_fw_bye(sc, sc->mbox);
 979
 980         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
 981                 pci_release_msi(dev);
 982
 983         if (sc->regs_res)
 984                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
 985                     sc->regs_res);
 986
 987         if (sc->udbs_res)
 988                 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
 989                     sc->udbs_res);
 990
 991         if (sc->msix_res)
 992                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
 993                     sc->msix_res);
 994
 995         if (sc->l2t)
 996                 t4_free_l2t(sc->l2t);
 997
 998 #ifdef TCP_OFFLOAD
 999         free(sc->sge.ofld_rxq, M_CXGBE);
1000         free(sc->sge.ofld_txq, M_CXGBE);
1001 #endif
1002 #ifdef DEV_NETMAP
1003         free(sc->sge.nm_rxq, M_CXGBE);
1004         free(sc->sge.nm_txq, M_CXGBE);
1005 #endif
1006         free(sc->irq, M_CXGBE);
1007         free(sc->sge.rxq, M_CXGBE);
1008         free(sc->sge.txq, M_CXGBE);
1009         free(sc->sge.ctrlq, M_CXGBE);
1010         free(sc->sge.iqmap, M_CXGBE);
1011         free(sc->sge.eqmap, M_CXGBE);
1012         free(sc->tids.ftid_tab, M_CXGBE);
1013         t4_destroy_dma_tag(sc);
1014         if (mtx_initialized(&sc->sc_lock)) {
1015                 sx_xlock(&t4_list_lock);
1016                 SLIST_REMOVE(&t4_list, sc, adapter, link);
1017                 sx_xunlock(&t4_list_lock);
1018                 mtx_destroy(&sc->sc_lock);
1019         }
1020
1021         if (mtx_initialized(&sc->tids.ftid_lock))
1022                 mtx_destroy(&sc->tids.ftid_lock);
1023         if (mtx_initialized(&sc->sfl_lock))
1024                 mtx_destroy(&sc->sfl_lock);
1025         if (mtx_initialized(&sc->ifp_lock))
1026                 mtx_destroy(&sc->ifp_lock);
1027
1028         bzero(sc, sizeof(*sc));
1029
1030         return (0);
1031 }
1032
1033 static int
1034 cxgbe_probe(device_t dev)
1035 {
1036         char buf[128];
1037         struct port_info *pi = device_get_softc(dev);
1038
1039         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
1040         device_set_desc_copy(dev, buf);
1041
1042         return (BUS_PROBE_DEFAULT);
1043 }
1044
1045 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
1046     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1047     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
1048 #define T4_CAP_ENABLE (T4_CAP)
1049
1050 static int
1051 cxgbe_attach(device_t dev)
1052 {
1053         struct port_info *pi = device_get_softc(dev);
1054         struct ifnet *ifp;
1055         char *s;
1056         int n, o;
1057
1058         /* Allocate an ifnet and set it up */
1059         ifp = if_alloc(IFT_ETHER);
1060         if (ifp == NULL) {
1061                 device_printf(dev, "Cannot allocate ifnet\n");
1062                 return (ENOMEM);
1063         }
1064         pi->ifp = ifp;
1065         ifp->if_softc = pi;
1066
1067         callout_init(&pi->tick, CALLOUT_MPSAFE);
1068
1069         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1070         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1071
1072         ifp->if_init = cxgbe_init;
1073         ifp->if_ioctl = cxgbe_ioctl;
1074         ifp->if_transmit = cxgbe_transmit;
1075         ifp->if_qflush = cxgbe_qflush;
1076
1077         ifp->if_capabilities = T4_CAP;
1078 #ifdef TCP_OFFLOAD
1079         if (is_offload(pi->adapter))
1080                 ifp->if_capabilities |= IFCAP_TOE;
1081 #endif
1082         ifp->if_capenable = T4_CAP_ENABLE;
1083         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1084             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1085
1086         /* Initialize ifmedia for this port */
1087         ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1088             cxgbe_media_status);
1089         build_medialist(pi, &pi->media);
1090
1091         pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1092             EVENTHANDLER_PRI_ANY);
1093
1094         ether_ifattach(ifp, pi->hw_addr);
1095
1096         n = 128;
1097         s = malloc(n, M_CXGBE, M_WAITOK);
1098         o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq);
1099         MPASS(n > o);
1100 #ifdef TCP_OFFLOAD
1101         if (is_offload(pi->adapter)) {
1102                 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)",
1103                     pi->nofldtxq, pi->nofldrxq);
1104                 MPASS(n > o);
1105         }
1106 #endif
1107 #ifdef DEV_NETMAP
1108         o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq,
1109             pi->nnmrxq);
1110         MPASS(n > o);
1111 #endif
1112         device_printf(dev, "%s\n", s);
1113         free(s, M_CXGBE);
1114
1115 #ifdef DEV_NETMAP
1116         /* nm_media handled here to keep implementation private to this file */
1117         ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change,
1118             cxgbe_media_status);
1119         build_medialist(pi, &pi->nm_media);
1120         create_netmap_ifnet(pi);        /* logs errors it something fails */
1121 #endif
1122         cxgbe_sysctls(pi);
1123
1124         return (0);
1125 }
1126
1127 static int
1128 cxgbe_detach(device_t dev)
1129 {
1130         struct port_info *pi = device_get_softc(dev);
1131         struct adapter *sc = pi->adapter;
1132         struct ifnet *ifp = pi->ifp;
1133
1134         /* Tell if_ioctl and if_init that the port is going away */
1135         ADAPTER_LOCK(sc);
1136         SET_DOOMED(pi);
1137         wakeup(&sc->flags);
1138         while (IS_BUSY(sc))
1139                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
1140         SET_BUSY(sc);
1141 #ifdef INVARIANTS
1142         sc->last_op = "t4detach";
1143         sc->last_op_thr = curthread;
1144 #endif
1145         ADAPTER_UNLOCK(sc);
1146
1147         if (pi->flags & HAS_TRACEQ) {
1148                 sc->traceq = -1;        /* cloner should not create ifnet */
1149                 t4_tracer_port_detach(sc);
1150         }
1151
1152         if (pi->vlan_c)
1153                 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
1154
1155         PORT_LOCK(pi);
1156         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1157         callout_stop(&pi->tick);
1158         PORT_UNLOCK(pi);
1159         callout_drain(&pi->tick);
1160
1161         /* Let detach proceed even if these fail. */
1162         cxgbe_uninit_synchronized(pi);
1163         port_full_uninit(pi);
1164
1165         ifmedia_removeall(&pi->media);
1166         ether_ifdetach(pi->ifp);
1167         if_free(pi->ifp);
1168
1169 #ifdef DEV_NETMAP
1170         /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */
1171         destroy_netmap_ifnet(pi);
1172 #endif
1173
1174         ADAPTER_LOCK(sc);
1175         CLR_BUSY(sc);
1176         wakeup(&sc->flags);
1177         ADAPTER_UNLOCK(sc);
1178
1179         return (0);
1180 }
1181
1182 static void
1183 cxgbe_init(void *arg)
1184 {
1185         struct port_info *pi = arg;
1186         struct adapter *sc = pi->adapter;
1187
1188         if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
1189                 return;
1190         cxgbe_init_synchronized(pi);
1191         end_synchronized_op(sc, 0);
1192 }
1193
1194 static int
1195 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1196 {
1197         int rc = 0, mtu, flags, can_sleep;
1198         struct port_info *pi = ifp->if_softc;
1199         struct adapter *sc = pi->adapter;
1200         struct ifreq *ifr = (struct ifreq *)data;
1201         uint32_t mask;
1202
1203         switch (cmd) {
1204         case SIOCSIFMTU:
1205                 mtu = ifr->ifr_mtu;
1206                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1207                         return (EINVAL);
1208
1209                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
1210                 if (rc)
1211                         return (rc);
1212                 ifp->if_mtu = mtu;
1213                 if (pi->flags & PORT_INIT_DONE) {
1214                         t4_update_fl_bufsize(ifp);
1215                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1216                                 rc = update_mac_settings(ifp, XGMAC_MTU);
1217                 }
1218                 end_synchronized_op(sc, 0);
1219                 break;
1220
1221         case SIOCSIFFLAGS:
1222                 can_sleep = 0;
1223 redo_sifflags:
1224                 rc = begin_synchronized_op(sc, pi,
1225                     can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg");
1226                 if (rc)
1227                         return (rc);
1228
1229                 if (ifp->if_flags & IFF_UP) {
1230                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1231                                 flags = pi->if_flags;
1232                                 if ((ifp->if_flags ^ flags) &
1233                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1234                                         if (can_sleep == 1) {
1235                                                 end_synchronized_op(sc, 0);
1236                                                 can_sleep = 0;
1237                                                 goto redo_sifflags;
1238                                         }
1239                                         rc = update_mac_settings(ifp,
1240                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1241                                 }
1242                         } else {
1243                                 if (can_sleep == 0) {
1244                                         end_synchronized_op(sc, LOCK_HELD);
1245                                         can_sleep = 1;
1246                                         goto redo_sifflags;
1247                                 }
1248                                 rc = cxgbe_init_synchronized(pi);
1249                         }
1250                         pi->if_flags = ifp->if_flags;
1251                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1252                         if (can_sleep == 0) {
1253                                 end_synchronized_op(sc, LOCK_HELD);
1254                                 can_sleep = 1;
1255                                 goto redo_sifflags;
1256                         }
1257                         rc = cxgbe_uninit_synchronized(pi);
1258                 }
1259                 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD);
1260                 break;
1261
1262         case SIOCADDMULTI:
1263         case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1264                 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
1265                 if (rc)
1266                         return (rc);
1267                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1268                         rc = update_mac_settings(ifp, XGMAC_MCADDRS);
1269                 end_synchronized_op(sc, LOCK_HELD);
1270                 break;
1271
1272         case SIOCSIFCAP:
1273                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
1274                 if (rc)
1275                         return (rc);
1276
1277                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1278                 if (mask & IFCAP_TXCSUM) {
1279                         ifp->if_capenable ^= IFCAP_TXCSUM;
1280                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1281
1282                         if (IFCAP_TSO4 & ifp->if_capenable &&
1283                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1284                                 ifp->if_capenable &= ~IFCAP_TSO4;
1285                                 if_printf(ifp,
1286                                     "tso4 disabled due to -txcsum.\n");
1287                         }
1288                 }
1289                 if (mask & IFCAP_TXCSUM_IPV6) {
1290                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1291                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1292
1293                         if (IFCAP_TSO6 & ifp->if_capenable &&
1294                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1295                                 ifp->if_capenable &= ~IFCAP_TSO6;
1296                                 if_printf(ifp,
1297                                     "tso6 disabled due to -txcsum6.\n");
1298                         }
1299                 }
1300                 if (mask & IFCAP_RXCSUM)
1301                         ifp->if_capenable ^= IFCAP_RXCSUM;
1302                 if (mask & IFCAP_RXCSUM_IPV6)
1303                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1304
1305                 /*
1306                  * Note that we leave CSUM_TSO alone (it is always set).  The
1307                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1308                  * sending a TSO request our way, so it's sufficient to toggle
1309                  * IFCAP_TSOx only.
1310                  */
1311                 if (mask & IFCAP_TSO4) {
1312                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1313                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1314                                 if_printf(ifp, "enable txcsum first.\n");
1315                                 rc = EAGAIN;
1316                                 goto fail;
1317                         }
1318                         ifp->if_capenable ^= IFCAP_TSO4;
1319                 }
1320                 if (mask & IFCAP_TSO6) {
1321                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1322                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1323                                 if_printf(ifp, "enable txcsum6 first.\n");
1324                                 rc = EAGAIN;
1325                                 goto fail;
1326                         }
1327                         ifp->if_capenable ^= IFCAP_TSO6;
1328                 }
1329                 if (mask & IFCAP_LRO) {
1330 #if defined(INET) || defined(INET6)
1331                         int i;
1332                         struct sge_rxq *rxq;
1333
1334                         ifp->if_capenable ^= IFCAP_LRO;
1335                         for_each_rxq(pi, i, rxq) {
1336                                 if (ifp->if_capenable & IFCAP_LRO)
1337                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1338                                 else
1339                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1340                         }
1341 #endif
1342                 }
1343 #ifdef TCP_OFFLOAD
1344                 if (mask & IFCAP_TOE) {
1345                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1346
1347                         rc = toe_capability(pi, enable);
1348                         if (rc != 0)
1349                                 goto fail;
1350
1351                         ifp->if_capenable ^= mask;
1352                 }
1353 #endif
1354                 if (mask & IFCAP_VLAN_HWTAGGING) {
1355                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1356                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1357                                 rc = update_mac_settings(ifp, XGMAC_VLANEX);
1358                 }
1359                 if (mask & IFCAP_VLAN_MTU) {
1360                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1361
1362                         /* Need to find out how to disable auto-mtu-inflation */
1363                 }
1364                 if (mask & IFCAP_VLAN_HWTSO)
1365                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1366                 if (mask & IFCAP_VLAN_HWCSUM)
1367                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1368
1369 #ifdef VLAN_CAPABILITIES
1370                 VLAN_CAPABILITIES(ifp);
1371 #endif
1372 fail:
1373                 end_synchronized_op(sc, 0);
1374                 break;
1375
1376         case SIOCSIFMEDIA:
1377         case SIOCGIFMEDIA:
1378                 ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1379                 break;
1380
1381         case SIOCGI2C: {
1382                 struct ifi2creq i2c;
1383
1384                 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
1385                 if (rc != 0)
1386                         break;
1387                 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
1388                         rc = EPERM;
1389                         break;
1390                 }
1391                 if (i2c.len > sizeof(i2c.data)) {
1392                         rc = EINVAL;
1393                         break;
1394                 }
1395                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4i2c");
1396                 if (rc)
1397                         return (rc);
1398                 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr,
1399                     i2c.offset, i2c.len, &i2c.data[0]);
1400                 end_synchronized_op(sc, 0);
1401                 if (rc == 0)
1402                         rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
1403                 break;
1404         }
1405
1406         default:
1407                 rc = ether_ioctl(ifp, cmd, data);
1408         }
1409
1410         return (rc);
1411 }
1412
1413 static int
1414 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1415 {
1416         struct port_info *pi = ifp->if_softc;
1417         struct adapter *sc = pi->adapter;
1418         struct sge_txq *txq = &sc->sge.txq[pi->first_txq];
1419         struct buf_ring *br;
1420         int rc;
1421
1422         M_ASSERTPKTHDR(m);
1423
1424         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1425                 m_freem(m);
1426                 return (ENETDOWN);
1427         }
1428
1429         if (m->m_flags & M_FLOWID)
1430                 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq))
1431                     + pi->rsrv_noflowq);
1432         br = txq->br;
1433
1434         if (TXQ_TRYLOCK(txq) == 0) {
1435                 struct sge_eq *eq = &txq->eq;
1436
1437                 /*
1438                  * It is possible that t4_eth_tx finishes up and releases the
1439                  * lock between the TRYLOCK above and the drbr_enqueue here.  We
1440                  * need to make sure that this mbuf doesn't just sit there in
1441                  * the drbr.
1442                  */
1443
1444                 rc = drbr_enqueue(ifp, br, m);
1445                 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 &&
1446                     !(eq->flags & EQ_DOOMED))
1447                         callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq);
1448                 return (rc);
1449         }
1450
1451         /*
1452          * txq->m is the mbuf that is held up due to a temporary shortage of
1453          * resources and it should be put on the wire first.  Then what's in
1454          * drbr and finally the mbuf that was just passed in to us.
1455          *
1456          * Return code should indicate the fate of the mbuf that was passed in
1457          * this time.
1458          */
1459
1460         TXQ_LOCK_ASSERT_OWNED(txq);
1461         if (drbr_needs_enqueue(ifp, br) || txq->m) {
1462
1463                 /* Queued for transmission. */
1464
1465                 rc = drbr_enqueue(ifp, br, m);
1466                 m = txq->m ? txq->m : drbr_dequeue(ifp, br);
1467                 (void) t4_eth_tx(ifp, txq, m);
1468                 TXQ_UNLOCK(txq);
1469                 return (rc);
1470         }
1471
1472         /* Direct transmission. */
1473         rc = t4_eth_tx(ifp, txq, m);
1474         if (rc != 0 && txq->m)
1475                 rc = 0; /* held, will be transmitted soon (hopefully) */
1476
1477         TXQ_UNLOCK(txq);
1478         return (rc);
1479 }
1480
1481 static void
1482 cxgbe_qflush(struct ifnet *ifp)
1483 {
1484         struct port_info *pi = ifp->if_softc;
1485         struct sge_txq *txq;
1486         int i;
1487         struct mbuf *m;
1488
1489         /* queues do not exist if !PORT_INIT_DONE. */
1490         if (pi->flags & PORT_INIT_DONE) {
1491                 for_each_txq(pi, i, txq) {
1492                         TXQ_LOCK(txq);
1493                         m_freem(txq->m);
1494                         txq->m = NULL;
1495                         while ((m = buf_ring_dequeue_sc(txq->br)) != NULL)
1496                                 m_freem(m);
1497                         TXQ_UNLOCK(txq);
1498                 }
1499         }
1500         if_qflush(ifp);
1501 }
1502
1503 static int
1504 cxgbe_media_change(struct ifnet *ifp)
1505 {
1506         struct port_info *pi = ifp->if_softc;
1507
1508         device_printf(pi->dev, "%s unimplemented.\n", __func__);
1509
1510         return (EOPNOTSUPP);
1511 }
1512
1513 static void
1514 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1515 {
1516         struct port_info *pi = ifp->if_softc;
1517         struct ifmedia *media = NULL;
1518         struct ifmedia_entry *cur;
1519         int speed = pi->link_cfg.speed;
1520         int data = (pi->port_type << 8) | pi->mod_type;
1521
1522         if (ifp == pi->ifp)
1523                 media = &pi->media;
1524 #ifdef DEV_NETMAP
1525         else if (ifp == pi->nm_ifp)
1526                 media = &pi->nm_media;
1527 #endif
1528         MPASS(media != NULL);
1529
1530         cur = media->ifm_cur;
1531         if (cur->ifm_data != data) {
1532                 build_medialist(pi, media);
1533                 cur = media->ifm_cur;
1534         }
1535
1536         ifmr->ifm_status = IFM_AVALID;
1537         if (!pi->link_cfg.link_ok)
1538                 return;
1539
1540         ifmr->ifm_status |= IFM_ACTIVE;
1541
1542         /* active and current will differ iff current media is autoselect. */
1543         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1544                 return;
1545
1546         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1547         if (speed == SPEED_10000)
1548                 ifmr->ifm_active |= IFM_10G_T;
1549         else if (speed == SPEED_1000)
1550                 ifmr->ifm_active |= IFM_1000_T;
1551         else if (speed == SPEED_100)
1552                 ifmr->ifm_active |= IFM_100_TX;
1553         else if (speed == SPEED_10)
1554                 ifmr->ifm_active |= IFM_10_T;
1555         else
1556                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1557                             speed));
1558 }
1559
1560 void
1561 t4_fatal_err(struct adapter *sc)
1562 {
1563         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1564         t4_intr_disable(sc);
1565         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1566             device_get_nameunit(sc->dev));
1567 }
1568
1569 static int
1570 map_bars_0_and_4(struct adapter *sc)
1571 {
1572         sc->regs_rid = PCIR_BAR(0);
1573         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1574             &sc->regs_rid, RF_ACTIVE);
1575         if (sc->regs_res == NULL) {
1576                 device_printf(sc->dev, "cannot map registers.\n");
1577                 return (ENXIO);
1578         }
1579         sc->bt = rman_get_bustag(sc->regs_res);
1580         sc->bh = rman_get_bushandle(sc->regs_res);
1581         sc->mmio_len = rman_get_size(sc->regs_res);
1582         setbit(&sc->doorbells, DOORBELL_KDB);
1583
1584         sc->msix_rid = PCIR_BAR(4);
1585         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1586             &sc->msix_rid, RF_ACTIVE);
1587         if (sc->msix_res == NULL) {
1588                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1589                 return (ENXIO);
1590         }
1591
1592         return (0);
1593 }
1594
1595 static int
1596 map_bar_2(struct adapter *sc)
1597 {
1598
1599         /*
1600          * T4: only iWARP driver uses the userspace doorbells.  There is no need
1601          * to map it if RDMA is disabled.
1602          */
1603         if (is_t4(sc) && sc->rdmacaps == 0)
1604                 return (0);
1605
1606         sc->udbs_rid = PCIR_BAR(2);
1607         sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1608             &sc->udbs_rid, RF_ACTIVE);
1609         if (sc->udbs_res == NULL) {
1610                 device_printf(sc->dev, "cannot map doorbell BAR.\n");
1611                 return (ENXIO);
1612         }
1613         sc->udbs_base = rman_get_virtual(sc->udbs_res);
1614
1615         if (is_t5(sc)) {
1616                 setbit(&sc->doorbells, DOORBELL_UDB);
1617 #if defined(__i386__) || defined(__amd64__)
1618                 if (t5_write_combine) {
1619                         int rc;
1620
1621                         /*
1622                          * Enable write combining on BAR2.  This is the
1623                          * userspace doorbell BAR and is split into 128B
1624                          * (UDBS_SEG_SIZE) doorbell regions, each associated
1625                          * with an egress queue.  The first 64B has the doorbell
1626                          * and the second 64B can be used to submit a tx work
1627                          * request with an implicit doorbell.
1628                          */
1629
1630                         rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
1631                             rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
1632                         if (rc == 0) {
1633                                 clrbit(&sc->doorbells, DOORBELL_UDB);
1634                                 setbit(&sc->doorbells, DOORBELL_WCWR);
1635                                 setbit(&sc->doorbells, DOORBELL_UDBWC);
1636                         } else {
1637                                 device_printf(sc->dev,
1638                                     "couldn't enable write combining: %d\n",
1639                                     rc);
1640                         }
1641
1642                         t4_write_reg(sc, A_SGE_STAT_CFG,
1643                             V_STATSOURCE_T5(7) | V_STATMODE(0));
1644                 }
1645 #endif
1646         }
1647
1648         return (0);
1649 }
1650
1651 static const struct memwin t4_memwin[] = {
1652         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1653         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1654         { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
1655 };
1656
1657 static const struct memwin t5_memwin[] = {
1658         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1659         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1660         { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1661 };
1662
1663 static void
1664 setup_memwin(struct adapter *sc)
1665 {
1666         const struct memwin *mw;
1667         int i, n;
1668         uint32_t bar0;
1669
1670         if (is_t4(sc)) {
1671                 /*
1672                  * Read low 32b of bar0 indirectly via the hardware backdoor
1673                  * mechanism.  Works from within PCI passthrough environments
1674                  * too, where rman_get_start() can return a different value.  We
1675                  * need to program the T4 memory window decoders with the actual
1676                  * addresses that will be coming across the PCIe link.
1677                  */
1678                 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1679                 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1680
1681                 mw = &t4_memwin[0];
1682                 n = nitems(t4_memwin);
1683         } else {
1684                 /* T5 uses the relative offset inside the PCIe BAR */
1685                 bar0 = 0;
1686
1687                 mw = &t5_memwin[0];
1688                 n = nitems(t5_memwin);
1689         }
1690
1691         for (i = 0; i < n; i++, mw++) {
1692                 t4_write_reg(sc,
1693                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
1694                     (mw->base + bar0) | V_BIR(0) |
1695                     V_WINDOW(ilog2(mw->aperture) - 10));
1696         }
1697
1698         /* flush */
1699         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1700 }
1701
1702 /*
1703  * Verify that the memory range specified by the addr/len pair is valid and lies
1704  * entirely within a single region (EDCx or MCx).
1705  */
1706 static int
1707 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
1708 {
1709         uint32_t em, addr_len, maddr, mlen;
1710
1711         /* Memory can only be accessed in naturally aligned 4 byte units */
1712         if (addr & 3 || len & 3 || len == 0)
1713                 return (EINVAL);
1714
1715         /* Enabled memories */
1716         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1717         if (em & F_EDRAM0_ENABLE) {
1718                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1719                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1720                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1721                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1722                     addr + len <= maddr + mlen)
1723                         return (0);
1724         }
1725         if (em & F_EDRAM1_ENABLE) {
1726                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1727                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1728                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1729                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1730                     addr + len <= maddr + mlen)
1731                         return (0);
1732         }
1733         if (em & F_EXT_MEM_ENABLE) {
1734                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1735                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1736                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1737                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1738                     addr + len <= maddr + mlen)
1739                         return (0);
1740         }
1741         if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
1742                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1743                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1744                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1745                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1746                     addr + len <= maddr + mlen)
1747                         return (0);
1748         }
1749
1750         return (EFAULT);
1751 }
1752
1753 static int
1754 fwmtype_to_hwmtype(int mtype)
1755 {
1756
1757         switch (mtype) {
1758         case FW_MEMTYPE_EDC0:
1759                 return (MEM_EDC0);
1760         case FW_MEMTYPE_EDC1:
1761                 return (MEM_EDC1);
1762         case FW_MEMTYPE_EXTMEM:
1763                 return (MEM_MC0);
1764         case FW_MEMTYPE_EXTMEM1:
1765                 return (MEM_MC1);
1766         default:
1767                 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
1768         }
1769 }
1770
1771 /*
1772  * Verify that the memory range specified by the memtype/offset/len pair is
1773  * valid and lies entirely within the memtype specified.  The global address of
1774  * the start of the range is returned in addr.
1775  */
1776 static int
1777 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1778     uint32_t *addr)
1779 {
1780         uint32_t em, addr_len, maddr, mlen;
1781
1782         /* Memory can only be accessed in naturally aligned 4 byte units */
1783         if (off & 3 || len & 3 || len == 0)
1784                 return (EINVAL);
1785
1786         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1787         switch (fwmtype_to_hwmtype(mtype)) {
1788         case MEM_EDC0:
1789                 if (!(em & F_EDRAM0_ENABLE))
1790                         return (EINVAL);
1791                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1792                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1793                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1794                 break;
1795         case MEM_EDC1:
1796                 if (!(em & F_EDRAM1_ENABLE))
1797                         return (EINVAL);
1798                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1799                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1800                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1801                 break;
1802         case MEM_MC:
1803                 if (!(em & F_EXT_MEM_ENABLE))
1804                         return (EINVAL);
1805                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1806                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1807                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1808                 break;
1809         case MEM_MC1:
1810                 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
1811                         return (EINVAL);
1812                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1813                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1814                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1815                 break;
1816         default:
1817                 return (EINVAL);
1818         }
1819
1820         if (mlen > 0 && off < mlen && off + len <= mlen) {
1821                 *addr = maddr + off;    /* global address */
1822                 return (0);
1823         }
1824
1825         return (EFAULT);
1826 }
1827
1828 static void
1829 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1830 {
1831         const struct memwin *mw;
1832
1833         if (is_t4(sc)) {
1834                 KASSERT(win >= 0 && win < nitems(t4_memwin),
1835                     ("%s: incorrect memwin# (%d)", __func__, win));
1836                 mw = &t4_memwin[win];
1837         } else {
1838                 KASSERT(win >= 0 && win < nitems(t5_memwin),
1839                     ("%s: incorrect memwin# (%d)", __func__, win));
1840                 mw = &t5_memwin[win];
1841         }
1842
1843         if (base != NULL)
1844                 *base = mw->base;
1845         if (aperture != NULL)
1846                 *aperture = mw->aperture;
1847 }
1848
1849 /*
1850  * Positions the memory window such that it can be used to access the specified
1851  * address in the chip's address space.  The return value is the offset of addr
1852  * from the start of the window.
1853  */
1854 static uint32_t
1855 position_memwin(struct adapter *sc, int n, uint32_t addr)
1856 {
1857         uint32_t start, pf;
1858         uint32_t reg;
1859
1860         KASSERT(n >= 0 && n <= 3,
1861             ("%s: invalid window %d.", __func__, n));
1862         KASSERT((addr & 3) == 0,
1863             ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
1864
1865         if (is_t4(sc)) {
1866                 pf = 0;
1867                 start = addr & ~0xf;    /* start must be 16B aligned */
1868         } else {
1869                 pf = V_PFNUM(sc->pf);
1870                 start = addr & ~0x7f;   /* start must be 128B aligned */
1871         }
1872         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1873
1874         t4_write_reg(sc, reg, start | pf);
1875         t4_read_reg(sc, reg);
1876
1877         return (addr - start);
1878 }
1879
1880 static int
1881 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1882     struct intrs_and_queues *iaq)
1883 {
1884         int rc, itype, navail, nrxq10g, nrxq1g, n;
1885         int nofldrxq10g = 0, nofldrxq1g = 0;
1886         int nnmrxq10g = 0, nnmrxq1g = 0;
1887
1888         bzero(iaq, sizeof(*iaq));
1889
1890         iaq->ntxq10g = t4_ntxq10g;
1891         iaq->ntxq1g = t4_ntxq1g;
1892         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1893         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1894         iaq->rsrv_noflowq = t4_rsrv_noflowq;
1895 #ifdef TCP_OFFLOAD
1896         if (is_offload(sc)) {
1897                 iaq->nofldtxq10g = t4_nofldtxq10g;
1898                 iaq->nofldtxq1g = t4_nofldtxq1g;
1899                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1900                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1901         }
1902 #endif
1903 #ifdef DEV_NETMAP
1904         iaq->nnmtxq10g = t4_nnmtxq10g;
1905         iaq->nnmtxq1g = t4_nnmtxq1g;
1906         iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g;
1907         iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g;
1908 #endif
1909
1910         for (itype = INTR_MSIX; itype; itype >>= 1) {
1911
1912                 if ((itype & t4_intr_types) == 0)
1913                         continue;       /* not allowed */
1914
1915                 if (itype == INTR_MSIX)
1916                         navail = pci_msix_count(sc->dev);
1917                 else if (itype == INTR_MSI)
1918                         navail = pci_msi_count(sc->dev);
1919                 else
1920                         navail = 1;
1921 restart:
1922                 if (navail == 0)
1923                         continue;
1924
1925                 iaq->intr_type = itype;
1926                 iaq->intr_flags_10g = 0;
1927                 iaq->intr_flags_1g = 0;
1928
1929                 /*
1930                  * Best option: an interrupt vector for errors, one for the
1931                  * firmware event queue, and one for every rxq (NIC, TOE, and
1932                  * netmap).
1933                  */
1934                 iaq->nirq = T4_EXTRA_INTR;
1935                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g);
1936                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g);
1937                 if (iaq->nirq <= navail &&
1938                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
1939                         iaq->intr_flags_10g = INTR_ALL;
1940                         iaq->intr_flags_1g = INTR_ALL;
1941                         goto allocate;
1942                 }
1943
1944                 /*
1945                  * Second best option: a vector for errors, one for the firmware
1946                  * event queue, and vectors for either all the NIC rx queues or
1947                  * all the TOE rx queues.  The queues that don't get vectors
1948                  * will forward their interrupts to those that do.
1949                  *
1950                  * Note: netmap rx queues cannot be created early and so they
1951                  * can't be setup to receive forwarded interrupts for others.
1952                  */
1953                 iaq->nirq = T4_EXTRA_INTR;
1954                 if (nrxq10g >= nofldrxq10g) {
1955                         iaq->intr_flags_10g = INTR_RXQ;
1956                         iaq->nirq += n10g * nrxq10g;
1957 #ifdef DEV_NETMAP
1958                         iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g);
1959 #endif
1960                 } else {
1961                         iaq->intr_flags_10g = INTR_OFLD_RXQ;
1962                         iaq->nirq += n10g * nofldrxq10g;
1963 #ifdef DEV_NETMAP
1964                         iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g);
1965 #endif
1966                 }
1967                 if (nrxq1g >= nofldrxq1g) {
1968                         iaq->intr_flags_1g = INTR_RXQ;
1969                         iaq->nirq += n1g * nrxq1g;
1970 #ifdef DEV_NETMAP
1971                         iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g);
1972 #endif
1973                 } else {
1974                         iaq->intr_flags_1g = INTR_OFLD_RXQ;
1975                         iaq->nirq += n1g * nofldrxq1g;
1976 #ifdef DEV_NETMAP
1977                         iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g);
1978 #endif
1979                 }
1980                 if (iaq->nirq <= navail &&
1981                     (itype != INTR_MSI || powerof2(iaq->nirq)))
1982                         goto allocate;
1983
1984                 /*
1985                  * Next best option: an interrupt vector for errors, one for the
1986                  * firmware event queue, and at least one per port.  At this
1987                  * point we know we'll have to downsize nrxq and/or nofldrxq
1988                  * and/or nnmrxq to fit what's available to us.
1989                  */
1990                 iaq->nirq = T4_EXTRA_INTR;
1991                 iaq->nirq += n10g + n1g;
1992                 if (iaq->nirq <= navail) {
1993                         int leftover = navail - iaq->nirq;
1994
1995                         if (n10g > 0) {
1996                                 int target = max(nrxq10g, nofldrxq10g);
1997
1998                                 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
1999                                     INTR_RXQ : INTR_OFLD_RXQ;
2000
2001                                 n = 1;
2002                                 while (n < target && leftover >= n10g) {
2003                                         leftover -= n10g;
2004                                         iaq->nirq += n10g;
2005                                         n++;
2006                                 }
2007                                 iaq->nrxq10g = min(n, nrxq10g);
2008 #ifdef TCP_OFFLOAD
2009                                 iaq->nofldrxq10g = min(n, nofldrxq10g);
2010 #endif
2011 #ifdef DEV_NETMAP
2012                                 iaq->nnmrxq10g = min(n, nnmrxq10g);
2013 #endif
2014                         }
2015
2016                         if (n1g > 0) {
2017                                 int target = max(nrxq1g, nofldrxq1g);
2018
2019                                 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
2020                                     INTR_RXQ : INTR_OFLD_RXQ;
2021
2022                                 n = 1;
2023                                 while (n < target && leftover >= n1g) {
2024                                         leftover -= n1g;
2025                                         iaq->nirq += n1g;
2026                                         n++;
2027                                 }
2028                                 iaq->nrxq1g = min(n, nrxq1g);
2029 #ifdef TCP_OFFLOAD
2030                                 iaq->nofldrxq1g = min(n, nofldrxq1g);
2031 #endif
2032 #ifdef DEV_NETMAP
2033                                 iaq->nnmrxq1g = min(n, nnmrxq1g);
2034 #endif
2035                         }
2036
2037                         if (itype != INTR_MSI || powerof2(iaq->nirq))
2038                                 goto allocate;
2039                 }
2040
2041                 /*
2042                  * Least desirable option: one interrupt vector for everything.
2043                  */
2044                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
2045                 iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
2046 #ifdef TCP_OFFLOAD
2047                 if (is_offload(sc))
2048                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2049 #endif
2050 #ifdef DEV_NETMAP
2051                 iaq->nnmrxq10g = iaq->nnmrxq1g = 1;
2052 #endif
2053
2054 allocate:
2055                 navail = iaq->nirq;
2056                 rc = 0;
2057                 if (itype == INTR_MSIX)
2058                         rc = pci_alloc_msix(sc->dev, &navail);
2059                 else if (itype == INTR_MSI)
2060                         rc = pci_alloc_msi(sc->dev, &navail);
2061
2062                 if (rc == 0) {
2063                         if (navail == iaq->nirq)
2064                                 return (0);
2065
2066                         /*
2067                          * Didn't get the number requested.  Use whatever number
2068                          * the kernel is willing to allocate (it's in navail).
2069                          */
2070                         device_printf(sc->dev, "fewer vectors than requested, "
2071                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
2072                             itype, iaq->nirq, navail);
2073                         pci_release_msi(sc->dev);
2074                         goto restart;
2075                 }
2076
2077                 device_printf(sc->dev,
2078                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
2079                     itype, rc, iaq->nirq, navail);
2080         }
2081
2082         device_printf(sc->dev,
2083             "failed to find a usable interrupt type.  "
2084             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
2085             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
2086
2087         return (ENXIO);
2088 }
2089
2090 #define FW_VERSION(chip) ( \
2091     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
2092     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
2093     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
2094     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
2095 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
2096
2097 struct fw_info {
2098         uint8_t chip;
2099         char *kld_name;
2100         char *fw_mod_name;
2101         struct fw_hdr fw_hdr;   /* XXX: waste of space, need a sparse struct */
2102 } fw_info[] = {
2103         {
2104                 .chip = CHELSIO_T4,
2105                 .kld_name = "t4fw_cfg",
2106                 .fw_mod_name = "t4fw",
2107                 .fw_hdr = {
2108                         .chip = FW_HDR_CHIP_T4,
2109                         .fw_ver = htobe32_const(FW_VERSION(T4)),
2110                         .intfver_nic = FW_INTFVER(T4, NIC),
2111                         .intfver_vnic = FW_INTFVER(T4, VNIC),
2112                         .intfver_ofld = FW_INTFVER(T4, OFLD),
2113                         .intfver_ri = FW_INTFVER(T4, RI),
2114                         .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
2115                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
2116                         .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
2117                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
2118                 },
2119         }, {
2120                 .chip = CHELSIO_T5,
2121                 .kld_name = "t5fw_cfg",
2122                 .fw_mod_name = "t5fw",
2123                 .fw_hdr = {
2124                         .chip = FW_HDR_CHIP_T5,
2125                         .fw_ver = htobe32_const(FW_VERSION(T5)),
2126                         .intfver_nic = FW_INTFVER(T5, NIC),
2127                         .intfver_vnic = FW_INTFVER(T5, VNIC),
2128                         .intfver_ofld = FW_INTFVER(T5, OFLD),
2129                         .intfver_ri = FW_INTFVER(T5, RI),
2130                         .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
2131                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
2132                         .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
2133                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
2134                 },
2135         }
2136 };
2137
2138 static struct fw_info *
2139 find_fw_info(int chip)
2140 {
2141         int i;
2142
2143         for (i = 0; i < nitems(fw_info); i++) {
2144                 if (fw_info[i].chip == chip)
2145                         return (&fw_info[i]);
2146         }
2147         return (NULL);
2148 }
2149
2150 /*
2151  * Is the given firmware API compatible with the one the driver was compiled
2152  * with?
2153  */
2154 static int
2155 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
2156 {
2157
2158         /* short circuit if it's the exact same firmware version */
2159         if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
2160                 return (1);
2161
2162         /*
2163          * XXX: Is this too conservative?  Perhaps I should limit this to the
2164          * features that are supported in the driver.
2165          */
2166 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
2167         if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
2168             SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
2169             SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
2170                 return (1);
2171 #undef SAME_INTF
2172
2173         return (0);
2174 }
2175
2176 /*
2177  * The firmware in the KLD is usable, but should it be installed?  This routine
2178  * explains itself in detail if it indicates the KLD firmware should be
2179  * installed.
2180  */
2181 static int
2182 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
2183 {
2184         const char *reason;
2185
2186         if (!card_fw_usable) {
2187                 reason = "incompatible or unusable";
2188                 goto install;
2189         }
2190
2191         if (k > c) {
2192                 reason = "older than the version bundled with this driver";
2193                 goto install;
2194         }
2195
2196         if (t4_fw_install == 2 && k != c) {
2197                 reason = "different than the version bundled with this driver";
2198                 goto install;
2199         }
2200
2201         return (0);
2202
2203 install:
2204         if (t4_fw_install == 0) {
2205                 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2206                     "but the driver is prohibited from installing a different "
2207                     "firmware on the card.\n",
2208                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2209                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2210
2211                 return (0);
2212         }
2213
2214         device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2215             "installing firmware %u.%u.%u.%u on card.\n",
2216             G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2217             G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2218             G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2219             G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2220
2221         return (1);
2222 }
2223 /*
2224  * Establish contact with the firmware and determine if we are the master driver
2225  * or not, and whether we are responsible for chip initialization.
2226  */
2227 static int
2228 prep_firmware(struct adapter *sc)
2229 {
2230         const struct firmware *fw = NULL, *default_cfg;
2231         int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2232         enum dev_state state;
2233         struct fw_info *fw_info;
2234         struct fw_hdr *card_fw;         /* fw on the card */
2235         const struct fw_hdr *kld_fw;    /* fw in the KLD */
2236         const struct fw_hdr *drv_fw;    /* fw header the driver was compiled
2237                                            against */
2238
2239         /* Contact firmware. */
2240         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2241         if (rc < 0 || state == DEV_STATE_ERR) {
2242                 rc = -rc;
2243                 device_printf(sc->dev,
2244                     "failed to connect to the firmware: %d, %d.\n", rc, state);
2245                 return (rc);
2246         }
2247         pf = rc;
2248         if (pf == sc->mbox)
2249                 sc->flags |= MASTER_PF;
2250         else if (state == DEV_STATE_UNINIT) {
2251                 /*
2252                  * We didn't get to be the master so we definitely won't be
2253                  * configuring the chip.  It's a bug if someone else hasn't
2254                  * configured it already.
2255                  */
2256                 device_printf(sc->dev, "couldn't be master(%d), "
2257                     "device not already initialized either(%d).\n", rc, state);
2258                 return (EDOOFUS);
2259         }
2260
2261         /* This is the firmware whose headers the driver was compiled against */
2262         fw_info = find_fw_info(chip_id(sc));
2263         if (fw_info == NULL) {
2264                 device_printf(sc->dev,
2265                     "unable to look up firmware information for chip %d.\n",
2266                     chip_id(sc));
2267                 return (EINVAL);
2268         }
2269         drv_fw = &fw_info->fw_hdr;
2270
2271         /*
2272          * The firmware KLD contains many modules.  The KLD name is also the
2273          * name of the module that contains the default config file.
2274          */
2275         default_cfg = firmware_get(fw_info->kld_name);
2276
2277         /* Read the header of the firmware on the card */
2278         card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2279         rc = -t4_read_flash(sc, FLASH_FW_START,
2280             sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2281         if (rc == 0)
2282                 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2283         else {
2284                 device_printf(sc->dev,
2285                     "Unable to read card's firmware header: %d\n", rc);
2286                 card_fw_usable = 0;
2287         }
2288
2289         /* This is the firmware in the KLD */
2290         fw = firmware_get(fw_info->fw_mod_name);
2291         if (fw != NULL) {
2292                 kld_fw = (const void *)fw->data;
2293                 kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2294         } else {
2295                 kld_fw = NULL;
2296                 kld_fw_usable = 0;
2297         }
2298
2299         if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2300             (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2301                 /*
2302                  * Common case: the firmware on the card is an exact match and
2303                  * the KLD is an exact match too, or the KLD is
2304                  * absent/incompatible.  Note that t4_fw_install = 2 is ignored
2305                  * here -- use cxgbetool loadfw if you want to reinstall the
2306                  * same firmware as the one on the card.
2307                  */
2308         } else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2309             should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2310             be32toh(card_fw->fw_ver))) {
2311
2312                 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
2313                 if (rc != 0) {
2314                         device_printf(sc->dev,
2315                             "failed to install firmware: %d\n", rc);
2316                         goto done;
2317                 }
2318
2319                 /* Installed successfully, update the cached header too. */
2320                 memcpy(card_fw, kld_fw, sizeof(*card_fw));
2321                 card_fw_usable = 1;
2322                 need_fw_reset = 0;      /* already reset as part of load_fw */
2323         }
2324
2325         if (!card_fw_usable) {
2326                 uint32_t d, c, k;
2327
2328                 d = ntohl(drv_fw->fw_ver);
2329                 c = ntohl(card_fw->fw_ver);
2330                 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
2331
2332                 device_printf(sc->dev, "Cannot find a usable firmware: "
2333                     "fw_install %d, chip state %d, "
2334                     "driver compiled with %d.%d.%d.%d, "
2335                     "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
2336                     t4_fw_install, state,
2337                     G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
2338                     G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
2339                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2340                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
2341                     G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2342                     G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2343                 rc = EINVAL;
2344                 goto done;
2345         }
2346
2347         /* We're using whatever's on the card and it's known to be good. */
2348         sc->params.fw_vers = ntohl(card_fw->fw_ver);
2349         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
2350             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2351             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2352             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2353             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2354         t4_get_tp_version(sc, &sc->params.tp_vers);
2355
2356         /* Reset device */
2357         if (need_fw_reset &&
2358             (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
2359                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
2360                 if (rc != ETIMEDOUT && rc != EIO)
2361                         t4_fw_bye(sc, sc->mbox);
2362                 goto done;
2363         }
2364         sc->flags |= FW_OK;
2365
2366         rc = get_params__pre_init(sc);
2367         if (rc != 0)
2368                 goto done; /* error message displayed already */
2369
2370         /* Partition adapter resources as specified in the config file. */
2371         if (state == DEV_STATE_UNINIT) {
2372
2373                 KASSERT(sc->flags & MASTER_PF,
2374                     ("%s: trying to change chip settings when not master.",
2375                     __func__));
2376
2377                 rc = partition_resources(sc, default_cfg, fw_info->kld_name);
2378                 if (rc != 0)
2379                         goto done;      /* error message displayed already */
2380
2381                 t4_tweak_chip_settings(sc);
2382
2383                 /* get basic stuff going */
2384                 rc = -t4_fw_initialize(sc, sc->mbox);
2385                 if (rc != 0) {
2386                         device_printf(sc->dev, "fw init failed: %d.\n", rc);
2387                         goto done;
2388                 }
2389         } else {
2390                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
2391                 sc->cfcsum = 0;
2392         }
2393
2394 done:
2395         free(card_fw, M_CXGBE);
2396         if (fw != NULL)
2397                 firmware_put(fw, FIRMWARE_UNLOAD);
2398         if (default_cfg != NULL)
2399                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
2400
2401         return (rc);
2402 }
2403
2404 #define FW_PARAM_DEV(param) \
2405         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2406          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2407 #define FW_PARAM_PFVF(param) \
2408         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2409          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2410
2411 /*
2412  * Partition chip resources for use between various PFs, VFs, etc.
2413  */
2414 static int
2415 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
2416     const char *name_prefix)
2417 {
2418         const struct firmware *cfg = NULL;
2419         int rc = 0;
2420         struct fw_caps_config_cmd caps;
2421         uint32_t mtype, moff, finicsum, cfcsum;
2422
2423         /*
2424          * Figure out what configuration file to use.  Pick the default config
2425          * file for the card if the user hasn't specified one explicitly.
2426          */
2427         snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
2428         if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
2429                 /* Card specific overrides go here. */
2430                 if (pci_get_device(sc->dev) == 0x440a)
2431                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
2432                 if (is_fpga(sc))
2433                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
2434         }
2435
2436         /*
2437          * We need to load another module if the profile is anything except
2438          * "default" or "flash".
2439          */
2440         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
2441             strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2442                 char s[32];
2443
2444                 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
2445                 cfg = firmware_get(s);
2446                 if (cfg == NULL) {
2447                         if (default_cfg != NULL) {
2448                                 device_printf(sc->dev,
2449                                     "unable to load module \"%s\" for "
2450                                     "configuration profile \"%s\", will use "
2451                                     "the default config file instead.\n",
2452                                     s, sc->cfg_file);
2453                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2454                                     "%s", DEFAULT_CF);
2455                         } else {
2456                                 device_printf(sc->dev,
2457                                     "unable to load module \"%s\" for "
2458                                     "configuration profile \"%s\", will use "
2459                                     "the config file on the card's flash "
2460                                     "instead.\n", s, sc->cfg_file);
2461                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2462                                     "%s", FLASH_CF);
2463                         }
2464                 }
2465         }
2466
2467         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
2468             default_cfg == NULL) {
2469                 device_printf(sc->dev,
2470                     "default config file not available, will use the config "
2471                     "file on the card's flash instead.\n");
2472                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
2473         }
2474
2475         if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2476                 u_int cflen, i, n;
2477                 const uint32_t *cfdata;
2478                 uint32_t param, val, addr, off, mw_base, mw_aperture;
2479
2480                 KASSERT(cfg != NULL || default_cfg != NULL,
2481                     ("%s: no config to upload", __func__));
2482
2483                 /*
2484                  * Ask the firmware where it wants us to upload the config file.
2485                  */
2486                 param = FW_PARAM_DEV(CF);
2487                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2488                 if (rc != 0) {
2489                         /* No support for config file?  Shouldn't happen. */
2490                         device_printf(sc->dev,
2491                             "failed to query config file location: %d.\n", rc);
2492                         goto done;
2493                 }
2494                 mtype = G_FW_PARAMS_PARAM_Y(val);
2495                 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
2496
2497                 /*
2498                  * XXX: sheer laziness.  We deliberately added 4 bytes of
2499                  * useless stuffing/comments at the end of the config file so
2500                  * it's ok to simply throw away the last remaining bytes when
2501                  * the config file is not an exact multiple of 4.  This also
2502                  * helps with the validate_mt_off_len check.
2503                  */
2504                 if (cfg != NULL) {
2505                         cflen = cfg->datasize & ~3;
2506                         cfdata = cfg->data;
2507                 } else {
2508                         cflen = default_cfg->datasize & ~3;
2509                         cfdata = default_cfg->data;
2510                 }
2511
2512                 if (cflen > FLASH_CFG_MAX_SIZE) {
2513                         device_printf(sc->dev,
2514                             "config file too long (%d, max allowed is %d).  "
2515                             "Will try to use the config on the card, if any.\n",
2516                             cflen, FLASH_CFG_MAX_SIZE);
2517                         goto use_config_on_flash;
2518                 }
2519
2520                 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
2521                 if (rc != 0) {
2522                         device_printf(sc->dev,
2523                             "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
2524                             "Will try to use the config on the card, if any.\n",
2525                             __func__, mtype, moff, cflen, rc);
2526                         goto use_config_on_flash;
2527                 }
2528
2529                 memwin_info(sc, 2, &mw_base, &mw_aperture);
2530                 while (cflen) {
2531                         off = position_memwin(sc, 2, addr);
2532                         n = min(cflen, mw_aperture - off);
2533                         for (i = 0; i < n; i += 4)
2534                                 t4_write_reg(sc, mw_base + off + i, *cfdata++);
2535                         cflen -= n;
2536                         addr += n;
2537                 }
2538         } else {
2539 use_config_on_flash:
2540                 mtype = FW_MEMTYPE_FLASH;
2541                 moff = t4_flash_cfg_addr(sc);
2542         }
2543
2544         bzero(&caps, sizeof(caps));
2545         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2546             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2547         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
2548             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
2549             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
2550         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2551         if (rc != 0) {
2552                 device_printf(sc->dev,
2553                     "failed to pre-process config file: %d "
2554                     "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
2555                 goto done;
2556         }
2557
2558         finicsum = be32toh(caps.finicsum);
2559         cfcsum = be32toh(caps.cfcsum);
2560         if (finicsum != cfcsum) {
2561                 device_printf(sc->dev,
2562                     "WARNING: config file checksum mismatch: %08x %08x\n",
2563                     finicsum, cfcsum);
2564         }
2565         sc->cfcsum = cfcsum;
2566
2567 #define LIMIT_CAPS(x) do { \
2568         caps.x &= htobe16(t4_##x##_allowed); \
2569 } while (0)
2570
2571         /*
2572          * Let the firmware know what features will (not) be used so it can tune
2573          * things accordingly.
2574          */
2575         LIMIT_CAPS(linkcaps);
2576         LIMIT_CAPS(niccaps);
2577         LIMIT_CAPS(toecaps);
2578         LIMIT_CAPS(rdmacaps);
2579         LIMIT_CAPS(iscsicaps);
2580         LIMIT_CAPS(fcoecaps);
2581 #undef LIMIT_CAPS
2582
2583         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2584             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
2585         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2586         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
2587         if (rc != 0) {
2588                 device_printf(sc->dev,
2589                     "failed to process config file: %d.\n", rc);
2590         }
2591 done:
2592         if (cfg != NULL)
2593                 firmware_put(cfg, FIRMWARE_UNLOAD);
2594         return (rc);
2595 }
2596
2597 /*
2598  * Retrieve parameters that are needed (or nice to have) very early.
2599  */
2600 static int
2601 get_params__pre_init(struct adapter *sc)
2602 {
2603         int rc;
2604         uint32_t param[2], val[2];
2605         struct fw_devlog_cmd cmd;
2606         struct devlog_params *dlog = &sc->params.devlog;
2607
2608         param[0] = FW_PARAM_DEV(PORTVEC);
2609         param[1] = FW_PARAM_DEV(CCLK);
2610         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2611         if (rc != 0) {
2612                 device_printf(sc->dev,
2613                     "failed to query parameters (pre_init): %d.\n", rc);
2614                 return (rc);
2615         }
2616
2617         sc->params.portvec = val[0];
2618         sc->params.nports = bitcount32(val[0]);
2619         sc->params.vpd.cclk = val[1];
2620
2621         /* Read device log parameters. */
2622         bzero(&cmd, sizeof(cmd));
2623         cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
2624             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2625         cmd.retval_len16 = htobe32(FW_LEN16(cmd));
2626         rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
2627         if (rc != 0) {
2628                 device_printf(sc->dev,
2629                     "failed to get devlog parameters: %d.\n", rc);
2630                 bzero(dlog, sizeof (*dlog));
2631                 rc = 0; /* devlog isn't critical for device operation */
2632         } else {
2633                 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
2634                 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
2635                 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
2636                 dlog->size = be32toh(cmd.memsize_devlog);
2637         }
2638
2639         return (rc);
2640 }
2641
2642 /*
2643  * Retrieve various parameters that are of interest to the driver.  The device
2644  * has been initialized by the firmware at this point.
2645  */
2646 static int
2647 get_params__post_init(struct adapter *sc)
2648 {
2649         int rc;
2650         uint32_t param[7], val[7];
2651         struct fw_caps_config_cmd caps;
2652
2653         param[0] = FW_PARAM_PFVF(IQFLINT_START);
2654         param[1] = FW_PARAM_PFVF(EQ_START);
2655         param[2] = FW_PARAM_PFVF(FILTER_START);
2656         param[3] = FW_PARAM_PFVF(FILTER_END);
2657         param[4] = FW_PARAM_PFVF(L2T_START);
2658         param[5] = FW_PARAM_PFVF(L2T_END);
2659         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2660         if (rc != 0) {
2661                 device_printf(sc->dev,
2662                     "failed to query parameters (post_init): %d.\n", rc);
2663                 return (rc);
2664         }
2665
2666         sc->sge.iq_start = val[0];
2667         sc->sge.eq_start = val[1];
2668         sc->tids.ftid_base = val[2];
2669         sc->tids.nftids = val[3] - val[2] + 1;
2670         sc->params.ftid_min = val[2];
2671         sc->params.ftid_max = val[3];
2672         sc->vres.l2t.start = val[4];
2673         sc->vres.l2t.size = val[5] - val[4] + 1;
2674         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
2675             ("%s: L2 table size (%u) larger than expected (%u)",
2676             __func__, sc->vres.l2t.size, L2T_SIZE));
2677
2678         /* get capabilites */
2679         bzero(&caps, sizeof(caps));
2680         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2681             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2682         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2683         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2684         if (rc != 0) {
2685                 device_printf(sc->dev,
2686                     "failed to get card capabilities: %d.\n", rc);
2687                 return (rc);
2688         }
2689
2690 #define READ_CAPS(x) do { \
2691         sc->x = htobe16(caps.x); \
2692 } while (0)
2693         READ_CAPS(linkcaps);
2694         READ_CAPS(niccaps);
2695         READ_CAPS(toecaps);
2696         READ_CAPS(rdmacaps);
2697         READ_CAPS(iscsicaps);
2698         READ_CAPS(fcoecaps);
2699
2700         if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
2701                 param[0] = FW_PARAM_PFVF(ETHOFLD_START);
2702                 param[1] = FW_PARAM_PFVF(ETHOFLD_END);
2703                 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2704                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
2705                 if (rc != 0) {
2706                         device_printf(sc->dev,
2707                             "failed to query NIC parameters: %d.\n", rc);
2708                         return (rc);
2709                 }
2710                 sc->tids.etid_base = val[0];
2711                 sc->params.etid_min = val[0];
2712                 sc->tids.netids = val[1] - val[0] + 1;
2713                 sc->params.netids = sc->tids.netids;
2714                 sc->params.eo_wr_cred = val[2];
2715                 sc->params.ethoffload = 1;
2716         }
2717
2718         if (sc->toecaps) {
2719                 /* query offload-related parameters */
2720                 param[0] = FW_PARAM_DEV(NTID);
2721                 param[1] = FW_PARAM_PFVF(SERVER_START);
2722                 param[2] = FW_PARAM_PFVF(SERVER_END);
2723                 param[3] = FW_PARAM_PFVF(TDDP_START);
2724                 param[4] = FW_PARAM_PFVF(TDDP_END);
2725                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2726                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2727                 if (rc != 0) {
2728                         device_printf(sc->dev,
2729                             "failed to query TOE parameters: %d.\n", rc);
2730                         return (rc);
2731                 }
2732                 sc->tids.ntids = val[0];
2733                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
2734                 sc->tids.stid_base = val[1];
2735                 sc->tids.nstids = val[2] - val[1] + 1;
2736                 sc->vres.ddp.start = val[3];
2737                 sc->vres.ddp.size = val[4] - val[3] + 1;
2738                 sc->params.ofldq_wr_cred = val[5];
2739                 sc->params.offload = 1;
2740         }
2741         if (sc->rdmacaps) {
2742                 param[0] = FW_PARAM_PFVF(STAG_START);
2743                 param[1] = FW_PARAM_PFVF(STAG_END);
2744                 param[2] = FW_PARAM_PFVF(RQ_START);
2745                 param[3] = FW_PARAM_PFVF(RQ_END);
2746                 param[4] = FW_PARAM_PFVF(PBL_START);
2747                 param[5] = FW_PARAM_PFVF(PBL_END);
2748                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2749                 if (rc != 0) {
2750                         device_printf(sc->dev,
2751                             "failed to query RDMA parameters(1): %d.\n", rc);
2752                         return (rc);
2753                 }
2754                 sc->vres.stag.start = val[0];
2755                 sc->vres.stag.size = val[1] - val[0] + 1;
2756                 sc->vres.rq.start = val[2];
2757                 sc->vres.rq.size = val[3] - val[2] + 1;
2758                 sc->vres.pbl.start = val[4];
2759                 sc->vres.pbl.size = val[5] - val[4] + 1;
2760
2761                 param[0] = FW_PARAM_PFVF(SQRQ_START);
2762                 param[1] = FW_PARAM_PFVF(SQRQ_END);
2763                 param[2] = FW_PARAM_PFVF(CQ_START);
2764                 param[3] = FW_PARAM_PFVF(CQ_END);
2765                 param[4] = FW_PARAM_PFVF(OCQ_START);
2766                 param[5] = FW_PARAM_PFVF(OCQ_END);
2767                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2768                 if (rc != 0) {
2769                         device_printf(sc->dev,
2770                             "failed to query RDMA parameters(2): %d.\n", rc);
2771                         return (rc);
2772                 }
2773                 sc->vres.qp.start = val[0];
2774                 sc->vres.qp.size = val[1] - val[0] + 1;
2775                 sc->vres.cq.start = val[2];
2776                 sc->vres.cq.size = val[3] - val[2] + 1;
2777                 sc->vres.ocq.start = val[4];
2778                 sc->vres.ocq.size = val[5] - val[4] + 1;
2779         }
2780         if (sc->iscsicaps) {
2781                 param[0] = FW_PARAM_PFVF(ISCSI_START);
2782                 param[1] = FW_PARAM_PFVF(ISCSI_END);
2783                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2784                 if (rc != 0) {
2785                         device_printf(sc->dev,
2786                             "failed to query iSCSI parameters: %d.\n", rc);
2787                         return (rc);
2788                 }
2789                 sc->vres.iscsi.start = val[0];
2790                 sc->vres.iscsi.size = val[1] - val[0] + 1;
2791         }
2792
2793         /*
2794          * We've got the params we wanted to query via the firmware.  Now grab
2795          * some others directly from the chip.
2796          */
2797         rc = t4_read_chip_settings(sc);
2798
2799         return (rc);
2800 }
2801
2802 static int
2803 set_params__post_init(struct adapter *sc)
2804 {
2805         uint32_t param, val;
2806
2807         /* ask for encapsulated CPLs */
2808         param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
2809         val = 1;
2810         (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2811
2812         return (0);
2813 }
2814
2815 #undef FW_PARAM_PFVF
2816 #undef FW_PARAM_DEV
2817
2818 static void
2819 t4_set_desc(struct adapter *sc)
2820 {
2821         char buf[128];
2822         struct adapter_params *p = &sc->params;
2823
2824         snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
2825             "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
2826             chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
2827
2828         device_set_desc_copy(sc->dev, buf);
2829 }
2830
2831 static void
2832 build_medialist(struct port_info *pi, struct ifmedia *media)
2833 {
2834         int data, m;
2835
2836         PORT_LOCK(pi);
2837
2838         ifmedia_removeall(media);
2839
2840         m = IFM_ETHER | IFM_FDX;
2841         data = (pi->port_type << 8) | pi->mod_type;
2842
2843         switch(pi->port_type) {
2844         case FW_PORT_TYPE_BT_XFI:
2845                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2846                 break;
2847
2848         case FW_PORT_TYPE_BT_XAUI:
2849                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2850                 /* fall through */
2851
2852         case FW_PORT_TYPE_BT_SGMII:
2853                 ifmedia_add(media, m | IFM_1000_T, data, NULL);
2854                 ifmedia_add(media, m | IFM_100_TX, data, NULL);
2855                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL);
2856                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2857                 break;
2858
2859         case FW_PORT_TYPE_CX4:
2860                 ifmedia_add(media, m | IFM_10G_CX4, data, NULL);
2861                 ifmedia_set(media, m | IFM_10G_CX4);
2862                 break;
2863
2864         case FW_PORT_TYPE_QSFP_10G:
2865         case FW_PORT_TYPE_SFP:
2866         case FW_PORT_TYPE_FIBER_XFI:
2867         case FW_PORT_TYPE_FIBER_XAUI:
2868                 switch (pi->mod_type) {
2869
2870                 case FW_PORT_MOD_TYPE_LR:
2871                         ifmedia_add(media, m | IFM_10G_LR, data, NULL);
2872                         ifmedia_set(media, m | IFM_10G_LR);
2873                         break;
2874
2875                 case FW_PORT_MOD_TYPE_SR:
2876                         ifmedia_add(media, m | IFM_10G_SR, data, NULL);
2877                         ifmedia_set(media, m | IFM_10G_SR);
2878                         break;
2879
2880                 case FW_PORT_MOD_TYPE_LRM:
2881                         ifmedia_add(media, m | IFM_10G_LRM, data, NULL);
2882                         ifmedia_set(media, m | IFM_10G_LRM);
2883                         break;
2884
2885                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2886                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2887                         ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL);
2888                         ifmedia_set(media, m | IFM_10G_TWINAX);
2889                         break;
2890
2891                 case FW_PORT_MOD_TYPE_NONE:
2892                         m &= ~IFM_FDX;
2893                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2894                         ifmedia_set(media, m | IFM_NONE);
2895                         break;
2896
2897                 case FW_PORT_MOD_TYPE_NA:
2898                 case FW_PORT_MOD_TYPE_ER:
2899                 default:
2900                         device_printf(pi->dev,
2901                             "unknown port_type (%d), mod_type (%d)\n",
2902                             pi->port_type, pi->mod_type);
2903                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2904                         ifmedia_set(media, m | IFM_UNKNOWN);
2905                         break;
2906                 }
2907                 break;
2908
2909         case FW_PORT_TYPE_QSFP:
2910                 switch (pi->mod_type) {
2911
2912                 case FW_PORT_MOD_TYPE_LR:
2913                         ifmedia_add(media, m | IFM_40G_LR4, data, NULL);
2914                         ifmedia_set(media, m | IFM_40G_LR4);
2915                         break;
2916
2917                 case FW_PORT_MOD_TYPE_SR:
2918                         ifmedia_add(media, m | IFM_40G_SR4, data, NULL);
2919                         ifmedia_set(media, m | IFM_40G_SR4);
2920                         break;
2921
2922                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2923                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2924                         ifmedia_add(media, m | IFM_40G_CR4, data, NULL);
2925                         ifmedia_set(media, m | IFM_40G_CR4);
2926                         break;
2927
2928                 case FW_PORT_MOD_TYPE_NONE:
2929                         m &= ~IFM_FDX;
2930                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2931                         ifmedia_set(media, m | IFM_NONE);
2932                         break;
2933
2934                 default:
2935                         device_printf(pi->dev,
2936                             "unknown port_type (%d), mod_type (%d)\n",
2937                             pi->port_type, pi->mod_type);
2938                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2939                         ifmedia_set(media, m | IFM_UNKNOWN);
2940                         break;
2941                 }
2942                 break;
2943
2944         default:
2945                 device_printf(pi->dev,
2946                     "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
2947                     pi->mod_type);
2948                 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2949                 ifmedia_set(media, m | IFM_UNKNOWN);
2950                 break;
2951         }
2952
2953         PORT_UNLOCK(pi);
2954 }
2955
2956 #define FW_MAC_EXACT_CHUNK      7
2957
2958 /*
2959  * Program the port's XGMAC based on parameters in ifnet.  The caller also
2960  * indicates which parameters should be programmed (the rest are left alone).
2961  */
2962 int
2963 update_mac_settings(struct ifnet *ifp, int flags)
2964 {
2965         int rc = 0;
2966         struct port_info *pi = ifp->if_softc;
2967         struct adapter *sc = pi->adapter;
2968         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
2969         uint16_t viid = 0xffff;
2970         int16_t *xact_addr_filt = NULL;
2971
2972         ASSERT_SYNCHRONIZED_OP(sc);
2973         KASSERT(flags, ("%s: not told what to update.", __func__));
2974
2975         if (ifp == pi->ifp) {
2976                 viid = pi->viid;
2977                 xact_addr_filt = &pi->xact_addr_filt;
2978         }
2979 #ifdef DEV_NETMAP
2980         else if (ifp == pi->nm_ifp) {
2981                 viid = pi->nm_viid;
2982                 xact_addr_filt = &pi->nm_xact_addr_filt;
2983         }
2984 #endif
2985         if (flags & XGMAC_MTU)
2986                 mtu = ifp->if_mtu;
2987
2988         if (flags & XGMAC_PROMISC)
2989                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
2990
2991         if (flags & XGMAC_ALLMULTI)
2992                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
2993
2994         if (flags & XGMAC_VLANEX)
2995                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
2996
2997         if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
2998                 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti,
2999                     1, vlanex, false);
3000                 if (rc) {
3001                         if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
3002                             rc);
3003                         return (rc);
3004                 }
3005         }
3006
3007         if (flags & XGMAC_UCADDR) {
3008                 uint8_t ucaddr[ETHER_ADDR_LEN];
3009
3010                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
3011                 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr,
3012                     true, true);
3013                 if (rc < 0) {
3014                         rc = -rc;
3015                         if_printf(ifp, "change_mac failed: %d\n", rc);
3016                         return (rc);
3017                 } else {
3018                         *xact_addr_filt = rc;
3019                         rc = 0;
3020                 }
3021         }
3022
3023         if (flags & XGMAC_MCADDRS) {
3024                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
3025                 int del = 1;
3026                 uint64_t hash = 0;
3027                 struct ifmultiaddr *ifma;
3028                 int i = 0, j;
3029
3030                 if_maddr_rlock(ifp);
3031                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
3032                         if (ifma->ifma_addr->sa_family != AF_LINK)
3033                                 continue;
3034                         mcaddr[i++] =
3035                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
3036
3037                         if (i == FW_MAC_EXACT_CHUNK) {
3038                                 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del,
3039                                     i, mcaddr, NULL, &hash, 0);
3040                                 if (rc < 0) {
3041                                         rc = -rc;
3042                                         for (j = 0; j < i; j++) {
3043                                                 if_printf(ifp,
3044                                                     "failed to add mc address"
3045                                                     " %02x:%02x:%02x:"
3046                                                     "%02x:%02x:%02x rc=%d\n",
3047                                                     mcaddr[j][0], mcaddr[j][1],
3048                                                     mcaddr[j][2], mcaddr[j][3],
3049                                                     mcaddr[j][4], mcaddr[j][5],
3050                                                     rc);
3051                                         }
3052                                         goto mcfail;
3053                                 }
3054                                 del = 0;
3055                                 i = 0;
3056                         }
3057                 }
3058                 if (i > 0) {
3059                         rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i,
3060                             mcaddr, NULL, &hash, 0);
3061                         if (rc < 0) {
3062                                 rc = -rc;
3063                                 for (j = 0; j < i; j++) {
3064                                         if_printf(ifp,
3065                                             "failed to add mc address"
3066                                             " %02x:%02x:%02x:"
3067                                             "%02x:%02x:%02x rc=%d\n",
3068                                             mcaddr[j][0], mcaddr[j][1],
3069                                             mcaddr[j][2], mcaddr[j][3],
3070                                             mcaddr[j][4], mcaddr[j][5],
3071                                             rc);
3072                                 }
3073                                 goto mcfail;
3074                         }
3075                 }
3076
3077                 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0);
3078                 if (rc != 0)
3079                         if_printf(ifp, "failed to set mc address hash: %d", rc);
3080 mcfail:
3081                 if_maddr_runlock(ifp);
3082         }
3083
3084         return (rc);
3085 }
3086
3087 int
3088 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
3089     char *wmesg)
3090 {
3091         int rc, pri;
3092
3093 #ifdef WITNESS
3094         /* the caller thinks it's ok to sleep, but is it really? */
3095         if (flags & SLEEP_OK)
3096                 pause("t4slptst", 1);
3097 #endif
3098
3099         if (INTR_OK)
3100                 pri = PCATCH;
3101         else
3102                 pri = 0;
3103
3104         ADAPTER_LOCK(sc);
3105         for (;;) {
3106
3107                 if (pi && IS_DOOMED(pi)) {
3108                         rc = ENXIO;
3109                         goto done;
3110                 }
3111
3112                 if (!IS_BUSY(sc)) {
3113                         rc = 0;
3114                         break;
3115                 }
3116
3117                 if (!(flags & SLEEP_OK)) {
3118                         rc = EBUSY;
3119                         goto done;
3120                 }
3121
3122                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
3123                         rc = EINTR;
3124                         goto done;
3125                 }
3126         }
3127
3128         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
3129         SET_BUSY(sc);
3130 #ifdef INVARIANTS
3131         sc->last_op = wmesg;
3132         sc->last_op_thr = curthread;
3133 #endif
3134
3135 done:
3136         if (!(flags & HOLD_LOCK) || rc)
3137                 ADAPTER_UNLOCK(sc);
3138
3139         return (rc);
3140 }
3141
3142 void
3143 end_synchronized_op(struct adapter *sc, int flags)
3144 {
3145
3146         if (flags & LOCK_HELD)
3147                 ADAPTER_LOCK_ASSERT_OWNED(sc);
3148         else
3149                 ADAPTER_LOCK(sc);
3150
3151         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
3152         CLR_BUSY(sc);
3153         wakeup(&sc->flags);
3154         ADAPTER_UNLOCK(sc);
3155 }
3156
3157 static int
3158 cxgbe_init_synchronized(struct port_info *pi)
3159 {
3160         struct adapter *sc = pi->adapter;
3161         struct ifnet *ifp = pi->ifp;
3162         int rc = 0;
3163
3164         ASSERT_SYNCHRONIZED_OP(sc);
3165
3166         if (isset(&sc->open_device_map, pi->port_id)) {
3167                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
3168                     ("mismatch between open_device_map and if_drv_flags"));
3169                 return (0);     /* already running */
3170         }
3171
3172         if (!(sc->flags & FULL_INIT_DONE) &&
3173             ((rc = adapter_full_init(sc)) != 0))
3174                 return (rc);    /* error message displayed already */
3175
3176         if (!(pi->flags & PORT_INIT_DONE) &&
3177             ((rc = port_full_init(pi)) != 0))
3178                 return (rc); /* error message displayed already */
3179
3180         rc = update_mac_settings(ifp, XGMAC_ALL);
3181         if (rc)
3182                 goto done;      /* error message displayed already */
3183
3184         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
3185         if (rc != 0) {
3186                 if_printf(ifp, "enable_vi failed: %d\n", rc);
3187                 goto done;
3188         }
3189
3190         /*
3191          * The first iq of the first port to come up is used for tracing.
3192          */
3193         if (sc->traceq < 0) {
3194                 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
3195                 t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
3196                     A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
3197                     V_QUEUENUMBER(sc->traceq));
3198                 pi->flags |= HAS_TRACEQ;
3199         }
3200
3201         /* all ok */
3202         setbit(&sc->open_device_map, pi->port_id);
3203         PORT_LOCK(pi);
3204         ifp->if_drv_flags |= IFF_DRV_RUNNING;
3205         PORT_UNLOCK(pi);
3206
3207         callout_reset(&pi->tick, hz, cxgbe_tick, pi);
3208 done:
3209         if (rc != 0)
3210                 cxgbe_uninit_synchronized(pi);
3211
3212         return (rc);
3213 }
3214
3215 /*
3216  * Idempotent.
3217  */
3218 static int
3219 cxgbe_uninit_synchronized(struct port_info *pi)
3220 {
3221         struct adapter *sc = pi->adapter;
3222         struct ifnet *ifp = pi->ifp;
3223         int rc;
3224
3225         ASSERT_SYNCHRONIZED_OP(sc);
3226
3227         /*
3228          * Disable the VI so that all its data in either direction is discarded
3229          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
3230          * tick) intact as the TP can deliver negative advice or data that it's
3231          * holding in its RAM (for an offloaded connection) even after the VI is
3232          * disabled.
3233          */
3234         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
3235         if (rc) {
3236                 if_printf(ifp, "disable_vi failed: %d\n", rc);
3237                 return (rc);
3238         }
3239
3240         clrbit(&sc->open_device_map, pi->port_id);
3241         PORT_LOCK(pi);
3242         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3243         PORT_UNLOCK(pi);
3244
3245         pi->link_cfg.link_ok = 0;
3246         pi->link_cfg.speed = 0;
3247         pi->linkdnrc = -1;
3248         t4_os_link_changed(sc, pi->port_id, 0, -1);
3249
3250         return (0);
3251 }
3252
3253 /*
3254  * It is ok for this function to fail midway and return right away.  t4_detach
3255  * will walk the entire sc->irq list and clean up whatever is valid.
3256  */
3257 static int
3258 setup_intr_handlers(struct adapter *sc)
3259 {
3260         int rc, rid, p, q;
3261         char s[8];
3262         struct irq *irq;
3263         struct port_info *pi;
3264         struct sge_rxq *rxq;
3265 #ifdef TCP_OFFLOAD
3266         struct sge_ofld_rxq *ofld_rxq;
3267 #endif
3268 #ifdef DEV_NETMAP
3269         struct sge_nm_rxq *nm_rxq;
3270 #endif
3271
3272         /*
3273          * Setup interrupts.
3274          */
3275         irq = &sc->irq[0];
3276         rid = sc->intr_type == INTR_INTX ? 0 : 1;
3277         if (sc->intr_count == 1)
3278                 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
3279
3280         /* Multiple interrupts. */
3281         KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
3282             ("%s: too few intr.", __func__));
3283
3284         /* The first one is always error intr */
3285         rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
3286         if (rc != 0)
3287                 return (rc);
3288         irq++;
3289         rid++;
3290
3291         /* The second one is always the firmware event queue */
3292         rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
3293         if (rc != 0)
3294                 return (rc);
3295         irq++;
3296         rid++;
3297
3298         for_each_port(sc, p) {
3299                 pi = sc->port[p];
3300
3301                 if (pi->flags & INTR_RXQ) {
3302                         for_each_rxq(pi, q, rxq) {
3303                                 snprintf(s, sizeof(s), "%d.%d", p, q);
3304                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
3305                                     s);
3306                                 if (rc != 0)
3307                                         return (rc);
3308                                 irq++;
3309                                 rid++;
3310                         }
3311                 }
3312 #ifdef TCP_OFFLOAD
3313                 if (pi->flags & INTR_OFLD_RXQ) {
3314                         for_each_ofld_rxq(pi, q, ofld_rxq) {
3315                                 snprintf(s, sizeof(s), "%d,%d", p, q);
3316                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr,
3317                                     ofld_rxq, s);
3318                                 if (rc != 0)
3319                                         return (rc);
3320                                 irq++;
3321                                 rid++;
3322                         }
3323                 }
3324 #endif
3325 #ifdef DEV_NETMAP
3326                 if (pi->flags & INTR_NM_RXQ) {
3327                         for_each_nm_rxq(pi, q, nm_rxq) {
3328                                 snprintf(s, sizeof(s), "%d-%d", p, q);
3329                                 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr,
3330                                     nm_rxq, s);
3331                                 if (rc != 0)
3332                                         return (rc);
3333                                 irq++;
3334                                 rid++;
3335                         }
3336                 }
3337 #endif
3338         }
3339         MPASS(irq == &sc->irq[sc->intr_count]);
3340
3341         return (0);
3342 }
3343
3344 int
3345 adapter_full_init(struct adapter *sc)
3346 {
3347         int rc, i;
3348
3349         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3350         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
3351             ("%s: FULL_INIT_DONE already", __func__));
3352
3353         /*
3354          * queues that belong to the adapter (not any particular port).
3355          */
3356         rc = t4_setup_adapter_queues(sc);
3357         if (rc != 0)
3358                 goto done;
3359
3360         for (i = 0; i < nitems(sc->tq); i++) {
3361                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
3362                     taskqueue_thread_enqueue, &sc->tq[i]);
3363                 if (sc->tq[i] == NULL) {
3364                         device_printf(sc->dev,
3365                             "failed to allocate task queue %d\n", i);
3366                         rc = ENOMEM;
3367                         goto done;
3368                 }
3369                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
3370                     device_get_nameunit(sc->dev), i);
3371         }
3372
3373         t4_intr_enable(sc);
3374         sc->flags |= FULL_INIT_DONE;
3375 done:
3376         if (rc != 0)
3377                 adapter_full_uninit(sc);
3378
3379         return (rc);
3380 }
3381
3382 int
3383 adapter_full_uninit(struct adapter *sc)
3384 {
3385         int i;
3386
3387         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3388
3389         t4_teardown_adapter_queues(sc);
3390
3391         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
3392                 taskqueue_free(sc->tq[i]);
3393                 sc->tq[i] = NULL;
3394         }
3395
3396         sc->flags &= ~FULL_INIT_DONE;
3397
3398         return (0);
3399 }
3400
3401 int
3402 port_full_init(struct port_info *pi)
3403 {
3404         struct adapter *sc = pi->adapter;
3405         struct ifnet *ifp = pi->ifp;
3406         uint16_t *rss;
3407         struct sge_rxq *rxq;
3408         int rc, i, j;
3409
3410         ASSERT_SYNCHRONIZED_OP(sc);
3411         KASSERT((pi->flags & PORT_INIT_DONE) == 0,
3412             ("%s: PORT_INIT_DONE already", __func__));
3413
3414         sysctl_ctx_init(&pi->ctx);
3415         pi->flags |= PORT_SYSCTL_CTX;
3416
3417         /*
3418          * Allocate tx/rx/fl queues for this port.
3419          */
3420         rc = t4_setup_port_queues(pi);
3421         if (rc != 0)
3422                 goto done;      /* error message displayed already */
3423
3424         /*
3425          * Setup RSS for this port.  Save a copy of the RSS table for later use.
3426          */
3427         rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
3428         for (i = 0; i < pi->rss_size;) {
3429                 for_each_rxq(pi, j, rxq) {
3430                         rss[i++] = rxq->iq.abs_id;
3431                         if (i == pi->rss_size)
3432                                 break;
3433                 }
3434         }
3435
3436         rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
3437             pi->rss_size);
3438         if (rc != 0) {
3439                 if_printf(ifp, "rss_config failed: %d\n", rc);
3440                 goto done;
3441         }
3442
3443         pi->rss = rss;
3444         pi->flags |= PORT_INIT_DONE;
3445 done:
3446         if (rc != 0)
3447                 port_full_uninit(pi);
3448
3449         return (rc);
3450 }
3451
3452 /*
3453  * Idempotent.
3454  */
3455 int
3456 port_full_uninit(struct port_info *pi)
3457 {
3458         struct adapter *sc = pi->adapter;
3459         int i;
3460         struct sge_rxq *rxq;
3461         struct sge_txq *txq;
3462 #ifdef TCP_OFFLOAD
3463         struct sge_ofld_rxq *ofld_rxq;
3464         struct sge_wrq *ofld_txq;
3465 #endif
3466
3467         if (pi->flags & PORT_INIT_DONE) {
3468
3469                 /* Need to quiesce queues.  XXX: ctrl queues? */
3470
3471                 for_each_txq(pi, i, txq) {
3472                         quiesce_eq(sc, &txq->eq);
3473                 }
3474
3475 #ifdef TCP_OFFLOAD
3476                 for_each_ofld_txq(pi, i, ofld_txq) {
3477                         quiesce_eq(sc, &ofld_txq->eq);
3478                 }
3479 #endif
3480
3481                 for_each_rxq(pi, i, rxq) {
3482                         quiesce_iq(sc, &rxq->iq);
3483                         quiesce_fl(sc, &rxq->fl);
3484                 }
3485
3486 #ifdef TCP_OFFLOAD
3487                 for_each_ofld_rxq(pi, i, ofld_rxq) {
3488                         quiesce_iq(sc, &ofld_rxq->iq);
3489                         quiesce_fl(sc, &ofld_rxq->fl);
3490                 }
3491 #endif
3492                 free(pi->rss, M_CXGBE);
3493         }
3494
3495         t4_teardown_port_queues(pi);
3496         pi->flags &= ~PORT_INIT_DONE;
3497
3498         return (0);
3499 }
3500
3501 static void
3502 quiesce_eq(struct adapter *sc, struct sge_eq *eq)
3503 {
3504         EQ_LOCK(eq);
3505         eq->flags |= EQ_DOOMED;
3506
3507         /*
3508          * Wait for the response to a credit flush if one's
3509          * pending.
3510          */
3511         while (eq->flags & EQ_CRFLUSHED)
3512                 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0);
3513         EQ_UNLOCK(eq);
3514
3515         callout_drain(&eq->tx_callout); /* XXX: iffy */
3516         pause("callout", 10);           /* Still iffy */
3517
3518         taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task);
3519 }
3520
3521 static void
3522 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
3523 {
3524         (void) sc;      /* unused */
3525
3526         /* Synchronize with the interrupt handler */
3527         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
3528                 pause("iqfree", 1);
3529 }
3530
3531 static void
3532 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
3533 {
3534         mtx_lock(&sc->sfl_lock);
3535         FL_LOCK(fl);
3536         fl->flags |= FL_DOOMED;
3537         FL_UNLOCK(fl);
3538         mtx_unlock(&sc->sfl_lock);
3539
3540         callout_drain(&sc->sfl_callout);
3541         KASSERT((fl->flags & FL_STARVING) == 0,
3542             ("%s: still starving", __func__));
3543 }
3544
3545 static int
3546 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
3547     driver_intr_t *handler, void *arg, char *name)
3548 {
3549         int rc;
3550
3551         irq->rid = rid;
3552         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
3553             RF_SHAREABLE | RF_ACTIVE);
3554         if (irq->res == NULL) {
3555                 device_printf(sc->dev,
3556                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
3557                 return (ENOMEM);
3558         }
3559
3560         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
3561             NULL, handler, arg, &irq->tag);
3562         if (rc != 0) {
3563                 device_printf(sc->dev,
3564                     "failed to setup interrupt for rid %d, name %s: %d\n",
3565                     rid, name, rc);
3566         } else if (name)
3567                 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
3568
3569         return (rc);
3570 }
3571
3572 static int
3573 t4_free_irq(struct adapter *sc, struct irq *irq)
3574 {
3575         if (irq->tag)
3576                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
3577         if (irq->res)
3578                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
3579
3580         bzero(irq, sizeof(*irq));
3581
3582         return (0);
3583 }
3584
3585 static void
3586 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
3587     unsigned int end)
3588 {
3589         uint32_t *p = (uint32_t *)(buf + start);
3590
3591         for ( ; start <= end; start += sizeof(uint32_t))
3592                 *p++ = t4_read_reg(sc, start);
3593 }
3594
3595 static void
3596 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
3597 {
3598         int i, n;
3599         const unsigned int *reg_ranges;
3600         static const unsigned int t4_reg_ranges[] = {
3601                 0x1008, 0x1108,
3602                 0x1180, 0x11b4,
3603                 0x11fc, 0x123c,
3604                 0x1300, 0x173c,
3605                 0x1800, 0x18fc,
3606                 0x3000, 0x30d8,
3607                 0x30e0, 0x5924,
3608                 0x5960, 0x59d4,
3609                 0x5a00, 0x5af8,
3610                 0x6000, 0x6098,
3611                 0x6100, 0x6150,
3612                 0x6200, 0x6208,
3613                 0x6240, 0x6248,
3614                 0x6280, 0x6338,
3615                 0x6370, 0x638c,
3616                 0x6400, 0x643c,
3617                 0x6500, 0x6524,
3618                 0x6a00, 0x6a38,
3619                 0x6a60, 0x6a78,
3620                 0x6b00, 0x6b84,
3621                 0x6bf0, 0x6c84,
3622                 0x6cf0, 0x6d84,
3623                 0x6df0, 0x6e84,
3624                 0x6ef0, 0x6f84,
3625                 0x6ff0, 0x7084,
3626                 0x70f0, 0x7184,
3627                 0x71f0, 0x7284,
3628                 0x72f0, 0x7384,
3629                 0x73f0, 0x7450,
3630                 0x7500, 0x7530,
3631                 0x7600, 0x761c,
3632                 0x7680, 0x76cc,
3633                 0x7700, 0x7798,
3634                 0x77c0, 0x77fc,
3635                 0x7900, 0x79fc,
3636                 0x7b00, 0x7c38,
3637                 0x7d00, 0x7efc,
3638                 0x8dc0, 0x8e1c,
3639                 0x8e30, 0x8e78,
3640                 0x8ea0, 0x8f6c,
3641                 0x8fc0, 0x9074,
3642                 0x90fc, 0x90fc,
3643                 0x9400, 0x9458,
3644                 0x9600, 0x96bc,
3645                 0x9800, 0x9808,
3646                 0x9820, 0x983c,
3647                 0x9850, 0x9864,
3648                 0x9c00, 0x9c6c,
3649                 0x9c80, 0x9cec,
3650                 0x9d00, 0x9d6c,
3651                 0x9d80, 0x9dec,
3652                 0x9e00, 0x9e6c,
3653                 0x9e80, 0x9eec,
3654                 0x9f00, 0x9f6c,
3655                 0x9f80, 0x9fec,
3656                 0xd004, 0xd03c,
3657                 0xdfc0, 0xdfe0,
3658                 0xe000, 0xea7c,
3659                 0xf000, 0x11110,
3660                 0x11118, 0x11190,
3661                 0x19040, 0x1906c,
3662                 0x19078, 0x19080,
3663                 0x1908c, 0x19124,
3664                 0x19150, 0x191b0,
3665                 0x191d0, 0x191e8,
3666                 0x19238, 0x1924c,
3667                 0x193f8, 0x19474,
3668                 0x19490, 0x194f8,
3669                 0x19800, 0x19f30,
3670                 0x1a000, 0x1a06c,
3671                 0x1a0b0, 0x1a120,
3672                 0x1a128, 0x1a138,
3673                 0x1a190, 0x1a1c4,
3674                 0x1a1fc, 0x1a1fc,
3675                 0x1e040, 0x1e04c,
3676                 0x1e284, 0x1e28c,
3677                 0x1e2c0, 0x1e2c0,
3678                 0x1e2e0, 0x1e2e0,
3679                 0x1e300, 0x1e384,
3680                 0x1e3c0, 0x1e3c8,
3681                 0x1e440, 0x1e44c,
3682                 0x1e684, 0x1e68c,
3683                 0x1e6c0, 0x1e6c0,
3684                 0x1e6e0, 0x1e6e0,
3685                 0x1e700, 0x1e784,
3686                 0x1e7c0, 0x1e7c8,
3687                 0x1e840, 0x1e84c,
3688                 0x1ea84, 0x1ea8c,
3689                 0x1eac0, 0x1eac0,
3690                 0x1eae0, 0x1eae0,
3691                 0x1eb00, 0x1eb84,
3692                 0x1ebc0, 0x1ebc8,
3693                 0x1ec40, 0x1ec4c,
3694                 0x1ee84, 0x1ee8c,
3695                 0x1eec0, 0x1eec0,
3696                 0x1eee0, 0x1eee0,
3697                 0x1ef00, 0x1ef84,
3698                 0x1efc0, 0x1efc8,
3699                 0x1f040, 0x1f04c,
3700                 0x1f284, 0x1f28c,
3701                 0x1f2c0, 0x1f2c0,
3702                 0x1f2e0, 0x1f2e0,
3703                 0x1f300, 0x1f384,
3704                 0x1f3c0, 0x1f3c8,
3705                 0x1f440, 0x1f44c,
3706                 0x1f684, 0x1f68c,
3707                 0x1f6c0, 0x1f6c0,
3708                 0x1f6e0, 0x1f6e0,
3709                 0x1f700, 0x1f784,
3710                 0x1f7c0, 0x1f7c8,
3711                 0x1f840, 0x1f84c,
3712                 0x1fa84, 0x1fa8c,
3713                 0x1fac0, 0x1fac0,
3714                 0x1fae0, 0x1fae0,
3715                 0x1fb00, 0x1fb84,
3716                 0x1fbc0, 0x1fbc8,
3717                 0x1fc40, 0x1fc4c,
3718                 0x1fe84, 0x1fe8c,
3719                 0x1fec0, 0x1fec0,
3720                 0x1fee0, 0x1fee0,
3721                 0x1ff00, 0x1ff84,
3722                 0x1ffc0, 0x1ffc8,
3723                 0x20000, 0x2002c,
3724                 0x20100, 0x2013c,
3725                 0x20190, 0x201c8,
3726                 0x20200, 0x20318,
3727                 0x20400, 0x20528,
3728                 0x20540, 0x20614,
3729                 0x21000, 0x21040,
3730                 0x2104c, 0x21060,
3731                 0x210c0, 0x210ec,
3732                 0x21200, 0x21268,
3733                 0x21270, 0x21284,
3734                 0x212fc, 0x21388,
3735                 0x21400, 0x21404,
3736                 0x21500, 0x21518,
3737                 0x2152c, 0x2153c,
3738                 0x21550, 0x21554,
3739                 0x21600, 0x21600,
3740                 0x21608, 0x21628,
3741                 0x21630, 0x2163c,
3742                 0x21700, 0x2171c,
3743                 0x21780, 0x2178c,
3744                 0x21800, 0x21c38,
3745                 0x21c80, 0x21d7c,
3746                 0x21e00, 0x21e04,
3747                 0x22000, 0x2202c,
3748                 0x22100, 0x2213c,
3749                 0x22190, 0x221c8,
3750                 0x22200, 0x22318,
3751                 0x22400, 0x22528,
3752                 0x22540, 0x22614,
3753                 0x23000, 0x23040,
3754                 0x2304c, 0x23060,
3755                 0x230c0, 0x230ec,
3756                 0x23200, 0x23268,
3757                 0x23270, 0x23284,
3758                 0x232fc, 0x23388,
3759                 0x23400, 0x23404,
3760                 0x23500, 0x23518,
3761                 0x2352c, 0x2353c,
3762                 0x23550, 0x23554,
3763                 0x23600, 0x23600,
3764                 0x23608, 0x23628,
3765                 0x23630, 0x2363c,
3766                 0x23700, 0x2371c,
3767                 0x23780, 0x2378c,
3768                 0x23800, 0x23c38,
3769                 0x23c80, 0x23d7c,
3770                 0x23e00, 0x23e04,
3771                 0x24000, 0x2402c,
3772                 0x24100, 0x2413c,
3773                 0x24190, 0x241c8,
3774                 0x24200, 0x24318,
3775                 0x24400, 0x24528,
3776                 0x24540, 0x24614,
3777                 0x25000, 0x25040,
3778                 0x2504c, 0x25060,
3779                 0x250c0, 0x250ec,
3780                 0x25200, 0x25268,
3781                 0x25270, 0x25284,
3782                 0x252fc, 0x25388,
3783                 0x25400, 0x25404,
3784                 0x25500, 0x25518,
3785                 0x2552c, 0x2553c,
3786                 0x25550, 0x25554,
3787                 0x25600, 0x25600,
3788                 0x25608, 0x25628,
3789                 0x25630, 0x2563c,
3790                 0x25700, 0x2571c,
3791                 0x25780, 0x2578c,
3792                 0x25800, 0x25c38,
3793                 0x25c80, 0x25d7c,
3794                 0x25e00, 0x25e04,
3795                 0x26000, 0x2602c,
3796                 0x26100, 0x2613c,
3797                 0x26190, 0x261c8,
3798                 0x26200, 0x26318,
3799                 0x26400, 0x26528,
3800                 0x26540, 0x26614,
3801                 0x27000, 0x27040,
3802                 0x2704c, 0x27060,
3803                 0x270c0, 0x270ec,
3804                 0x27200, 0x27268,
3805                 0x27270, 0x27284,
3806                 0x272fc, 0x27388,
3807                 0x27400, 0x27404,
3808                 0x27500, 0x27518,
3809                 0x2752c, 0x2753c,
3810                 0x27550, 0x27554,
3811                 0x27600, 0x27600,
3812                 0x27608, 0x27628,
3813                 0x27630, 0x2763c,
3814                 0x27700, 0x2771c,
3815                 0x27780, 0x2778c,
3816                 0x27800, 0x27c38,
3817                 0x27c80, 0x27d7c,
3818                 0x27e00, 0x27e04
3819         };
3820         static const unsigned int t5_reg_ranges[] = {
3821                 0x1008, 0x1148,
3822                 0x1180, 0x11b4,
3823                 0x11fc, 0x123c,
3824                 0x1280, 0x173c,
3825                 0x1800, 0x18fc,
3826                 0x3000, 0x3028,
3827                 0x3060, 0x30d8,
3828                 0x30e0, 0x30fc,
3829                 0x3140, 0x357c,
3830                 0x35a8, 0x35cc,
3831                 0x35ec, 0x35ec,
3832                 0x3600, 0x5624,
3833                 0x56cc, 0x575c,
3834                 0x580c, 0x5814,
3835                 0x5890, 0x58bc,
3836                 0x5940, 0x59dc,
3837                 0x59fc, 0x5a18,
3838                 0x5a60, 0x5a9c,
3839                 0x5b94, 0x5bfc,
3840                 0x6000, 0x6040,
3841                 0x6058, 0x614c,
3842                 0x7700, 0x7798,
3843                 0x77c0, 0x78fc,
3844                 0x7b00, 0x7c54,
3845                 0x7d00, 0x7efc,
3846                 0x8dc0, 0x8de0,
3847                 0x8df8, 0x8e84,
3848                 0x8ea0, 0x8f84,
3849                 0x8fc0, 0x90f8,
3850                 0x9400, 0x9470,
3851                 0x9600, 0x96f4,
3852                 0x9800, 0x9808,
3853                 0x9820, 0x983c,
3854                 0x9850, 0x9864,
3855                 0x9c00, 0x9c6c,
3856                 0x9c80, 0x9cec,
3857                 0x9d00, 0x9d6c,
3858                 0x9d80, 0x9dec,
3859                 0x9e00, 0x9e6c,
3860                 0x9e80, 0x9eec,
3861                 0x9f00, 0x9f6c,
3862                 0x9f80, 0xa020,
3863                 0xd004, 0xd03c,
3864                 0xdfc0, 0xdfe0,
3865                 0xe000, 0x11088,
3866                 0x1109c, 0x11110,
3867                 0x11118, 0x1117c,
3868                 0x11190, 0x11204,
3869                 0x19040, 0x1906c,
3870                 0x19078, 0x19080,
3871                 0x1908c, 0x19124,
3872                 0x19150, 0x191b0,
3873                 0x191d0, 0x191e8,
3874                 0x19238, 0x19290,
3875                 0x193f8, 0x19474,
3876                 0x19490, 0x194cc,
3877                 0x194f0, 0x194f8,
3878                 0x19c00, 0x19c60,
3879                 0x19c94, 0x19e10,
3880                 0x19e50, 0x19f34,
3881                 0x19f40, 0x19f50,
3882                 0x19f90, 0x19fe4,
3883                 0x1a000, 0x1a06c,
3884                 0x1a0b0, 0x1a120,
3885                 0x1a128, 0x1a138,
3886                 0x1a190, 0x1a1c4,
3887                 0x1a1fc, 0x1a1fc,
3888                 0x1e008, 0x1e00c,
3889                 0x1e040, 0x1e04c,
3890                 0x1e284, 0x1e290,
3891                 0x1e2c0, 0x1e2c0,
3892                 0x1e2e0, 0x1e2e0,
3893                 0x1e300, 0x1e384,
3894                 0x1e3c0, 0x1e3c8,
3895                 0x1e408, 0x1e40c,
3896                 0x1e440, 0x1e44c,
3897                 0x1e684, 0x1e690,
3898                 0x1e6c0, 0x1e6c0,
3899                 0x1e6e0, 0x1e6e0,
3900                 0x1e700, 0x1e784,
3901                 0x1e7c0, 0x1e7c8,
3902                 0x1e808, 0x1e80c,
3903                 0x1e840, 0x1e84c,
3904                 0x1ea84, 0x1ea90,
3905                 0x1eac0, 0x1eac0,
3906                 0x1eae0, 0x1eae0,
3907                 0x1eb00, 0x1eb84,
3908                 0x1ebc0, 0x1ebc8,
3909                 0x1ec08, 0x1ec0c,
3910                 0x1ec40, 0x1ec4c,
3911                 0x1ee84, 0x1ee90,
3912                 0x1eec0, 0x1eec0,
3913                 0x1eee0, 0x1eee0,
3914                 0x1ef00, 0x1ef84,
3915                 0x1efc0, 0x1efc8,
3916                 0x1f008, 0x1f00c,
3917                 0x1f040, 0x1f04c,
3918                 0x1f284, 0x1f290,
3919                 0x1f2c0, 0x1f2c0,
3920                 0x1f2e0, 0x1f2e0,
3921                 0x1f300, 0x1f384,
3922                 0x1f3c0, 0x1f3c8,
3923                 0x1f408, 0x1f40c,
3924                 0x1f440, 0x1f44c,
3925                 0x1f684, 0x1f690,
3926                 0x1f6c0, 0x1f6c0,
3927                 0x1f6e0, 0x1f6e0,
3928                 0x1f700, 0x1f784,
3929                 0x1f7c0, 0x1f7c8,
3930                 0x1f808, 0x1f80c,
3931                 0x1f840, 0x1f84c,
3932                 0x1fa84, 0x1fa90,
3933                 0x1fac0, 0x1fac0,
3934                 0x1fae0, 0x1fae0,
3935                 0x1fb00, 0x1fb84,
3936                 0x1fbc0, 0x1fbc8,
3937                 0x1fc08, 0x1fc0c,
3938                 0x1fc40, 0x1fc4c,
3939                 0x1fe84, 0x1fe90,
3940                 0x1fec0, 0x1fec0,
3941                 0x1fee0, 0x1fee0,
3942                 0x1ff00, 0x1ff84,
3943                 0x1ffc0, 0x1ffc8,
3944                 0x30000, 0x30030,
3945                 0x30100, 0x30144,
3946                 0x30190, 0x301d0,
3947                 0x30200, 0x30318,
3948                 0x30400, 0x3052c,
3949                 0x30540, 0x3061c,
3950                 0x30800, 0x30834,
3951                 0x308c0, 0x30908,
3952                 0x30910, 0x309ac,
3953                 0x30a00, 0x30a2c,
3954                 0x30a44, 0x30a50,
3955                 0x30a74, 0x30c24,
3956                 0x30d00, 0x30d00,
3957                 0x30d08, 0x30d14,
3958                 0x30d1c, 0x30d20,
3959                 0x30d3c, 0x30d50,
3960                 0x31200, 0x3120c,
3961                 0x31220, 0x31220,
3962                 0x31240, 0x31240,
3963                 0x31600, 0x3160c,
3964                 0x31a00, 0x31a1c,
3965                 0x31e00, 0x31e20,
3966                 0x31e38, 0x31e3c,
3967                 0x31e80, 0x31e80,
3968                 0x31e88, 0x31ea8,
3969                 0x31eb0, 0x31eb4,
3970                 0x31ec8, 0x31ed4,
3971                 0x31fb8, 0x32004,
3972                 0x32200, 0x32200,
3973                 0x32208, 0x32240,
3974                 0x32248, 0x32280,
3975                 0x32288, 0x322c0,
3976                 0x322c8, 0x322fc,
3977                 0x32600, 0x32630,
3978                 0x32a00, 0x32abc,
3979                 0x32b00, 0x32b70,
3980                 0x33000, 0x33048,
3981                 0x33060, 0x3309c,
3982                 0x330f0, 0x33148,
3983                 0x33160, 0x3319c,
3984                 0x331f0, 0x332e4,
3985                 0x332f8, 0x333e4,
3986                 0x333f8, 0x33448,
3987                 0x33460, 0x3349c,
3988                 0x334f0, 0x33548,
3989                 0x33560, 0x3359c,
3990                 0x335f0, 0x336e4,
3991                 0x336f8, 0x337e4,
3992                 0x337f8, 0x337fc,
3993                 0x33814, 0x33814,
3994                 0x3382c, 0x3382c,
3995                 0x33880, 0x3388c,
3996                 0x338e8, 0x338ec,
3997                 0x33900, 0x33948,
3998                 0x33960, 0x3399c,
3999                 0x339f0, 0x33ae4,
4000                 0x33af8, 0x33b10,
4001                 0x33b28, 0x33b28,
4002                 0x33b3c, 0x33b50,
4003                 0x33bf0, 0x33c10,
4004                 0x33c28, 0x33c28,
4005                 0x33c3c, 0x33c50,
4006                 0x33cf0, 0x33cfc,
4007                 0x34000, 0x34030,
4008                 0x34100, 0x34144,
4009                 0x34190, 0x341d0,
4010                 0x34200, 0x34318,
4011                 0x34400, 0x3452c,
4012                 0x34540, 0x3461c,
4013                 0x34800, 0x34834,
4014                 0x348c0, 0x34908,
4015                 0x34910, 0x349ac,
4016                 0x34a00, 0x34a2c,
4017                 0x34a44, 0x34a50,
4018                 0x34a74, 0x34c24,
4019                 0x34d00, 0x34d00,
4020                 0x34d08, 0x34d14,
4021                 0x34d1c, 0x34d20,
4022                 0x34d3c, 0x34d50,
4023                 0x35200, 0x3520c,
4024                 0x35220, 0x35220,
4025                 0x35240, 0x35240,
4026                 0x35600, 0x3560c,
4027                 0x35a00, 0x35a1c,
4028                 0x35e00, 0x35e20,
4029                 0x35e38, 0x35e3c,
4030                 0x35e80, 0x35e80,
4031                 0x35e88, 0x35ea8,
4032                 0x35eb0, 0x35eb4,
4033                 0x35ec8, 0x35ed4,
4034                 0x35fb8, 0x36004,
4035                 0x36200, 0x36200,
4036                 0x36208, 0x36240,
4037                 0x36248, 0x36280,
4038                 0x36288, 0x362c0,
4039                 0x362c8, 0x362fc,
4040                 0x36600, 0x36630,
4041                 0x36a00, 0x36abc,
4042                 0x36b00, 0x36b70,
4043                 0x37000, 0x37048,
4044                 0x37060, 0x3709c,
4045                 0x370f0, 0x37148,
4046                 0x37160, 0x3719c,
4047                 0x371f0, 0x372e4,
4048                 0x372f8, 0x373e4,
4049                 0x373f8, 0x37448,
4050                 0x37460, 0x3749c,
4051                 0x374f0, 0x37548,
4052                 0x37560, 0x3759c,
4053                 0x375f0, 0x376e4,
4054                 0x376f8, 0x377e4,
4055                 0x377f8, 0x377fc,
4056                 0x37814, 0x37814,
4057                 0x3782c, 0x3782c,
4058                 0x37880, 0x3788c,
4059                 0x378e8, 0x378ec,
4060                 0x37900, 0x37948,
4061                 0x37960, 0x3799c,
4062                 0x379f0, 0x37ae4,
4063                 0x37af8, 0x37b10,
4064                 0x37b28, 0x37b28,
4065                 0x37b3c, 0x37b50,
4066                 0x37bf0, 0x37c10,
4067                 0x37c28, 0x37c28,
4068                 0x37c3c, 0x37c50,
4069                 0x37cf0, 0x37cfc,
4070                 0x38000, 0x38030,
4071                 0x38100, 0x38144,
4072                 0x38190, 0x381d0,
4073                 0x38200, 0x38318,
4074                 0x38400, 0x3852c,
4075                 0x38540, 0x3861c,
4076                 0x38800, 0x38834,
4077                 0x388c0, 0x38908,
4078                 0x38910, 0x389ac,
4079                 0x38a00, 0x38a2c,
4080                 0x38a44, 0x38a50,
4081                 0x38a74, 0x38c24,
4082                 0x38d00, 0x38d00,
4083                 0x38d08, 0x38d14,
4084                 0x38d1c, 0x38d20,
4085                 0x38d3c, 0x38d50,
4086                 0x39200, 0x3920c,
4087                 0x39220, 0x39220,
4088                 0x39240, 0x39240,
4089                 0x39600, 0x3960c,
4090                 0x39a00, 0x39a1c,
4091                 0x39e00, 0x39e20,
4092                 0x39e38, 0x39e3c,
4093                 0x39e80, 0x39e80,
4094                 0x39e88, 0x39ea8,
4095                 0x39eb0, 0x39eb4,
4096                 0x39ec8, 0x39ed4,
4097                 0x39fb8, 0x3a004,
4098                 0x3a200, 0x3a200,
4099                 0x3a208, 0x3a240,
4100                 0x3a248, 0x3a280,
4101                 0x3a288, 0x3a2c0,
4102                 0x3a2c8, 0x3a2fc,
4103                 0x3a600, 0x3a630,
4104                 0x3aa00, 0x3aabc,
4105                 0x3ab00, 0x3ab70,
4106                 0x3b000, 0x3b048,
4107                 0x3b060, 0x3b09c,
4108                 0x3b0f0, 0x3b148,
4109                 0x3b160, 0x3b19c,
4110                 0x3b1f0, 0x3b2e4,
4111                 0x3b2f8, 0x3b3e4,
4112                 0x3b3f8, 0x3b448,
4113                 0x3b460, 0x3b49c,
4114                 0x3b4f0, 0x3b548,
4115                 0x3b560, 0x3b59c,
4116                 0x3b5f0, 0x3b6e4,
4117                 0x3b6f8, 0x3b7e4,
4118                 0x3b7f8, 0x3b7fc,
4119                 0x3b814, 0x3b814,
4120                 0x3b82c, 0x3b82c,
4121                 0x3b880, 0x3b88c,
4122                 0x3b8e8, 0x3b8ec,
4123                 0x3b900, 0x3b948,
4124                 0x3b960, 0x3b99c,
4125                 0x3b9f0, 0x3bae4,
4126                 0x3baf8, 0x3bb10,
4127                 0x3bb28, 0x3bb28,
4128                 0x3bb3c, 0x3bb50,
4129                 0x3bbf0, 0x3bc10,
4130                 0x3bc28, 0x3bc28,
4131                 0x3bc3c, 0x3bc50,
4132                 0x3bcf0, 0x3bcfc,
4133                 0x3c000, 0x3c030,
4134                 0x3c100, 0x3c144,
4135                 0x3c190, 0x3c1d0,
4136                 0x3c200, 0x3c318,
4137                 0x3c400, 0x3c52c,
4138                 0x3c540, 0x3c61c,
4139                 0x3c800, 0x3c834,
4140                 0x3c8c0, 0x3c908,
4141                 0x3c910, 0x3c9ac,
4142                 0x3ca00, 0x3ca2c,
4143                 0x3ca44, 0x3ca50,
4144                 0x3ca74, 0x3cc24,
4145                 0x3cd00, 0x3cd00,
4146                 0x3cd08, 0x3cd14,
4147                 0x3cd1c, 0x3cd20,
4148                 0x3cd3c, 0x3cd50,
4149                 0x3d200, 0x3d20c,
4150                 0x3d220, 0x3d220,
4151                 0x3d240, 0x3d240,
4152                 0x3d600, 0x3d60c,
4153                 0x3da00, 0x3da1c,
4154                 0x3de00, 0x3de20,
4155                 0x3de38, 0x3de3c,
4156                 0x3de80, 0x3de80,
4157                 0x3de88, 0x3dea8,
4158                 0x3deb0, 0x3deb4,
4159                 0x3dec8, 0x3ded4,
4160                 0x3dfb8, 0x3e004,
4161                 0x3e200, 0x3e200,
4162                 0x3e208, 0x3e240,
4163                 0x3e248, 0x3e280,
4164                 0x3e288, 0x3e2c0,
4165                 0x3e2c8, 0x3e2fc,
4166                 0x3e600, 0x3e630,
4167                 0x3ea00, 0x3eabc,
4168                 0x3eb00, 0x3eb70,
4169                 0x3f000, 0x3f048,
4170                 0x3f060, 0x3f09c,
4171                 0x3f0f0, 0x3f148,
4172                 0x3f160, 0x3f19c,
4173                 0x3f1f0, 0x3f2e4,
4174                 0x3f2f8, 0x3f3e4,
4175                 0x3f3f8, 0x3f448,
4176                 0x3f460, 0x3f49c,
4177                 0x3f4f0, 0x3f548,
4178                 0x3f560, 0x3f59c,
4179                 0x3f5f0, 0x3f6e4,
4180                 0x3f6f8, 0x3f7e4,
4181                 0x3f7f8, 0x3f7fc,
4182                 0x3f814, 0x3f814,
4183                 0x3f82c, 0x3f82c,
4184                 0x3f880, 0x3f88c,
4185                 0x3f8e8, 0x3f8ec,
4186                 0x3f900, 0x3f948,
4187                 0x3f960, 0x3f99c,
4188                 0x3f9f0, 0x3fae4,
4189                 0x3faf8, 0x3fb10,
4190                 0x3fb28, 0x3fb28,
4191                 0x3fb3c, 0x3fb50,
4192                 0x3fbf0, 0x3fc10,
4193                 0x3fc28, 0x3fc28,
4194                 0x3fc3c, 0x3fc50,
4195                 0x3fcf0, 0x3fcfc,
4196                 0x40000, 0x4000c,
4197                 0x40040, 0x40068,
4198                 0x4007c, 0x40144,
4199                 0x40180, 0x4018c,
4200                 0x40200, 0x40298,
4201                 0x402ac, 0x4033c,
4202                 0x403f8, 0x403fc,
4203                 0x41304, 0x413c4,
4204                 0x41400, 0x4141c,
4205                 0x41480, 0x414d0,
4206                 0x44000, 0x44078,
4207                 0x440c0, 0x44278,
4208                 0x442c0, 0x44478,
4209                 0x444c0, 0x44678,
4210                 0x446c0, 0x44878,
4211                 0x448c0, 0x449fc,
4212                 0x45000, 0x45068,
4213                 0x45080, 0x45084,
4214                 0x450a0, 0x450b0,
4215                 0x45200, 0x45268,
4216                 0x45280, 0x45284,
4217                 0x452a0, 0x452b0,
4218                 0x460c0, 0x460e4,
4219                 0x47000, 0x4708c,
4220                 0x47200, 0x47250,
4221                 0x47400, 0x47420,
4222                 0x47600, 0x47618,
4223                 0x47800, 0x47814,
4224                 0x48000, 0x4800c,
4225                 0x48040, 0x48068,
4226                 0x4807c, 0x48144,
4227                 0x48180, 0x4818c,
4228                 0x48200, 0x48298,
4229                 0x482ac, 0x4833c,
4230                 0x483f8, 0x483fc,
4231                 0x49304, 0x493c4,
4232                 0x49400, 0x4941c,
4233                 0x49480, 0x494d0,
4234                 0x4c000, 0x4c078,
4235                 0x4c0c0, 0x4c278,
4236                 0x4c2c0, 0x4c478,
4237                 0x4c4c0, 0x4c678,
4238                 0x4c6c0, 0x4c878,
4239                 0x4c8c0, 0x4c9fc,
4240                 0x4d000, 0x4d068,
4241                 0x4d080, 0x4d084,
4242                 0x4d0a0, 0x4d0b0,
4243                 0x4d200, 0x4d268,
4244                 0x4d280, 0x4d284,
4245                 0x4d2a0, 0x4d2b0,
4246                 0x4e0c0, 0x4e0e4,
4247                 0x4f000, 0x4f08c,
4248                 0x4f200, 0x4f250,
4249                 0x4f400, 0x4f420,
4250                 0x4f600, 0x4f618,
4251                 0x4f800, 0x4f814,
4252                 0x50000, 0x500cc,
4253                 0x50400, 0x50400,
4254                 0x50800, 0x508cc,
4255                 0x50c00, 0x50c00,
4256                 0x51000, 0x5101c,
4257                 0x51300, 0x51308,
4258         };
4259
4260         if (is_t4(sc)) {
4261                 reg_ranges = &t4_reg_ranges[0];
4262                 n = nitems(t4_reg_ranges);
4263         } else {
4264                 reg_ranges = &t5_reg_ranges[0];
4265                 n = nitems(t5_reg_ranges);
4266         }
4267
4268         regs->version = chip_id(sc) | chip_rev(sc) << 10;
4269         for (i = 0; i < n; i += 2)
4270                 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
4271 }
4272
4273 static void
4274 cxgbe_tick(void *arg)
4275 {
4276         struct port_info *pi = arg;
4277         struct adapter *sc = pi->adapter;
4278         struct ifnet *ifp = pi->ifp;
4279         struct sge_txq *txq;
4280         int i, drops;
4281         struct port_stats *s = &pi->stats;
4282
4283         PORT_LOCK(pi);
4284         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4285                 PORT_UNLOCK(pi);
4286                 return; /* without scheduling another callout */
4287         }
4288
4289         t4_get_port_stats(sc, pi->tx_chan, s);
4290
4291         ifp->if_opackets = s->tx_frames - s->tx_pause;
4292         ifp->if_ipackets = s->rx_frames - s->rx_pause;
4293         ifp->if_obytes = s->tx_octets - s->tx_pause * 64;
4294         ifp->if_ibytes = s->rx_octets - s->rx_pause * 64;
4295         ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause;
4296         ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause;
4297         ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
4298             s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
4299             s->rx_trunc3;
4300         for (i = 0; i < 4; i++) {
4301                 if (pi->rx_chan_map & (1 << i)) {
4302                         uint32_t v;
4303
4304                         /*
4305                          * XXX: indirect reads from the same ADDR/DATA pair can
4306                          * race with each other.
4307                          */
4308                         t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
4309                             1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4310                         ifp->if_iqdrops += v;
4311                 }
4312         }
4313
4314         drops = s->tx_drop;
4315         for_each_txq(pi, i, txq)
4316                 drops += txq->br->br_drops;
4317         ifp->if_oqdrops = drops;
4318
4319         ifp->if_oerrors = s->tx_error_frames;
4320         ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long +
4321             s->rx_fcs_err + s->rx_len_err;
4322
4323         callout_schedule(&pi->tick, hz);
4324         PORT_UNLOCK(pi);
4325 }
4326
4327 static void
4328 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4329 {
4330         struct ifnet *vlan;
4331
4332         if (arg != ifp || ifp->if_type != IFT_ETHER)
4333                 return;
4334
4335         vlan = VLAN_DEVAT(ifp, vid);
4336         VLAN_SETCOOKIE(vlan, ifp);
4337 }
4338
4339 static int
4340 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4341 {
4342
4343 #ifdef INVARIANTS
4344         panic("%s: opcode 0x%02x on iq %p with payload %p",
4345             __func__, rss->opcode, iq, m);
4346 #else
4347         log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
4348             __func__, rss->opcode, iq, m);
4349         m_freem(m);
4350 #endif
4351         return (EDOOFUS);
4352 }
4353
4354 int
4355 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
4356 {
4357         uintptr_t *loc, new;
4358
4359         if (opcode >= nitems(sc->cpl_handler))
4360                 return (EINVAL);
4361
4362         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
4363         loc = (uintptr_t *) &sc->cpl_handler[opcode];
4364         atomic_store_rel_ptr(loc, new);
4365
4366         return (0);
4367 }
4368
4369 static int
4370 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
4371 {
4372
4373 #ifdef INVARIANTS
4374         panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
4375 #else
4376         log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
4377             __func__, iq, ctrl);
4378 #endif
4379         return (EDOOFUS);
4380 }
4381
4382 int
4383 t4_register_an_handler(struct adapter *sc, an_handler_t h)
4384 {
4385         uintptr_t *loc, new;
4386
4387         new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
4388         loc = (uintptr_t *) &sc->an_handler;
4389         atomic_store_rel_ptr(loc, new);
4390
4391         return (0);
4392 }
4393
4394 static int
4395 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
4396 {
4397         const struct cpl_fw6_msg *cpl =
4398             __containerof(rpl, struct cpl_fw6_msg, data[0]);
4399
4400 #ifdef INVARIANTS
4401         panic("%s: fw_msg type %d", __func__, cpl->type);
4402 #else
4403         log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
4404 #endif
4405         return (EDOOFUS);
4406 }
4407
4408 int
4409 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
4410 {
4411         uintptr_t *loc, new;
4412
4413         if (type >= nitems(sc->fw_msg_handler))
4414                 return (EINVAL);
4415
4416         /*
4417          * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
4418          * handler dispatch table.  Reject any attempt to install a handler for
4419          * this subtype.
4420          */
4421         if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
4422                 return (EINVAL);
4423
4424         new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
4425         loc = (uintptr_t *) &sc->fw_msg_handler[type];
4426         atomic_store_rel_ptr(loc, new);
4427
4428         return (0);
4429 }
4430
4431 static int
4432 t4_sysctls(struct adapter *sc)
4433 {
4434         struct sysctl_ctx_list *ctx;
4435         struct sysctl_oid *oid;
4436         struct sysctl_oid_list *children, *c0;
4437         static char *caps[] = {
4438                 "\20\1PPP\2QFC\3DCBX",                  /* caps[0] linkcaps */
4439                 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL"        /* caps[1] niccaps */
4440                     "\6HASHFILTER\7ETHOFLD",
4441                 "\20\1TOE",                             /* caps[2] toecaps */
4442                 "\20\1RDDP\2RDMAC",                     /* caps[3] rdmacaps */
4443                 "\20\1INITIATOR_PDU\2TARGET_PDU"        /* caps[4] iscsicaps */
4444                     "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
4445                     "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
4446                 "\20\1INITIATOR\2TARGET\3CTRL_OFLD"     /* caps[5] fcoecaps */
4447                     "\4PO_INITIAOR\5PO_TARGET"
4448         };
4449         static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
4450
4451         ctx = device_get_sysctl_ctx(sc->dev);
4452
4453         /*
4454          * dev.t4nex.X.
4455          */
4456         oid = device_get_sysctl_tree(sc->dev);
4457         c0 = children = SYSCTL_CHILDREN(oid);
4458
4459         sc->sc_do_rxcopy = 1;
4460         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
4461             &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
4462
4463         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
4464             sc->params.nports, "# of ports");
4465
4466         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
4467             NULL, chip_rev(sc), "chip hardware revision");
4468
4469         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
4470             CTLFLAG_RD, &sc->fw_version, 0, "firmware version");
4471
4472         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
4473             CTLFLAG_RD, &sc->cfg_file, 0, "configuration file");
4474
4475         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
4476             sc->cfcsum, "config file checksum");
4477
4478         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
4479             CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
4480             sysctl_bitfield, "A", "available doorbells");
4481
4482         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
4483             CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
4484             sysctl_bitfield, "A", "available link capabilities");
4485
4486         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
4487             CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
4488             sysctl_bitfield, "A", "available NIC capabilities");
4489
4490         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
4491             CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
4492             sysctl_bitfield, "A", "available TCP offload capabilities");
4493
4494         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
4495             CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
4496             sysctl_bitfield, "A", "available RDMA capabilities");
4497
4498         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
4499             CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
4500             sysctl_bitfield, "A", "available iSCSI capabilities");
4501
4502         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
4503             CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
4504             sysctl_bitfield, "A", "available FCoE capabilities");
4505
4506         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
4507             sc->params.vpd.cclk, "core clock frequency (in KHz)");
4508
4509         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
4510             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
4511             sizeof(sc->sge.timer_val), sysctl_int_array, "A",
4512             "interrupt holdoff timer values (us)");
4513
4514         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
4515             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
4516             sizeof(sc->sge.counter_val), sysctl_int_array, "A",
4517             "interrupt holdoff packet counter values");
4518
4519         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
4520             NULL, sc->tids.nftids, "number of filters");
4521
4522         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
4523             CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
4524             "chip temperature (in Celsius)");
4525
4526         t4_sge_sysctls(sc, ctx, children);
4527
4528         sc->lro_timeout = 100;
4529         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
4530             &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
4531
4532 #ifdef SBUF_DRAIN
4533         /*
4534          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
4535          */
4536         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
4537             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
4538             "logs and miscellaneous information");
4539         children = SYSCTL_CHILDREN(oid);
4540
4541         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
4542             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4543             sysctl_cctrl, "A", "congestion control");
4544
4545         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
4546             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4547             sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
4548
4549         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
4550             CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
4551             sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
4552
4553         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
4554             CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
4555             sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
4556
4557         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
4558             CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
4559             sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
4560
4561         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
4562             CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
4563             sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
4564
4565         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
4566             CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
4567             sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
4568
4569         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
4570             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4571             sysctl_cim_la, "A", "CIM logic analyzer");
4572
4573         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
4574             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4575             sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
4576
4577         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
4578             CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
4579             sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
4580
4581         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
4582             CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
4583             sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
4584
4585         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
4586             CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
4587             sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
4588
4589         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
4590             CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
4591             sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
4592
4593         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
4594             CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
4595             sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
4596
4597         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
4598             CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
4599             sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
4600
4601         if (is_t5(sc)) {
4602                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
4603                     CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
4604                     sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
4605
4606                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
4607                     CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
4608                     sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
4609         }
4610
4611         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
4612             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4613             sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
4614
4615         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
4616             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4617             sysctl_cim_qcfg, "A", "CIM queue configuration");
4618
4619         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
4620             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4621             sysctl_cpl_stats, "A", "CPL statistics");
4622
4623         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
4624             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4625             sysctl_ddp_stats, "A", "DDP statistics");
4626
4627         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
4628             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4629             sysctl_devlog, "A", "firmware's device log");
4630
4631         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
4632             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4633             sysctl_fcoe_stats, "A", "FCoE statistics");
4634
4635         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
4636             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4637             sysctl_hw_sched, "A", "hardware scheduler ");
4638
4639         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
4640             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4641             sysctl_l2t, "A", "hardware L2 table");
4642
4643         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
4644             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4645             sysctl_lb_stats, "A", "loopback statistics");
4646
4647         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
4648             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4649             sysctl_meminfo, "A", "memory regions");
4650
4651         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
4652             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4653             sysctl_mps_tcam, "A", "MPS TCAM entries");
4654
4655         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
4656             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4657             sysctl_path_mtus, "A", "path MTUs");
4658
4659         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
4660             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4661             sysctl_pm_stats, "A", "PM statistics");
4662
4663         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
4664             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4665             sysctl_rdma_stats, "A", "RDMA statistics");
4666
4667         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
4668             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4669             sysctl_tcp_stats, "A", "TCP statistics");
4670
4671         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
4672             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4673             sysctl_tids, "A", "TID information");
4674
4675         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
4676             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4677             sysctl_tp_err_stats, "A", "TP error statistics");
4678
4679         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
4680             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4681             sysctl_tp_la, "A", "TP logic analyzer");
4682
4683         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
4684             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4685             sysctl_tx_rate, "A", "Tx rate");
4686
4687         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
4688             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4689             sysctl_ulprx_la, "A", "ULPRX logic analyzer");
4690
4691         if (is_t5(sc)) {
4692                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
4693                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4694                     sysctl_wcwr_stats, "A", "write combined work requests");
4695         }
4696 #endif
4697
4698 #ifdef TCP_OFFLOAD
4699         if (is_offload(sc)) {
4700                 /*
4701                  * dev.t4nex.X.toe.
4702                  */
4703                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
4704                     NULL, "TOE parameters");
4705                 children = SYSCTL_CHILDREN(oid);
4706
4707                 sc->tt.sndbuf = 256 * 1024;
4708                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
4709                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
4710
4711                 sc->tt.ddp = 0;
4712                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
4713                     &sc->tt.ddp, 0, "DDP allowed");
4714
4715                 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
4716                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
4717                     &sc->tt.indsz, 0, "DDP max indicate size allowed");
4718
4719                 sc->tt.ddp_thres =
4720                     G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
4721                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
4722                     &sc->tt.ddp_thres, 0, "DDP threshold");
4723
4724                 sc->tt.rx_coalesce = 1;
4725                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
4726                     CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
4727         }
4728 #endif
4729
4730
4731         return (0);
4732 }
4733
4734 static int
4735 cxgbe_sysctls(struct port_info *pi)
4736 {
4737         struct sysctl_ctx_list *ctx;
4738         struct sysctl_oid *oid;
4739         struct sysctl_oid_list *children;
4740         struct adapter *sc = pi->adapter;
4741
4742         ctx = device_get_sysctl_ctx(pi->dev);
4743
4744         /*
4745          * dev.cxgbe.X.
4746          */
4747         oid = device_get_sysctl_tree(pi->dev);
4748         children = SYSCTL_CHILDREN(oid);
4749
4750         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
4751            CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
4752         if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
4753                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
4754                     CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
4755                     "PHY temperature (in Celsius)");
4756                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
4757                     CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
4758                     "PHY firmware version");
4759         }
4760         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
4761             &pi->nrxq, 0, "# of rx queues");
4762         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
4763             &pi->ntxq, 0, "# of tx queues");
4764         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
4765             &pi->first_rxq, 0, "index of first rx queue");
4766         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
4767             &pi->first_txq, 0, "index of first tx queue");
4768         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
4769             CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU",
4770             "Reserve queue 0 for non-flowid packets");
4771
4772 #ifdef TCP_OFFLOAD
4773         if (is_offload(sc)) {
4774                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
4775                     &pi->nofldrxq, 0,
4776                     "# of rx queues for offloaded TCP connections");
4777                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
4778                     &pi->nofldtxq, 0,
4779                     "# of tx queues for offloaded TCP connections");
4780                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
4781                     CTLFLAG_RD, &pi->first_ofld_rxq, 0,
4782                     "index of first TOE rx queue");
4783                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
4784                     CTLFLAG_RD, &pi->first_ofld_txq, 0,
4785                     "index of first TOE tx queue");
4786         }
4787 #endif
4788 #ifdef DEV_NETMAP
4789         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
4790             &pi->nnmrxq, 0, "# of rx queues for netmap");
4791         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
4792             &pi->nnmtxq, 0, "# of tx queues for netmap");
4793         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
4794             CTLFLAG_RD, &pi->first_nm_rxq, 0,
4795             "index of first netmap rx queue");
4796         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
4797             CTLFLAG_RD, &pi->first_nm_txq, 0,
4798             "index of first netmap tx queue");
4799 #endif
4800
4801         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
4802             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
4803             "holdoff timer index");
4804         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
4805             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
4806             "holdoff packet counter index");
4807
4808         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
4809             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
4810             "rx queue size");
4811         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
4812             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
4813             "tx queue size");
4814
4815         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
4816             CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings,
4817             "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
4818
4819         /*
4820          * dev.cxgbe.X.stats.
4821          */
4822         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
4823             NULL, "port statistics");
4824         children = SYSCTL_CHILDREN(oid);
4825
4826 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
4827         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
4828             CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
4829             sysctl_handle_t4_reg64, "QU", desc)
4830
4831         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
4832             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
4833         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
4834             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
4835         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
4836             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
4837         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
4838             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
4839         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
4840             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
4841         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
4842             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
4843         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
4844             "# of tx frames in this range",
4845             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
4846         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
4847             "# of tx frames in this range",
4848             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
4849         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
4850             "# of tx frames in this range",
4851             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
4852         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
4853             "# of tx frames in this range",
4854             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
4855         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
4856             "# of tx frames in this range",
4857             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
4858         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
4859             "# of tx frames in this range",
4860             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
4861         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
4862             "# of tx frames in this range",
4863             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
4864         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
4865             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
4866         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
4867             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
4868         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
4869             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
4870         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
4871             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
4872         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
4873             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
4874         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
4875             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
4876         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
4877             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
4878         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
4879             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
4880         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
4881             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
4882         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
4883             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
4884
4885         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
4886             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
4887         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
4888             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
4889         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
4890             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
4891         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
4892             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
4893         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
4894             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
4895         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
4896             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
4897         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
4898             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
4899         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
4900             "# of frames received with bad FCS",
4901             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
4902         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
4903             "# of frames received with length error",
4904             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
4905         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
4906             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
4907         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
4908             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
4909         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
4910             "# of rx frames in this range",
4911             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
4912         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
4913             "# of rx frames in this range",
4914             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
4915         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
4916             "# of rx frames in this range",
4917             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
4918         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
4919             "# of rx frames in this range",
4920             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
4921         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
4922             "# of rx frames in this range",
4923             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
4924         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
4925             "# of rx frames in this range",
4926             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
4927         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
4928             "# of rx frames in this range",
4929             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
4930         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
4931             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
4932         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
4933             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
4934         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
4935             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
4936         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
4937             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
4938         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
4939             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
4940         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
4941             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
4942         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
4943             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
4944         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
4945             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
4946         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
4947             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
4948
4949 #undef SYSCTL_ADD_T4_REG64
4950
4951 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
4952         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
4953             &pi->stats.name, desc)
4954
4955         /* We get these from port_stats and they may be stale by upto 1s */
4956         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
4957             "# drops due to buffer-group 0 overflows");
4958         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
4959             "# drops due to buffer-group 1 overflows");
4960         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
4961             "# drops due to buffer-group 2 overflows");
4962         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
4963             "# drops due to buffer-group 3 overflows");
4964         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
4965             "# of buffer-group 0 truncated packets");
4966         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
4967             "# of buffer-group 1 truncated packets");
4968         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
4969             "# of buffer-group 2 truncated packets");
4970         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
4971             "# of buffer-group 3 truncated packets");
4972
4973 #undef SYSCTL_ADD_T4_PORTSTAT
4974
4975         return (0);
4976 }
4977
4978 static int
4979 sysctl_int_array(SYSCTL_HANDLER_ARGS)
4980 {
4981         int rc, *i;
4982         struct sbuf sb;
4983
4984         sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
4985         for (i = arg1; arg2; arg2 -= sizeof(int), i++)
4986                 sbuf_printf(&sb, "%d ", *i);
4987         sbuf_trim(&sb);
4988         sbuf_finish(&sb);
4989         rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
4990         sbuf_delete(&sb);
4991         return (rc);
4992 }
4993
4994 static int
4995 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
4996 {
4997         int rc;
4998         struct sbuf *sb;
4999
5000         rc = sysctl_wire_old_buffer(req, 0);
5001         if (rc != 0)
5002                 return(rc);
5003
5004         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5005         if (sb == NULL)
5006                 return (ENOMEM);
5007
5008         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
5009         rc = sbuf_finish(sb);
5010         sbuf_delete(sb);
5011
5012         return (rc);
5013 }
5014
5015 static int
5016 sysctl_btphy(SYSCTL_HANDLER_ARGS)
5017 {
5018         struct port_info *pi = arg1;
5019         int op = arg2;
5020         struct adapter *sc = pi->adapter;
5021         u_int v;
5022         int rc;
5023
5024         rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt");
5025         if (rc)
5026                 return (rc);
5027         /* XXX: magic numbers */
5028         rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
5029             &v);
5030         end_synchronized_op(sc, 0);
5031         if (rc)
5032                 return (rc);
5033         if (op == 0)
5034                 v /= 256;
5035
5036         rc = sysctl_handle_int(oidp, &v, 0, req);
5037         return (rc);
5038 }
5039
5040 static int
5041 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
5042 {
5043         struct port_info *pi = arg1;
5044         int rc, val;
5045
5046         val = pi->rsrv_noflowq;
5047         rc = sysctl_handle_int(oidp, &val, 0, req);
5048         if (rc != 0 || req->newptr == NULL)
5049                 return (rc);
5050
5051         if ((val >= 1) && (pi->ntxq > 1))
5052                 pi->rsrv_noflowq = 1;
5053         else
5054                 pi->rsrv_noflowq = 0;
5055
5056         return (rc);
5057 }
5058
5059 static int
5060 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
5061 {
5062         struct port_info *pi = arg1;
5063         struct adapter *sc = pi->adapter;
5064         int idx, rc, i;
5065         struct sge_rxq *rxq;
5066 #ifdef TCP_OFFLOAD
5067         struct sge_ofld_rxq *ofld_rxq;
5068 #endif
5069         uint8_t v;
5070
5071         idx = pi->tmr_idx;
5072
5073         rc = sysctl_handle_int(oidp, &idx, 0, req);
5074         if (rc != 0 || req->newptr == NULL)
5075                 return (rc);
5076
5077         if (idx < 0 || idx >= SGE_NTIMERS)
5078                 return (EINVAL);
5079
5080         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5081             "t4tmr");
5082         if (rc)
5083                 return (rc);
5084
5085         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
5086         for_each_rxq(pi, i, rxq) {
5087 #ifdef atomic_store_rel_8
5088                 atomic_store_rel_8(&rxq->iq.intr_params, v);
5089 #else
5090                 rxq->iq.intr_params = v;
5091 #endif
5092         }
5093 #ifdef TCP_OFFLOAD
5094         for_each_ofld_rxq(pi, i, ofld_rxq) {
5095 #ifdef atomic_store_rel_8
5096                 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
5097 #else
5098                 ofld_rxq->iq.intr_params = v;
5099 #endif
5100         }
5101 #endif
5102         pi->tmr_idx = idx;
5103
5104         end_synchronized_op(sc, LOCK_HELD);
5105         return (0);
5106 }
5107
5108 static int
5109 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
5110 {
5111         struct port_info *pi = arg1;
5112         struct adapter *sc = pi->adapter;
5113         int idx, rc;
5114
5115         idx = pi->pktc_idx;
5116
5117         rc = sysctl_handle_int(oidp, &idx, 0, req);
5118         if (rc != 0 || req->newptr == NULL)
5119                 return (rc);
5120
5121         if (idx < -1 || idx >= SGE_NCOUNTERS)
5122                 return (EINVAL);
5123
5124         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5125             "t4pktc");
5126         if (rc)
5127                 return (rc);
5128
5129         if (pi->flags & PORT_INIT_DONE)
5130                 rc = EBUSY; /* cannot be changed once the queues are created */
5131         else
5132                 pi->pktc_idx = idx;
5133
5134         end_synchronized_op(sc, LOCK_HELD);
5135         return (rc);
5136 }
5137
5138 static int
5139 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
5140 {
5141         struct port_info *pi = arg1;
5142         struct adapter *sc = pi->adapter;
5143         int qsize, rc;
5144
5145         qsize = pi->qsize_rxq;
5146
5147         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5148         if (rc != 0 || req->newptr == NULL)
5149                 return (rc);
5150
5151         if (qsize < 128 || (qsize & 7))
5152                 return (EINVAL);
5153
5154         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5155             "t4rxqs");
5156         if (rc)
5157                 return (rc);
5158
5159         if (pi->flags & PORT_INIT_DONE)
5160                 rc = EBUSY; /* cannot be changed once the queues are created */
5161         else
5162                 pi->qsize_rxq = qsize;
5163
5164         end_synchronized_op(sc, LOCK_HELD);
5165         return (rc);
5166 }
5167
5168 static int
5169 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
5170 {
5171         struct port_info *pi = arg1;
5172         struct adapter *sc = pi->adapter;
5173         int qsize, rc;
5174
5175         qsize = pi->qsize_txq;
5176
5177         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5178         if (rc != 0 || req->newptr == NULL)
5179                 return (rc);
5180
5181         /* bufring size must be powerof2 */
5182         if (qsize < 128 || !powerof2(qsize))
5183                 return (EINVAL);
5184
5185         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5186             "t4txqs");
5187         if (rc)
5188                 return (rc);
5189
5190         if (pi->flags & PORT_INIT_DONE)
5191                 rc = EBUSY; /* cannot be changed once the queues are created */
5192         else
5193                 pi->qsize_txq = qsize;
5194
5195         end_synchronized_op(sc, LOCK_HELD);
5196         return (rc);
5197 }
5198
5199 static int
5200 sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
5201 {
5202         struct port_info *pi = arg1;
5203         struct adapter *sc = pi->adapter;
5204         struct link_config *lc = &pi->link_cfg;
5205         int rc;
5206
5207         if (req->newptr == NULL) {
5208                 struct sbuf *sb;
5209                 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX";
5210
5211                 rc = sysctl_wire_old_buffer(req, 0);
5212                 if (rc != 0)
5213                         return(rc);
5214
5215                 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5216                 if (sb == NULL)
5217                         return (ENOMEM);
5218
5219                 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits);
5220                 rc = sbuf_finish(sb);
5221                 sbuf_delete(sb);
5222         } else {
5223                 char s[2];
5224                 int n;
5225
5226                 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX));
5227                 s[1] = 0;
5228
5229                 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
5230                 if (rc != 0)
5231                         return(rc);
5232
5233                 if (s[1] != 0)
5234                         return (EINVAL);
5235                 if (s[0] < '0' || s[0] > '9')
5236                         return (EINVAL);        /* not a number */
5237                 n = s[0] - '0';
5238                 if (n & ~(PAUSE_TX | PAUSE_RX))
5239                         return (EINVAL);        /* some other bit is set too */
5240
5241                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE");
5242                 if (rc)
5243                         return (rc);
5244                 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) {
5245                         int link_ok = lc->link_ok;
5246
5247                         lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX);
5248                         lc->requested_fc |= n;
5249                         rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc);
5250                         lc->link_ok = link_ok;  /* restore */
5251                 }
5252                 end_synchronized_op(sc, 0);
5253         }
5254
5255         return (rc);
5256 }
5257
5258 static int
5259 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
5260 {
5261         struct adapter *sc = arg1;
5262         int reg = arg2;
5263         uint64_t val;
5264
5265         val = t4_read_reg64(sc, reg);
5266
5267         return (sysctl_handle_64(oidp, &val, 0, req));
5268 }
5269
5270 static int
5271 sysctl_temperature(SYSCTL_HANDLER_ARGS)
5272 {
5273         struct adapter *sc = arg1;
5274         int rc, t;
5275         uint32_t param, val;
5276
5277         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
5278         if (rc)
5279                 return (rc);
5280         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5281             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5282             V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
5283         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
5284         end_synchronized_op(sc, 0);
5285         if (rc)
5286                 return (rc);
5287
5288         /* unknown is returned as 0 but we display -1 in that case */
5289         t = val == 0 ? -1 : val;
5290
5291         rc = sysctl_handle_int(oidp, &t, 0, req);
5292         return (rc);
5293 }
5294
5295 #ifdef SBUF_DRAIN
5296 static int
5297 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
5298 {
5299         struct adapter *sc = arg1;
5300         struct sbuf *sb;
5301         int rc, i;
5302         uint16_t incr[NMTUS][NCCTRL_WIN];
5303         static const char *dec_fac[] = {
5304                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
5305                 "0.9375"
5306         };
5307
5308         rc = sysctl_wire_old_buffer(req, 0);
5309         if (rc != 0)
5310                 return (rc);
5311
5312         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5313         if (sb == NULL)
5314                 return (ENOMEM);
5315
5316         t4_read_cong_tbl(sc, incr);
5317
5318         for (i = 0; i < NCCTRL_WIN; ++i) {
5319                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
5320                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
5321                     incr[5][i], incr[6][i], incr[7][i]);
5322                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
5323                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
5324                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
5325                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
5326         }
5327
5328         rc = sbuf_finish(sb);
5329         sbuf_delete(sb);
5330
5331         return (rc);
5332 }
5333
5334 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
5335         "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",   /* ibq's */
5336         "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
5337         "SGE0-RX", "SGE1-RX"    /* additional obq's (T5 onwards) */
5338 };
5339
5340 static int
5341 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
5342 {
5343         struct adapter *sc = arg1;
5344         struct sbuf *sb;
5345         int rc, i, n, qid = arg2;
5346         uint32_t *buf, *p;
5347         char *qtype;
5348         u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
5349
5350         KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
5351             ("%s: bad qid %d\n", __func__, qid));
5352
5353         if (qid < CIM_NUM_IBQ) {
5354                 /* inbound queue */
5355                 qtype = "IBQ";
5356                 n = 4 * CIM_IBQ_SIZE;
5357                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5358                 rc = t4_read_cim_ibq(sc, qid, buf, n);
5359         } else {
5360                 /* outbound queue */
5361                 qtype = "OBQ";
5362                 qid -= CIM_NUM_IBQ;
5363                 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
5364                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5365                 rc = t4_read_cim_obq(sc, qid, buf, n);
5366         }
5367
5368         if (rc < 0) {
5369                 rc = -rc;
5370                 goto done;
5371         }
5372         n = rc * sizeof(uint32_t);      /* rc has # of words actually read */
5373
5374         rc = sysctl_wire_old_buffer(req, 0);
5375         if (rc != 0)
5376                 goto done;
5377
5378         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5379         if (sb == NULL) {
5380                 rc = ENOMEM;
5381                 goto done;
5382         }
5383
5384         sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
5385         for (i = 0, p = buf; i < n; i += 16, p += 4)
5386                 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
5387                     p[2], p[3]);
5388
5389         rc = sbuf_finish(sb);
5390         sbuf_delete(sb);
5391 done:
5392         free(buf, M_CXGBE);
5393         return (rc);
5394 }
5395
5396 static int
5397 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
5398 {
5399         struct adapter *sc = arg1;
5400         u_int cfg;
5401         struct sbuf *sb;
5402         uint32_t *buf, *p;
5403         int rc;
5404
5405         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
5406         if (rc != 0)
5407                 return (rc);
5408
5409         rc = sysctl_wire_old_buffer(req, 0);
5410         if (rc != 0)
5411                 return (rc);
5412
5413         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5414         if (sb == NULL)
5415                 return (ENOMEM);
5416
5417         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
5418             M_ZERO | M_WAITOK);
5419
5420         rc = -t4_cim_read_la(sc, buf, NULL);
5421         if (rc != 0)
5422                 goto done;
5423
5424         sbuf_printf(sb, "Status   Data      PC%s",
5425             cfg & F_UPDBGLACAPTPCONLY ? "" :
5426             "     LS0Stat  LS0Addr             LS0Data");
5427
5428         KASSERT((sc->params.cim_la_size & 7) == 0,
5429             ("%s: p will walk off the end of buf", __func__));
5430
5431         for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
5432                 if (cfg & F_UPDBGLACAPTPCONLY) {
5433                         sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
5434                             p[6], p[7]);
5435                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
5436                             (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
5437                             p[4] & 0xff, p[5] >> 8);
5438                         sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
5439                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5440                             p[1] & 0xf, p[2] >> 4);
5441                 } else {
5442                         sbuf_printf(sb,
5443                             "\n  %02x   %x%07x %x%07x %08x %08x "
5444                             "%08x%08x%08x%08x",
5445                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5446                             p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
5447                             p[6], p[7]);
5448                 }
5449         }
5450
5451         rc = sbuf_finish(sb);
5452         sbuf_delete(sb);
5453 done:
5454         free(buf, M_CXGBE);
5455         return (rc);
5456 }
5457
5458 static int
5459 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
5460 {
5461         struct adapter *sc = arg1;
5462         u_int i;
5463         struct sbuf *sb;
5464         uint32_t *buf, *p;
5465         int rc;
5466
5467         rc = sysctl_wire_old_buffer(req, 0);
5468         if (rc != 0)
5469                 return (rc);
5470
5471         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5472         if (sb == NULL)
5473                 return (ENOMEM);
5474
5475         buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
5476             M_ZERO | M_WAITOK);
5477
5478         t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
5479         p = buf;
5480
5481         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5482                 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
5483                     p[1], p[0]);
5484         }
5485
5486         sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
5487         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5488                 sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
5489                     (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
5490                     (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
5491                     (p[1] >> 2) | ((p[2] & 3) << 30),
5492                     (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
5493                     p[0] & 1);
5494         }
5495
5496         rc = sbuf_finish(sb);
5497         sbuf_delete(sb);
5498         free(buf, M_CXGBE);
5499         return (rc);
5500 }
5501
5502 static int
5503 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
5504 {
5505         struct adapter *sc = arg1;
5506         u_int i;
5507         struct sbuf *sb;
5508         uint32_t *buf, *p;
5509         int rc;
5510
5511         rc = sysctl_wire_old_buffer(req, 0);
5512         if (rc != 0)
5513                 return (rc);
5514
5515         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5516         if (sb == NULL)
5517                 return (ENOMEM);
5518
5519         buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
5520             M_ZERO | M_WAITOK);
5521
5522         t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
5523         p = buf;
5524
5525         sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
5526         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5527                 sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
5528                     (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
5529                     p[4], p[3], p[2], p[1], p[0]);
5530         }
5531
5532         sbuf_printf(sb, "\n\nCntl ID               Data");
5533         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5534                 sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
5535                     (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
5536         }
5537
5538         rc = sbuf_finish(sb);
5539         sbuf_delete(sb);
5540         free(buf, M_CXGBE);
5541         return (rc);
5542 }
5543
5544 static int
5545 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
5546 {
5547         struct adapter *sc = arg1;
5548         struct sbuf *sb;
5549         int rc, i;
5550         uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5551         uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5552         uint16_t thres[CIM_NUM_IBQ];
5553         uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
5554         uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
5555         u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
5556
5557         if (is_t4(sc)) {
5558                 cim_num_obq = CIM_NUM_OBQ;
5559                 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
5560                 obq_rdaddr = A_UP_OBQ_0_REALADDR;
5561         } else {
5562                 cim_num_obq = CIM_NUM_OBQ_T5;
5563                 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
5564                 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
5565         }
5566         nq = CIM_NUM_IBQ + cim_num_obq;
5567
5568         rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
5569         if (rc == 0)
5570                 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
5571         if (rc != 0)
5572                 return (rc);
5573
5574         t4_read_cimq_cfg(sc, base, size, thres);
5575
5576         rc = sysctl_wire_old_buffer(req, 0);
5577         if (rc != 0)
5578                 return (rc);
5579
5580         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5581         if (sb == NULL)
5582                 return (ENOMEM);
5583
5584         sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
5585
5586         for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
5587                 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
5588                     qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
5589                     G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5590                     G_QUEREMFLITS(p[2]) * 16);
5591         for ( ; i < nq; i++, p += 4, wr += 2)
5592                 sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
5593                     base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
5594                     wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5595                     G_QUEREMFLITS(p[2]) * 16);
5596
5597         rc = sbuf_finish(sb);
5598         sbuf_delete(sb);
5599
5600         return (rc);
5601 }
5602
5603 static int
5604 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
5605 {
5606         struct adapter *sc = arg1;
5607         struct sbuf *sb;
5608         int rc;
5609         struct tp_cpl_stats stats;
5610
5611         rc = sysctl_wire_old_buffer(req, 0);
5612         if (rc != 0)
5613                 return (rc);
5614
5615         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5616         if (sb == NULL)
5617                 return (ENOMEM);
5618
5619         t4_tp_get_cpl_stats(sc, &stats);
5620
5621         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
5622             "channel 3\n");
5623         sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
5624                    stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
5625         sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
5626                    stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
5627
5628         rc = sbuf_finish(sb);
5629         sbuf_delete(sb);
5630
5631         return (rc);
5632 }
5633
5634 static int
5635 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
5636 {
5637         struct adapter *sc = arg1;
5638         struct sbuf *sb;
5639         int rc;
5640         struct tp_usm_stats stats;
5641
5642         rc = sysctl_wire_old_buffer(req, 0);
5643         if (rc != 0)
5644                 return(rc);
5645
5646         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5647         if (sb == NULL)
5648                 return (ENOMEM);
5649
5650         t4_get_usm_stats(sc, &stats);
5651
5652         sbuf_printf(sb, "Frames: %u\n", stats.frames);
5653         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
5654         sbuf_printf(sb, "Drops:  %u", stats.drops);
5655
5656         rc = sbuf_finish(sb);
5657         sbuf_delete(sb);
5658
5659         return (rc);
5660 }
5661
5662 const char *devlog_level_strings[] = {
5663         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
5664         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
5665         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
5666         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
5667         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
5668         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
5669 };
5670
5671 const char *devlog_facility_strings[] = {
5672         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
5673         [FW_DEVLOG_FACILITY_CF]         = "CF",
5674         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
5675         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
5676         [FW_DEVLOG_FACILITY_RES]        = "RES",
5677         [FW_DEVLOG_FACILITY_HW]         = "HW",
5678         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
5679         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
5680         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
5681         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
5682         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
5683         [FW_DEVLOG_FACILITY_VI]         = "VI",
5684         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
5685         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
5686         [FW_DEVLOG_FACILITY_TM]         = "TM",
5687         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
5688         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
5689         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
5690         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
5691         [FW_DEVLOG_FACILITY_RI]         = "RI",
5692         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
5693         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
5694         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
5695         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE"
5696 };
5697
5698 static int
5699 sysctl_devlog(SYSCTL_HANDLER_ARGS)
5700 {
5701         struct adapter *sc = arg1;
5702         struct devlog_params *dparams = &sc->params.devlog;
5703         struct fw_devlog_e *buf, *e;
5704         int i, j, rc, nentries, first = 0, m;
5705         struct sbuf *sb;
5706         uint64_t ftstamp = UINT64_MAX;
5707
5708         if (dparams->start == 0) {
5709                 dparams->memtype = FW_MEMTYPE_EDC0;
5710                 dparams->start = 0x84000;
5711                 dparams->size = 32768;
5712         }
5713
5714         nentries = dparams->size / sizeof(struct fw_devlog_e);
5715
5716         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
5717         if (buf == NULL)
5718                 return (ENOMEM);
5719
5720         m = fwmtype_to_hwmtype(dparams->memtype);
5721         rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
5722         if (rc != 0)
5723                 goto done;
5724
5725         for (i = 0; i < nentries; i++) {
5726                 e = &buf[i];
5727
5728                 if (e->timestamp == 0)
5729                         break;  /* end */
5730
5731                 e->timestamp = be64toh(e->timestamp);
5732                 e->seqno = be32toh(e->seqno);
5733                 for (j = 0; j < 8; j++)
5734                         e->params[j] = be32toh(e->params[j]);
5735
5736                 if (e->timestamp < ftstamp) {
5737                         ftstamp = e->timestamp;
5738                         first = i;
5739                 }
5740         }
5741
5742         if (buf[first].timestamp == 0)
5743                 goto done;      /* nothing in the log */
5744
5745         rc = sysctl_wire_old_buffer(req, 0);
5746         if (rc != 0)
5747                 goto done;
5748
5749         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5750         if (sb == NULL) {
5751                 rc = ENOMEM;
5752                 goto done;
5753         }
5754         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
5755             "Seq#", "Tstamp", "Level", "Facility", "Message");
5756
5757         i = first;
5758         do {
5759                 e = &buf[i];
5760                 if (e->timestamp == 0)
5761                         break;  /* end */
5762
5763                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
5764                     e->seqno, e->timestamp,
5765                     (e->level < nitems(devlog_level_strings) ?
5766                         devlog_level_strings[e->level] : "UNKNOWN"),
5767                     (e->facility < nitems(devlog_facility_strings) ?
5768                         devlog_facility_strings[e->facility] : "UNKNOWN"));
5769                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
5770                     e->params[2], e->params[3], e->params[4],
5771                     e->params[5], e->params[6], e->params[7]);
5772
5773                 if (++i == nentries)
5774                         i = 0;
5775         } while (i != first);
5776
5777         rc = sbuf_finish(sb);
5778         sbuf_delete(sb);
5779 done:
5780         free(buf, M_CXGBE);
5781         return (rc);
5782 }
5783
5784 static int
5785 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
5786 {
5787         struct adapter *sc = arg1;
5788         struct sbuf *sb;
5789         int rc;
5790         struct tp_fcoe_stats stats[4];
5791
5792         rc = sysctl_wire_old_buffer(req, 0);
5793         if (rc != 0)
5794                 return (rc);
5795
5796         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5797         if (sb == NULL)
5798                 return (ENOMEM);
5799
5800         t4_get_fcoe_stats(sc, 0, &stats[0]);
5801         t4_get_fcoe_stats(sc, 1, &stats[1]);
5802         t4_get_fcoe_stats(sc, 2, &stats[2]);
5803         t4_get_fcoe_stats(sc, 3, &stats[3]);
5804
5805         sbuf_printf(sb, "                   channel 0        channel 1        "
5806             "channel 2        channel 3\n");
5807         sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
5808             stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
5809             stats[3].octetsDDP);
5810         sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
5811             stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
5812         sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
5813             stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
5814             stats[3].framesDrop);
5815
5816         rc = sbuf_finish(sb);
5817         sbuf_delete(sb);
5818
5819         return (rc);
5820 }
5821
5822 static int
5823 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
5824 {
5825         struct adapter *sc = arg1;
5826         struct sbuf *sb;
5827         int rc, i;
5828         unsigned int map, kbps, ipg, mode;
5829         unsigned int pace_tab[NTX_SCHED];
5830
5831         rc = sysctl_wire_old_buffer(req, 0);
5832         if (rc != 0)
5833                 return (rc);
5834
5835         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5836         if (sb == NULL)
5837                 return (ENOMEM);
5838
5839         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
5840         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
5841         t4_read_pace_tbl(sc, pace_tab);
5842
5843         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
5844             "Class IPG (0.1 ns)   Flow IPG (us)");
5845
5846         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
5847                 t4_get_tx_sched(sc, i, &kbps, &ipg);
5848                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
5849                     (mode & (1 << i)) ? "flow" : "class", map & 3);
5850                 if (kbps)
5851                         sbuf_printf(sb, "%9u     ", kbps);
5852                 else
5853                         sbuf_printf(sb, " disabled     ");
5854
5855                 if (ipg)
5856                         sbuf_printf(sb, "%13u        ", ipg);
5857                 else
5858                         sbuf_printf(sb, "     disabled        ");
5859
5860                 if (pace_tab[i])
5861                         sbuf_printf(sb, "%10u", pace_tab[i]);
5862                 else
5863                         sbuf_printf(sb, "  disabled");
5864         }
5865
5866         rc = sbuf_finish(sb);
5867         sbuf_delete(sb);
5868
5869         return (rc);
5870 }
5871
5872 static int
5873 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
5874 {
5875         struct adapter *sc = arg1;
5876         struct sbuf *sb;
5877         int rc, i, j;
5878         uint64_t *p0, *p1;
5879         struct lb_port_stats s[2];
5880         static const char *stat_name[] = {
5881                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
5882                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
5883                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
5884                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
5885                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
5886                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
5887                 "BG2FramesTrunc:", "BG3FramesTrunc:"
5888         };
5889
5890         rc = sysctl_wire_old_buffer(req, 0);
5891         if (rc != 0)
5892                 return (rc);
5893
5894         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5895         if (sb == NULL)
5896                 return (ENOMEM);
5897
5898         memset(s, 0, sizeof(s));
5899
5900         for (i = 0; i < 4; i += 2) {
5901                 t4_get_lb_stats(sc, i, &s[0]);
5902                 t4_get_lb_stats(sc, i + 1, &s[1]);
5903
5904                 p0 = &s[0].octets;
5905                 p1 = &s[1].octets;
5906                 sbuf_printf(sb, "%s                       Loopback %u"
5907                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
5908
5909                 for (j = 0; j < nitems(stat_name); j++)
5910                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
5911                                    *p0++, *p1++);
5912         }
5913
5914         rc = sbuf_finish(sb);
5915         sbuf_delete(sb);
5916
5917         return (rc);
5918 }
5919
5920 static int
5921 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
5922 {
5923         int rc = 0;
5924         struct port_info *pi = arg1;
5925         struct sbuf *sb;
5926         static const char *linkdnreasons[] = {
5927                 "non-specific", "remote fault", "autoneg failed", "reserved3",
5928                 "PHY overheated", "unknown", "rx los", "reserved7"
5929         };
5930
5931         rc = sysctl_wire_old_buffer(req, 0);
5932         if (rc != 0)
5933                 return(rc);
5934         sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
5935         if (sb == NULL)
5936                 return (ENOMEM);
5937
5938         if (pi->linkdnrc < 0)
5939                 sbuf_printf(sb, "n/a");
5940         else if (pi->linkdnrc < nitems(linkdnreasons))
5941                 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
5942         else
5943                 sbuf_printf(sb, "%d", pi->linkdnrc);
5944
5945         rc = sbuf_finish(sb);
5946         sbuf_delete(sb);
5947
5948         return (rc);
5949 }
5950
5951 struct mem_desc {
5952         unsigned int base;
5953         unsigned int limit;
5954         unsigned int idx;
5955 };
5956
5957 static int
5958 mem_desc_cmp(const void *a, const void *b)
5959 {
5960         return ((const struct mem_desc *)a)->base -
5961                ((const struct mem_desc *)b)->base;
5962 }
5963
5964 static void
5965 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
5966     unsigned int to)
5967 {
5968         unsigned int size;
5969
5970         size = to - from + 1;
5971         if (size == 0)
5972                 return;
5973
5974         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
5975         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
5976 }
5977
5978 static int
5979 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
5980 {
5981         struct adapter *sc = arg1;
5982         struct sbuf *sb;
5983         int rc, i, n;
5984         uint32_t lo, hi, used, alloc;
5985         static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
5986         static const char *region[] = {
5987                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
5988                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
5989                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
5990                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
5991                 "RQUDP region:", "PBL region:", "TXPBL region:",
5992                 "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
5993                 "On-chip queues:"
5994         };
5995         struct mem_desc avail[4];
5996         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
5997         struct mem_desc *md = mem;
5998
5999         rc = sysctl_wire_old_buffer(req, 0);
6000         if (rc != 0)
6001                 return (rc);
6002
6003         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6004         if (sb == NULL)
6005                 return (ENOMEM);
6006
6007         for (i = 0; i < nitems(mem); i++) {
6008                 mem[i].limit = 0;
6009                 mem[i].idx = i;
6010         }
6011
6012         /* Find and sort the populated memory ranges */
6013         i = 0;
6014         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
6015         if (lo & F_EDRAM0_ENABLE) {
6016                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
6017                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
6018                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
6019                 avail[i].idx = 0;
6020                 i++;
6021         }
6022         if (lo & F_EDRAM1_ENABLE) {
6023                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
6024                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
6025                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
6026                 avail[i].idx = 1;
6027                 i++;
6028         }
6029         if (lo & F_EXT_MEM_ENABLE) {
6030                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
6031                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
6032                 avail[i].limit = avail[i].base +
6033                     (G_EXT_MEM_SIZE(hi) << 20);
6034                 avail[i].idx = is_t4(sc) ? 2 : 3;       /* Call it MC for T4 */
6035                 i++;
6036         }
6037         if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
6038                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
6039                 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
6040                 avail[i].limit = avail[i].base +
6041                     (G_EXT_MEM1_SIZE(hi) << 20);
6042                 avail[i].idx = 4;
6043                 i++;
6044         }
6045         if (!i)                                    /* no memory available */
6046                 return 0;
6047         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
6048
6049         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
6050         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
6051         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
6052         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
6053         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
6054         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
6055         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
6056         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
6057         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
6058
6059         /* the next few have explicit upper bounds */
6060         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
6061         md->limit = md->base - 1 +
6062                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
6063                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
6064         md++;
6065
6066         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
6067         md->limit = md->base - 1 +
6068                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
6069                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
6070         md++;
6071
6072         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6073                 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
6074                 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
6075                 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
6076         } else {
6077                 md->base = 0;
6078                 md->idx = nitems(region);  /* hide it */
6079         }
6080         md++;
6081
6082 #define ulp_region(reg) \
6083         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
6084         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
6085
6086         ulp_region(RX_ISCSI);
6087         ulp_region(RX_TDDP);
6088         ulp_region(TX_TPT);
6089         ulp_region(RX_STAG);
6090         ulp_region(RX_RQ);
6091         ulp_region(RX_RQUDP);
6092         ulp_region(RX_PBL);
6093         ulp_region(TX_PBL);
6094 #undef ulp_region
6095
6096         md->base = 0;
6097         md->idx = nitems(region);
6098         if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
6099                 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
6100                 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
6101                     A_SGE_DBVFIFO_SIZE))) << 2) - 1;
6102         }
6103         md++;
6104
6105         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
6106         md->limit = md->base + sc->tids.ntids - 1;
6107         md++;
6108         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
6109         md->limit = md->base + sc->tids.ntids - 1;
6110         md++;
6111
6112         md->base = sc->vres.ocq.start;
6113         if (sc->vres.ocq.size)
6114                 md->limit = md->base + sc->vres.ocq.size - 1;
6115         else
6116                 md->idx = nitems(region);  /* hide it */
6117         md++;
6118
6119         /* add any address-space holes, there can be up to 3 */
6120         for (n = 0; n < i - 1; n++)
6121                 if (avail[n].limit < avail[n + 1].base)
6122                         (md++)->base = avail[n].limit;
6123         if (avail[n].limit)
6124                 (md++)->base = avail[n].limit;
6125
6126         n = md - mem;
6127         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
6128
6129         for (lo = 0; lo < i; lo++)
6130                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
6131                                 avail[lo].limit - 1);
6132
6133         sbuf_printf(sb, "\n");
6134         for (i = 0; i < n; i++) {
6135                 if (mem[i].idx >= nitems(region))
6136                         continue;                        /* skip holes */
6137                 if (!mem[i].limit)
6138                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
6139                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
6140                                 mem[i].limit);
6141         }
6142
6143         sbuf_printf(sb, "\n");
6144         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
6145         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
6146         mem_region_show(sb, "uP RAM:", lo, hi);
6147
6148         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
6149         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
6150         mem_region_show(sb, "uP Extmem2:", lo, hi);
6151
6152         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
6153         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
6154                    G_PMRXMAXPAGE(lo),
6155                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
6156                    (lo & F_PMRXNUMCHN) ? 2 : 1);
6157
6158         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
6159         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
6160         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
6161                    G_PMTXMAXPAGE(lo),
6162                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
6163                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
6164         sbuf_printf(sb, "%u p-structs\n",
6165                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
6166
6167         for (i = 0; i < 4; i++) {
6168                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
6169                 if (is_t4(sc)) {
6170                         used = G_USED(lo);
6171                         alloc = G_ALLOC(lo);
6172                 } else {
6173                         used = G_T5_USED(lo);
6174                         alloc = G_T5_ALLOC(lo);
6175                 }
6176                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
6177                            i, used, alloc);
6178         }
6179         for (i = 0; i < 4; i++) {
6180                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
6181                 if (is_t4(sc)) {
6182                         used = G_USED(lo);
6183                         alloc = G_ALLOC(lo);
6184                 } else {
6185                         used = G_T5_USED(lo);
6186                         alloc = G_T5_ALLOC(lo);
6187                 }
6188                 sbuf_printf(sb,
6189                            "\nLoopback %d using %u pages out of %u allocated",
6190                            i, used, alloc);
6191         }
6192
6193         rc = sbuf_finish(sb);
6194         sbuf_delete(sb);
6195
6196         return (rc);
6197 }
6198
6199 static inline void
6200 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
6201 {
6202         *mask = x | y;
6203         y = htobe64(y);
6204         memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
6205 }
6206
6207 static int
6208 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
6209 {
6210         struct adapter *sc = arg1;
6211         struct sbuf *sb;
6212         int rc, i, n;
6213
6214         rc = sysctl_wire_old_buffer(req, 0);
6215         if (rc != 0)
6216                 return (rc);
6217
6218         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6219         if (sb == NULL)
6220                 return (ENOMEM);
6221
6222         sbuf_printf(sb,
6223             "Idx  Ethernet address     Mask     Vld Ports PF"
6224             "  VF              Replication             P0 P1 P2 P3  ML");
6225         n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
6226             NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
6227         for (i = 0; i < n; i++) {
6228                 uint64_t tcamx, tcamy, mask;
6229                 uint32_t cls_lo, cls_hi;
6230                 uint8_t addr[ETHER_ADDR_LEN];
6231
6232                 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
6233                 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
6234                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
6235                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
6236
6237                 if (tcamx & tcamy)
6238                         continue;
6239
6240                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
6241                 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
6242                            "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
6243                            addr[3], addr[4], addr[5], (uintmax_t)mask,
6244                            (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
6245                            G_PORTMAP(cls_hi), G_PF(cls_lo),
6246                            (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
6247
6248                 if (cls_lo & F_REPLICATE) {
6249                         struct fw_ldst_cmd ldst_cmd;
6250
6251                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
6252                         ldst_cmd.op_to_addrspace =
6253                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
6254                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
6255                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
6256                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
6257                         ldst_cmd.u.mps.fid_ctl =
6258                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
6259                                 V_FW_LDST_CMD_CTL(i));
6260
6261                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
6262                             "t4mps");
6263                         if (rc)
6264                                 break;
6265                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
6266                             sizeof(ldst_cmd), &ldst_cmd);
6267                         end_synchronized_op(sc, 0);
6268
6269                         if (rc != 0) {
6270                                 sbuf_printf(sb,
6271                                     " ------------ error %3u ------------", rc);
6272                                 rc = 0;
6273                         } else {
6274                                 sbuf_printf(sb, " %08x %08x %08x %08x",
6275                                     be32toh(ldst_cmd.u.mps.rplc127_96),
6276                                     be32toh(ldst_cmd.u.mps.rplc95_64),
6277                                     be32toh(ldst_cmd.u.mps.rplc63_32),
6278                                     be32toh(ldst_cmd.u.mps.rplc31_0));
6279                         }
6280                 } else
6281                         sbuf_printf(sb, "%36s", "");
6282
6283                 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
6284                     G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
6285                     G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
6286         }
6287
6288         if (rc)
6289                 (void) sbuf_finish(sb);
6290         else
6291                 rc = sbuf_finish(sb);
6292         sbuf_delete(sb);
6293
6294         return (rc);
6295 }
6296
6297 static int
6298 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
6299 {
6300         struct adapter *sc = arg1;
6301         struct sbuf *sb;
6302         int rc;
6303         uint16_t mtus[NMTUS];
6304
6305         rc = sysctl_wire_old_buffer(req, 0);
6306         if (rc != 0)
6307                 return (rc);
6308
6309         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6310         if (sb == NULL)
6311                 return (ENOMEM);
6312
6313         t4_read_mtu_tbl(sc, mtus, NULL);
6314
6315         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
6316             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
6317             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
6318             mtus[14], mtus[15]);
6319
6320         rc = sbuf_finish(sb);
6321         sbuf_delete(sb);
6322
6323         return (rc);
6324 }
6325
6326 static int
6327 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
6328 {
6329         struct adapter *sc = arg1;
6330         struct sbuf *sb;
6331         int rc, i;
6332         uint32_t cnt[PM_NSTATS];
6333         uint64_t cyc[PM_NSTATS];
6334         static const char *rx_stats[] = {
6335                 "Read:", "Write bypass:", "Write mem:", "Flush:"
6336         };
6337         static const char *tx_stats[] = {
6338                 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
6339         };
6340
6341         rc = sysctl_wire_old_buffer(req, 0);
6342         if (rc != 0)
6343                 return (rc);
6344
6345         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6346         if (sb == NULL)
6347                 return (ENOMEM);
6348
6349         t4_pmtx_get_stats(sc, cnt, cyc);
6350         sbuf_printf(sb, "                Tx pcmds             Tx bytes");
6351         for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
6352                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
6353                     cyc[i]);
6354
6355         t4_pmrx_get_stats(sc, cnt, cyc);
6356         sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
6357         for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
6358                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
6359                     cyc[i]);
6360
6361         rc = sbuf_finish(sb);
6362         sbuf_delete(sb);
6363
6364         return (rc);
6365 }
6366
6367 static int
6368 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
6369 {
6370         struct adapter *sc = arg1;
6371         struct sbuf *sb;
6372         int rc;
6373         struct tp_rdma_stats stats;
6374
6375         rc = sysctl_wire_old_buffer(req, 0);
6376         if (rc != 0)
6377                 return (rc);
6378
6379         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6380         if (sb == NULL)
6381                 return (ENOMEM);
6382
6383         t4_tp_get_rdma_stats(sc, &stats);
6384         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
6385         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
6386
6387         rc = sbuf_finish(sb);
6388         sbuf_delete(sb);
6389
6390         return (rc);
6391 }
6392
6393 static int
6394 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
6395 {
6396         struct adapter *sc = arg1;
6397         struct sbuf *sb;
6398         int rc;
6399         struct tp_tcp_stats v4, v6;
6400
6401         rc = sysctl_wire_old_buffer(req, 0);
6402         if (rc != 0)
6403                 return (rc);
6404
6405         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6406         if (sb == NULL)
6407                 return (ENOMEM);
6408
6409         t4_tp_get_tcp_stats(sc, &v4, &v6);
6410         sbuf_printf(sb,
6411             "                                IP                 IPv6\n");
6412         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
6413             v4.tcpOutRsts, v6.tcpOutRsts);
6414         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
6415             v4.tcpInSegs, v6.tcpInSegs);
6416         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
6417             v4.tcpOutSegs, v6.tcpOutSegs);
6418         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
6419             v4.tcpRetransSegs, v6.tcpRetransSegs);
6420
6421         rc = sbuf_finish(sb);
6422         sbuf_delete(sb);
6423
6424         return (rc);
6425 }
6426
6427 static int
6428 sysctl_tids(SYSCTL_HANDLER_ARGS)
6429 {
6430         struct adapter *sc = arg1;
6431         struct sbuf *sb;
6432         int rc;
6433         struct tid_info *t = &sc->tids;
6434
6435         rc = sysctl_wire_old_buffer(req, 0);
6436         if (rc != 0)
6437                 return (rc);
6438
6439         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6440         if (sb == NULL)
6441                 return (ENOMEM);
6442
6443         if (t->natids) {
6444                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
6445                     t->atids_in_use);
6446         }
6447
6448         if (t->ntids) {
6449                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6450                         uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
6451
6452                         if (b) {
6453                                 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
6454                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6455                                     t->ntids - 1);
6456                         } else {
6457                                 sbuf_printf(sb, "TID range: %u-%u",
6458                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6459                                     t->ntids - 1);
6460                         }
6461                 } else
6462                         sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
6463                 sbuf_printf(sb, ", in use: %u\n",
6464                     atomic_load_acq_int(&t->tids_in_use));
6465         }
6466
6467         if (t->nstids) {
6468                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
6469                     t->stid_base + t->nstids - 1, t->stids_in_use);
6470         }
6471
6472         if (t->nftids) {
6473                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
6474                     t->ftid_base + t->nftids - 1);
6475         }
6476
6477         if (t->netids) {
6478                 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
6479                     t->etid_base + t->netids - 1);
6480         }
6481
6482         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
6483             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
6484             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
6485
6486         rc = sbuf_finish(sb);
6487         sbuf_delete(sb);
6488
6489         return (rc);
6490 }
6491
6492 static int
6493 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
6494 {
6495         struct adapter *sc = arg1;
6496         struct sbuf *sb;
6497         int rc;
6498         struct tp_err_stats stats;
6499
6500         rc = sysctl_wire_old_buffer(req, 0);
6501         if (rc != 0)
6502                 return (rc);
6503
6504         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6505         if (sb == NULL)
6506                 return (ENOMEM);
6507
6508         t4_tp_get_err_stats(sc, &stats);
6509
6510         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
6511                       "channel 3\n");
6512         sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
6513             stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
6514             stats.macInErrs[3]);
6515         sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
6516             stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
6517             stats.hdrInErrs[3]);
6518         sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
6519             stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
6520             stats.tcpInErrs[3]);
6521         sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
6522             stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
6523             stats.tcp6InErrs[3]);
6524         sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
6525             stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
6526             stats.tnlCongDrops[3]);
6527         sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
6528             stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
6529             stats.tnlTxDrops[3]);
6530         sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
6531             stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
6532             stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
6533         sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
6534             stats.ofldChanDrops[0], stats.ofldChanDrops[1],
6535             stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
6536         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
6537             stats.ofldNoNeigh, stats.ofldCongDefer);
6538
6539         rc = sbuf_finish(sb);
6540         sbuf_delete(sb);
6541
6542         return (rc);
6543 }
6544
6545 struct field_desc {
6546         const char *name;
6547         u_int start;
6548         u_int width;
6549 };
6550
6551 static void
6552 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
6553 {
6554         char buf[32];
6555         int line_size = 0;
6556
6557         while (f->name) {
6558                 uint64_t mask = (1ULL << f->width) - 1;
6559                 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
6560                     ((uintmax_t)v >> f->start) & mask);
6561
6562                 if (line_size + len >= 79) {
6563                         line_size = 8;
6564                         sbuf_printf(sb, "\n        ");
6565                 }
6566                 sbuf_printf(sb, "%s ", buf);
6567                 line_size += len + 1;
6568                 f++;
6569         }
6570         sbuf_printf(sb, "\n");
6571 }
6572
6573 static struct field_desc tp_la0[] = {
6574         { "RcfOpCodeOut", 60, 4 },
6575         { "State", 56, 4 },
6576         { "WcfState", 52, 4 },
6577         { "RcfOpcSrcOut", 50, 2 },
6578         { "CRxError", 49, 1 },
6579         { "ERxError", 48, 1 },
6580         { "SanityFailed", 47, 1 },
6581         { "SpuriousMsg", 46, 1 },
6582         { "FlushInputMsg", 45, 1 },
6583         { "FlushInputCpl", 44, 1 },
6584         { "RssUpBit", 43, 1 },
6585         { "RssFilterHit", 42, 1 },
6586         { "Tid", 32, 10 },
6587         { "InitTcb", 31, 1 },
6588         { "LineNumber", 24, 7 },
6589         { "Emsg", 23, 1 },
6590         { "EdataOut", 22, 1 },
6591         { "Cmsg", 21, 1 },
6592         { "CdataOut", 20, 1 },
6593         { "EreadPdu", 19, 1 },
6594         { "CreadPdu", 18, 1 },
6595         { "TunnelPkt", 17, 1 },
6596         { "RcfPeerFin", 16, 1 },
6597         { "RcfReasonOut", 12, 4 },
6598         { "TxCchannel", 10, 2 },
6599         { "RcfTxChannel", 8, 2 },
6600         { "RxEchannel", 6, 2 },
6601         { "RcfRxChannel", 5, 1 },
6602         { "RcfDataOutSrdy", 4, 1 },
6603         { "RxDvld", 3, 1 },
6604         { "RxOoDvld", 2, 1 },
6605         { "RxCongestion", 1, 1 },
6606         { "TxCongestion", 0, 1 },
6607         { NULL }
6608 };
6609
6610 static struct field_desc tp_la1[] = {
6611         { "CplCmdIn", 56, 8 },
6612         { "CplCmdOut", 48, 8 },
6613         { "ESynOut", 47, 1 },
6614         { "EAckOut", 46, 1 },
6615         { "EFinOut", 45, 1 },
6616         { "ERstOut", 44, 1 },
6617         { "SynIn", 43, 1 },
6618         { "AckIn", 42, 1 },
6619         { "FinIn", 41, 1 },
6620         { "RstIn", 40, 1 },
6621         { "DataIn", 39, 1 },
6622         { "DataInVld", 38, 1 },
6623         { "PadIn", 37, 1 },
6624         { "RxBufEmpty", 36, 1 },
6625         { "RxDdp", 35, 1 },
6626         { "RxFbCongestion", 34, 1 },
6627         { "TxFbCongestion", 33, 1 },
6628         { "TxPktSumSrdy", 32, 1 },
6629         { "RcfUlpType", 28, 4 },
6630         { "Eread", 27, 1 },
6631         { "Ebypass", 26, 1 },
6632         { "Esave", 25, 1 },
6633         { "Static0", 24, 1 },
6634         { "Cread", 23, 1 },
6635         { "Cbypass", 22, 1 },
6636         { "Csave", 21, 1 },
6637         { "CPktOut", 20, 1 },
6638         { "RxPagePoolFull", 18, 2 },
6639         { "RxLpbkPkt", 17, 1 },
6640         { "TxLpbkPkt", 16, 1 },
6641         { "RxVfValid", 15, 1 },
6642         { "SynLearned", 14, 1 },
6643         { "SetDelEntry", 13, 1 },
6644         { "SetInvEntry", 12, 1 },
6645         { "CpcmdDvld", 11, 1 },
6646         { "CpcmdSave", 10, 1 },
6647         { "RxPstructsFull", 8, 2 },
6648         { "EpcmdDvld", 7, 1 },
6649         { "EpcmdFlush", 6, 1 },
6650         { "EpcmdTrimPrefix", 5, 1 },
6651         { "EpcmdTrimPostfix", 4, 1 },
6652         { "ERssIp4Pkt", 3, 1 },
6653         { "ERssIp6Pkt", 2, 1 },
6654         { "ERssTcpUdpPkt", 1, 1 },
6655         { "ERssFceFipPkt", 0, 1 },
6656         { NULL }
6657 };
6658
6659 static struct field_desc tp_la2[] = {
6660         { "CplCmdIn", 56, 8 },
6661         { "MpsVfVld", 55, 1 },
6662         { "MpsPf", 52, 3 },
6663         { "MpsVf", 44, 8 },
6664         { "SynIn", 43, 1 },
6665         { "AckIn", 42, 1 },
6666         { "FinIn", 41, 1 },
6667         { "RstIn", 40, 1 },
6668         { "DataIn", 39, 1 },
6669         { "DataInVld", 38, 1 },
6670         { "PadIn", 37, 1 },
6671         { "RxBufEmpty", 36, 1 },
6672         { "RxDdp", 35, 1 },
6673         { "RxFbCongestion", 34, 1 },
6674         { "TxFbCongestion", 33, 1 },
6675         { "TxPktSumSrdy", 32, 1 },
6676         { "RcfUlpType", 28, 4 },
6677         { "Eread", 27, 1 },
6678         { "Ebypass", 26, 1 },
6679         { "Esave", 25, 1 },
6680         { "Static0", 24, 1 },
6681         { "Cread", 23, 1 },
6682         { "Cbypass", 22, 1 },
6683         { "Csave", 21, 1 },
6684         { "CPktOut", 20, 1 },
6685         { "RxPagePoolFull", 18, 2 },
6686         { "RxLpbkPkt", 17, 1 },
6687         { "TxLpbkPkt", 16, 1 },
6688         { "RxVfValid", 15, 1 },
6689         { "SynLearned", 14, 1 },
6690         { "SetDelEntry", 13, 1 },
6691         { "SetInvEntry", 12, 1 },
6692         { "CpcmdDvld", 11, 1 },
6693         { "CpcmdSave", 10, 1 },
6694         { "RxPstructsFull", 8, 2 },
6695         { "EpcmdDvld", 7, 1 },
6696         { "EpcmdFlush", 6, 1 },
6697         { "EpcmdTrimPrefix", 5, 1 },
6698         { "EpcmdTrimPostfix", 4, 1 },
6699         { "ERssIp4Pkt", 3, 1 },
6700         { "ERssIp6Pkt", 2, 1 },
6701         { "ERssTcpUdpPkt", 1, 1 },
6702         { "ERssFceFipPkt", 0, 1 },
6703         { NULL }
6704 };
6705
6706 static void
6707 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
6708 {
6709
6710         field_desc_show(sb, *p, tp_la0);
6711 }
6712
6713 static void
6714 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
6715 {
6716
6717         if (idx)
6718                 sbuf_printf(sb, "\n");
6719         field_desc_show(sb, p[0], tp_la0);
6720         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6721                 field_desc_show(sb, p[1], tp_la0);
6722 }
6723
6724 static void
6725 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
6726 {
6727
6728         if (idx)
6729                 sbuf_printf(sb, "\n");
6730         field_desc_show(sb, p[0], tp_la0);
6731         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6732                 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
6733 }
6734
6735 static int
6736 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
6737 {
6738         struct adapter *sc = arg1;
6739         struct sbuf *sb;
6740         uint64_t *buf, *p;
6741         int rc;
6742         u_int i, inc;
6743         void (*show_func)(struct sbuf *, uint64_t *, int);
6744
6745         rc = sysctl_wire_old_buffer(req, 0);
6746         if (rc != 0)
6747                 return (rc);
6748
6749         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6750         if (sb == NULL)
6751                 return (ENOMEM);
6752
6753         buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
6754
6755         t4_tp_read_la(sc, buf, NULL);
6756         p = buf;
6757
6758         switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
6759         case 2:
6760                 inc = 2;
6761                 show_func = tp_la_show2;
6762                 break;
6763         case 3:
6764                 inc = 2;
6765                 show_func = tp_la_show3;
6766                 break;
6767         default:
6768                 inc = 1;
6769                 show_func = tp_la_show;
6770         }
6771
6772         for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
6773                 (*show_func)(sb, p, i);
6774
6775         rc = sbuf_finish(sb);
6776         sbuf_delete(sb);
6777         free(buf, M_CXGBE);
6778         return (rc);
6779 }
6780
6781 static int
6782 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
6783 {
6784         struct adapter *sc = arg1;
6785         struct sbuf *sb;
6786         int rc;
6787         u64 nrate[NCHAN], orate[NCHAN];
6788
6789         rc = sysctl_wire_old_buffer(req, 0);
6790         if (rc != 0)
6791                 return (rc);
6792
6793         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6794         if (sb == NULL)
6795                 return (ENOMEM);
6796
6797         t4_get_chan_txrate(sc, nrate, orate);
6798         sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
6799                  "channel 3\n");
6800         sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
6801             nrate[0], nrate[1], nrate[2], nrate[3]);
6802         sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
6803             orate[0], orate[1], orate[2], orate[3]);
6804
6805         rc = sbuf_finish(sb);
6806         sbuf_delete(sb);
6807
6808         return (rc);
6809 }
6810
6811 static int
6812 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
6813 {
6814         struct adapter *sc = arg1;
6815         struct sbuf *sb;
6816         uint32_t *buf, *p;
6817         int rc, i;
6818
6819         rc = sysctl_wire_old_buffer(req, 0);
6820         if (rc != 0)
6821                 return (rc);
6822
6823         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6824         if (sb == NULL)
6825                 return (ENOMEM);
6826
6827         buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
6828             M_ZERO | M_WAITOK);
6829
6830         t4_ulprx_read_la(sc, buf);
6831         p = buf;
6832
6833         sbuf_printf(sb, "      Pcmd        Type   Message"
6834             "                Data");
6835         for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
6836                 sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
6837                     p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
6838         }
6839
6840         rc = sbuf_finish(sb);
6841         sbuf_delete(sb);
6842         free(buf, M_CXGBE);
6843         return (rc);
6844 }
6845
6846 static int
6847 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
6848 {
6849         struct adapter *sc = arg1;
6850         struct sbuf *sb;
6851         int rc, v;
6852
6853         rc = sysctl_wire_old_buffer(req, 0);
6854         if (rc != 0)
6855                 return (rc);
6856
6857         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6858         if (sb == NULL)
6859                 return (ENOMEM);
6860
6861         v = t4_read_reg(sc, A_SGE_STAT_CFG);
6862         if (G_STATSOURCE_T5(v) == 7) {
6863                 if (G_STATMODE(v) == 0) {
6864                         sbuf_printf(sb, "total %d, incomplete %d",
6865                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6866                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6867                 } else if (G_STATMODE(v) == 1) {
6868                         sbuf_printf(sb, "total %d, data overflow %d",
6869                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6870                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6871                 }
6872         }
6873         rc = sbuf_finish(sb);
6874         sbuf_delete(sb);
6875
6876         return (rc);
6877 }
6878 #endif
6879
6880 static inline void
6881 txq_start(struct ifnet *ifp, struct sge_txq *txq)
6882 {
6883         struct buf_ring *br;
6884         struct mbuf *m;
6885
6886         TXQ_LOCK_ASSERT_OWNED(txq);
6887
6888         br = txq->br;
6889         m = txq->m ? txq->m : drbr_dequeue(ifp, br);
6890         if (m)
6891                 t4_eth_tx(ifp, txq, m);
6892 }
6893
6894 void
6895 t4_tx_callout(void *arg)
6896 {
6897         struct sge_eq *eq = arg;
6898         struct adapter *sc;
6899
6900         if (EQ_TRYLOCK(eq) == 0)
6901                 goto reschedule;
6902
6903         if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) {
6904                 EQ_UNLOCK(eq);
6905 reschedule:
6906                 if (__predict_true(!(eq->flags && EQ_DOOMED)))
6907                         callout_schedule(&eq->tx_callout, 1);
6908                 return;
6909         }
6910
6911         EQ_LOCK_ASSERT_OWNED(eq);
6912
6913         if (__predict_true((eq->flags & EQ_DOOMED) == 0)) {
6914
6915                 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
6916                         struct sge_txq *txq = arg;
6917                         struct port_info *pi = txq->ifp->if_softc;
6918
6919                         sc = pi->adapter;
6920                 } else {
6921                         struct sge_wrq *wrq = arg;
6922
6923                         sc = wrq->adapter;
6924                 }
6925
6926                 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task);
6927         }
6928
6929         EQ_UNLOCK(eq);
6930 }
6931
6932 void
6933 t4_tx_task(void *arg, int count)
6934 {
6935         struct sge_eq *eq = arg;
6936
6937         EQ_LOCK(eq);
6938         if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
6939                 struct sge_txq *txq = arg;
6940                 txq_start(txq->ifp, txq);
6941         } else {
6942                 struct sge_wrq *wrq = arg;
6943                 t4_wrq_tx_locked(wrq->adapter, wrq, NULL);
6944         }
6945         EQ_UNLOCK(eq);
6946 }
6947
6948 static uint32_t
6949 fconf_to_mode(uint32_t fconf)
6950 {
6951         uint32_t mode;
6952
6953         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
6954             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
6955
6956         if (fconf & F_FRAGMENTATION)
6957                 mode |= T4_FILTER_IP_FRAGMENT;
6958
6959         if (fconf & F_MPSHITTYPE)
6960                 mode |= T4_FILTER_MPS_HIT_TYPE;
6961
6962         if (fconf & F_MACMATCH)
6963                 mode |= T4_FILTER_MAC_IDX;
6964
6965         if (fconf & F_ETHERTYPE)
6966                 mode |= T4_FILTER_ETH_TYPE;
6967
6968         if (fconf & F_PROTOCOL)
6969                 mode |= T4_FILTER_IP_PROTO;
6970
6971         if (fconf & F_TOS)
6972                 mode |= T4_FILTER_IP_TOS;
6973
6974         if (fconf & F_VLAN)
6975                 mode |= T4_FILTER_VLAN;
6976
6977         if (fconf & F_VNIC_ID)
6978                 mode |= T4_FILTER_VNIC;
6979
6980         if (fconf & F_PORT)
6981                 mode |= T4_FILTER_PORT;
6982
6983         if (fconf & F_FCOE)
6984                 mode |= T4_FILTER_FCoE;
6985
6986         return (mode);
6987 }
6988
6989 static uint32_t
6990 mode_to_fconf(uint32_t mode)
6991 {
6992         uint32_t fconf = 0;
6993
6994         if (mode & T4_FILTER_IP_FRAGMENT)
6995                 fconf |= F_FRAGMENTATION;
6996
6997         if (mode & T4_FILTER_MPS_HIT_TYPE)
6998                 fconf |= F_MPSHITTYPE;
6999
7000         if (mode & T4_FILTER_MAC_IDX)
7001                 fconf |= F_MACMATCH;
7002
7003         if (mode & T4_FILTER_ETH_TYPE)
7004                 fconf |= F_ETHERTYPE;
7005
7006         if (mode & T4_FILTER_IP_PROTO)
7007                 fconf |= F_PROTOCOL;
7008
7009         if (mode & T4_FILTER_IP_TOS)
7010                 fconf |= F_TOS;
7011
7012         if (mode & T4_FILTER_VLAN)
7013                 fconf |= F_VLAN;
7014
7015         if (mode & T4_FILTER_VNIC)
7016                 fconf |= F_VNIC_ID;
7017
7018         if (mode & T4_FILTER_PORT)
7019                 fconf |= F_PORT;
7020
7021         if (mode & T4_FILTER_FCoE)
7022                 fconf |= F_FCOE;
7023
7024         return (fconf);
7025 }
7026
7027 static uint32_t
7028 fspec_to_fconf(struct t4_filter_specification *fs)
7029 {
7030         uint32_t fconf = 0;
7031
7032         if (fs->val.frag || fs->mask.frag)
7033                 fconf |= F_FRAGMENTATION;
7034
7035         if (fs->val.matchtype || fs->mask.matchtype)
7036                 fconf |= F_MPSHITTYPE;
7037
7038         if (fs->val.macidx || fs->mask.macidx)
7039                 fconf |= F_MACMATCH;
7040
7041         if (fs->val.ethtype || fs->mask.ethtype)
7042                 fconf |= F_ETHERTYPE;
7043
7044         if (fs->val.proto || fs->mask.proto)
7045                 fconf |= F_PROTOCOL;
7046
7047         if (fs->val.tos || fs->mask.tos)
7048                 fconf |= F_TOS;
7049
7050         if (fs->val.vlan_vld || fs->mask.vlan_vld)
7051                 fconf |= F_VLAN;
7052
7053         if (fs->val.vnic_vld || fs->mask.vnic_vld)
7054                 fconf |= F_VNIC_ID;
7055
7056         if (fs->val.iport || fs->mask.iport)
7057                 fconf |= F_PORT;
7058
7059         if (fs->val.fcoe || fs->mask.fcoe)
7060                 fconf |= F_FCOE;
7061
7062         return (fconf);
7063 }
7064
7065 static int
7066 get_filter_mode(struct adapter *sc, uint32_t *mode)
7067 {
7068         int rc;
7069         uint32_t fconf;
7070
7071         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7072             "t4getfm");
7073         if (rc)
7074                 return (rc);
7075
7076         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
7077             A_TP_VLAN_PRI_MAP);
7078
7079         if (sc->params.tp.vlan_pri_map != fconf) {
7080                 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
7081                     device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
7082                     fconf);
7083                 sc->params.tp.vlan_pri_map = fconf;
7084         }
7085
7086         *mode = fconf_to_mode(sc->params.tp.vlan_pri_map);
7087
7088         end_synchronized_op(sc, LOCK_HELD);
7089         return (0);
7090 }
7091
7092 static int
7093 set_filter_mode(struct adapter *sc, uint32_t mode)
7094 {
7095         uint32_t fconf;
7096         int rc;
7097
7098         fconf = mode_to_fconf(mode);
7099
7100         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7101             "t4setfm");
7102         if (rc)
7103                 return (rc);
7104
7105         if (sc->tids.ftids_in_use > 0) {
7106                 rc = EBUSY;
7107                 goto done;
7108         }
7109
7110 #ifdef TCP_OFFLOAD
7111         if (sc->offload_map) {
7112                 rc = EBUSY;
7113                 goto done;
7114         }
7115 #endif
7116
7117 #ifdef notyet
7118         rc = -t4_set_filter_mode(sc, fconf);
7119         if (rc == 0)
7120                 sc->filter_mode = fconf;
7121 #else
7122         rc = ENOTSUP;
7123 #endif
7124
7125 done:
7126         end_synchronized_op(sc, LOCK_HELD);
7127         return (rc);
7128 }
7129
7130 static inline uint64_t
7131 get_filter_hits(struct adapter *sc, uint32_t fid)
7132 {
7133         uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
7134         uint64_t hits;
7135
7136         memwin_info(sc, 0, &mw_base, NULL);
7137         off = position_memwin(sc, 0,
7138             tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
7139         if (is_t4(sc)) {
7140                 hits = t4_read_reg64(sc, mw_base + off + 16);
7141                 hits = be64toh(hits);
7142         } else {
7143                 hits = t4_read_reg(sc, mw_base + off + 24);
7144                 hits = be32toh(hits);
7145         }
7146
7147         return (hits);
7148 }
7149
7150 static int
7151 get_filter(struct adapter *sc, struct t4_filter *t)
7152 {
7153         int i, rc, nfilters = sc->tids.nftids;
7154         struct filter_entry *f;
7155
7156         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7157             "t4getf");
7158         if (rc)
7159                 return (rc);
7160
7161         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
7162             t->idx >= nfilters) {
7163                 t->idx = 0xffffffff;
7164                 goto done;
7165         }
7166
7167         f = &sc->tids.ftid_tab[t->idx];
7168         for (i = t->idx; i < nfilters; i++, f++) {
7169                 if (f->valid) {
7170                         t->idx = i;
7171                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
7172                         t->smtidx = f->smtidx;
7173                         if (f->fs.hitcnts)
7174                                 t->hits = get_filter_hits(sc, t->idx);
7175                         else
7176                                 t->hits = UINT64_MAX;
7177                         t->fs = f->fs;
7178
7179                         goto done;
7180                 }
7181         }
7182
7183         t->idx = 0xffffffff;
7184 done:
7185         end_synchronized_op(sc, LOCK_HELD);
7186         return (0);
7187 }
7188
7189 static int
7190 set_filter(struct adapter *sc, struct t4_filter *t)
7191 {
7192         unsigned int nfilters, nports;
7193         struct filter_entry *f;
7194         int i, rc;
7195
7196         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
7197         if (rc)
7198                 return (rc);
7199
7200         nfilters = sc->tids.nftids;
7201         nports = sc->params.nports;
7202
7203         if (nfilters == 0) {
7204                 rc = ENOTSUP;
7205                 goto done;
7206         }
7207
7208         if (!(sc->flags & FULL_INIT_DONE)) {
7209                 rc = EAGAIN;
7210                 goto done;
7211         }
7212
7213         if (t->idx >= nfilters) {
7214                 rc = EINVAL;
7215                 goto done;
7216         }
7217
7218         /* Validate against the global filter mode */
7219         if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
7220             sc->params.tp.vlan_pri_map) {
7221                 rc = E2BIG;
7222                 goto done;
7223         }
7224
7225         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
7226                 rc = EINVAL;
7227                 goto done;
7228         }
7229
7230         if (t->fs.val.iport >= nports) {
7231                 rc = EINVAL;
7232                 goto done;
7233         }
7234
7235         /* Can't specify an iq if not steering to it */
7236         if (!t->fs.dirsteer && t->fs.iq) {
7237                 rc = EINVAL;
7238                 goto done;
7239         }
7240
7241         /* IPv6 filter idx must be 4 aligned */
7242         if (t->fs.type == 1 &&
7243             ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
7244                 rc = EINVAL;
7245                 goto done;
7246         }
7247
7248         if (sc->tids.ftid_tab == NULL) {
7249                 KASSERT(sc->tids.ftids_in_use == 0,
7250                     ("%s: no memory allocated but filters_in_use > 0",
7251                     __func__));
7252
7253                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
7254                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
7255                 if (sc->tids.ftid_tab == NULL) {
7256                         rc = ENOMEM;
7257                         goto done;
7258                 }
7259                 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
7260         }
7261
7262         for (i = 0; i < 4; i++) {
7263                 f = &sc->tids.ftid_tab[t->idx + i];
7264
7265                 if (f->pending || f->valid) {
7266                         rc = EBUSY;
7267                         goto done;
7268                 }
7269                 if (f->locked) {
7270                         rc = EPERM;
7271                         goto done;
7272                 }
7273
7274                 if (t->fs.type == 0)
7275                         break;
7276         }
7277
7278         f = &sc->tids.ftid_tab[t->idx];
7279         f->fs = t->fs;
7280
7281         rc = set_filter_wr(sc, t->idx);
7282 done:
7283         end_synchronized_op(sc, 0);
7284
7285         if (rc == 0) {
7286                 mtx_lock(&sc->tids.ftid_lock);
7287                 for (;;) {
7288                         if (f->pending == 0) {
7289                                 rc = f->valid ? 0 : EIO;
7290                                 break;
7291                         }
7292
7293                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7294                             PCATCH, "t4setfw", 0)) {
7295                                 rc = EINPROGRESS;
7296                                 break;
7297                         }
7298                 }
7299                 mtx_unlock(&sc->tids.ftid_lock);
7300         }
7301         return (rc);
7302 }
7303
7304 static int
7305 del_filter(struct adapter *sc, struct t4_filter *t)
7306 {
7307         unsigned int nfilters;
7308         struct filter_entry *f;
7309         int rc;
7310
7311         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
7312         if (rc)
7313                 return (rc);
7314
7315         nfilters = sc->tids.nftids;
7316
7317         if (nfilters == 0) {
7318                 rc = ENOTSUP;
7319                 goto done;
7320         }
7321
7322         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
7323             t->idx >= nfilters) {
7324                 rc = EINVAL;
7325                 goto done;
7326         }
7327
7328         if (!(sc->flags & FULL_INIT_DONE)) {
7329                 rc = EAGAIN;
7330                 goto done;
7331         }
7332
7333         f = &sc->tids.ftid_tab[t->idx];
7334
7335         if (f->pending) {
7336                 rc = EBUSY;
7337                 goto done;
7338         }
7339         if (f->locked) {
7340                 rc = EPERM;
7341                 goto done;
7342         }
7343
7344         if (f->valid) {
7345                 t->fs = f->fs;  /* extra info for the caller */
7346                 rc = del_filter_wr(sc, t->idx);
7347         }
7348
7349 done:
7350         end_synchronized_op(sc, 0);
7351
7352         if (rc == 0) {
7353                 mtx_lock(&sc->tids.ftid_lock);
7354                 for (;;) {
7355                         if (f->pending == 0) {
7356                                 rc = f->valid ? EIO : 0;
7357                                 break;
7358                         }
7359
7360                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7361                             PCATCH, "t4delfw", 0)) {
7362                                 rc = EINPROGRESS;
7363                                 break;
7364                         }
7365                 }
7366                 mtx_unlock(&sc->tids.ftid_lock);
7367         }
7368
7369         return (rc);
7370 }
7371
7372 static void
7373 clear_filter(struct filter_entry *f)
7374 {
7375         if (f->l2t)
7376                 t4_l2t_release(f->l2t);
7377
7378         bzero(f, sizeof (*f));
7379 }
7380
7381 static int
7382 set_filter_wr(struct adapter *sc, int fidx)
7383 {
7384         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7385         struct wrqe *wr;
7386         struct fw_filter_wr *fwr;
7387         unsigned int ftid;
7388
7389         ASSERT_SYNCHRONIZED_OP(sc);
7390
7391         if (f->fs.newdmac || f->fs.newvlan) {
7392                 /* This filter needs an L2T entry; allocate one. */
7393                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
7394                 if (f->l2t == NULL)
7395                         return (EAGAIN);
7396                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
7397                     f->fs.dmac)) {
7398                         t4_l2t_release(f->l2t);
7399                         f->l2t = NULL;
7400                         return (ENOMEM);
7401                 }
7402         }
7403
7404         ftid = sc->tids.ftid_base + fidx;
7405
7406         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7407         if (wr == NULL)
7408                 return (ENOMEM);
7409
7410         fwr = wrtod(wr);
7411         bzero(fwr, sizeof (*fwr));
7412
7413         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
7414         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
7415         fwr->tid_to_iq =
7416             htobe32(V_FW_FILTER_WR_TID(ftid) |
7417                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
7418                 V_FW_FILTER_WR_NOREPLY(0) |
7419                 V_FW_FILTER_WR_IQ(f->fs.iq));
7420         fwr->del_filter_to_l2tix =
7421             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
7422                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
7423                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
7424                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
7425                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
7426                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
7427                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
7428                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
7429                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
7430                     f->fs.newvlan == VLAN_REWRITE) |
7431                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
7432                     f->fs.newvlan == VLAN_REWRITE) |
7433                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
7434                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
7435                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
7436                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
7437         fwr->ethtype = htobe16(f->fs.val.ethtype);
7438         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
7439         fwr->frag_to_ovlan_vldm =
7440             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
7441                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
7442                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
7443                 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
7444                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
7445                 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
7446         fwr->smac_sel = 0;
7447         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
7448             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
7449         fwr->maci_to_matchtypem =
7450             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
7451                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
7452                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
7453                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
7454                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
7455                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
7456                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
7457                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
7458         fwr->ptcl = f->fs.val.proto;
7459         fwr->ptclm = f->fs.mask.proto;
7460         fwr->ttyp = f->fs.val.tos;
7461         fwr->ttypm = f->fs.mask.tos;
7462         fwr->ivlan = htobe16(f->fs.val.vlan);
7463         fwr->ivlanm = htobe16(f->fs.mask.vlan);
7464         fwr->ovlan = htobe16(f->fs.val.vnic);
7465         fwr->ovlanm = htobe16(f->fs.mask.vnic);
7466         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
7467         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
7468         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
7469         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
7470         fwr->lp = htobe16(f->fs.val.dport);
7471         fwr->lpm = htobe16(f->fs.mask.dport);
7472         fwr->fp = htobe16(f->fs.val.sport);
7473         fwr->fpm = htobe16(f->fs.mask.sport);
7474         if (f->fs.newsmac)
7475                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
7476
7477         f->pending = 1;
7478         sc->tids.ftids_in_use++;
7479
7480         t4_wrq_tx(sc, wr);
7481         return (0);
7482 }
7483
7484 static int
7485 del_filter_wr(struct adapter *sc, int fidx)
7486 {
7487         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7488         struct wrqe *wr;
7489         struct fw_filter_wr *fwr;
7490         unsigned int ftid;
7491
7492         ftid = sc->tids.ftid_base + fidx;
7493
7494         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7495         if (wr == NULL)
7496                 return (ENOMEM);
7497         fwr = wrtod(wr);
7498         bzero(fwr, sizeof (*fwr));
7499
7500         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
7501
7502         f->pending = 1;
7503         t4_wrq_tx(sc, wr);
7504         return (0);
7505 }
7506
7507 int
7508 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
7509 {
7510         struct adapter *sc = iq->adapter;
7511         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
7512         unsigned int idx = GET_TID(rpl);
7513         unsigned int rc;
7514         struct filter_entry *f;
7515
7516         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
7517             rss->opcode));
7518
7519         if (is_ftid(sc, idx)) {
7520
7521                 idx -= sc->tids.ftid_base;
7522                 f = &sc->tids.ftid_tab[idx];
7523                 rc = G_COOKIE(rpl->cookie);
7524
7525                 mtx_lock(&sc->tids.ftid_lock);
7526                 if (rc == FW_FILTER_WR_FLT_ADDED) {
7527                         KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
7528                             __func__, idx));
7529                         f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
7530                         f->pending = 0;  /* asynchronous setup completed */
7531                         f->valid = 1;
7532                 } else {
7533                         if (rc != FW_FILTER_WR_FLT_DELETED) {
7534                                 /* Add or delete failed, display an error */
7535                                 log(LOG_ERR,
7536                                     "filter %u setup failed with error %u\n",
7537                                     idx, rc);
7538                         }
7539
7540                         clear_filter(f);
7541                         sc->tids.ftids_in_use--;
7542                 }
7543                 wakeup(&sc->tids.ftid_tab);
7544                 mtx_unlock(&sc->tids.ftid_lock);
7545         }
7546
7547         return (0);
7548 }
7549
7550 static int
7551 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
7552 {
7553         int rc;
7554
7555         if (cntxt->cid > M_CTXTQID)
7556                 return (EINVAL);
7557
7558         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
7559             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
7560                 return (EINVAL);
7561
7562         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
7563         if (rc)
7564                 return (rc);
7565
7566         if (sc->flags & FW_OK) {
7567                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
7568                     &cntxt->data[0]);
7569                 if (rc == 0)
7570                         goto done;
7571         }
7572
7573         /*
7574          * Read via firmware failed or wasn't even attempted.  Read directly via
7575          * the backdoor.
7576          */
7577         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
7578 done:
7579         end_synchronized_op(sc, 0);
7580         return (rc);
7581 }
7582
7583 static int
7584 load_fw(struct adapter *sc, struct t4_data *fw)
7585 {
7586         int rc;
7587         uint8_t *fw_data;
7588
7589         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
7590         if (rc)
7591                 return (rc);
7592
7593         if (sc->flags & FULL_INIT_DONE) {
7594                 rc = EBUSY;
7595                 goto done;
7596         }
7597
7598         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
7599         if (fw_data == NULL) {
7600                 rc = ENOMEM;
7601                 goto done;
7602         }
7603
7604         rc = copyin(fw->data, fw_data, fw->len);
7605         if (rc == 0)
7606                 rc = -t4_load_fw(sc, fw_data, fw->len);
7607
7608         free(fw_data, M_CXGBE);
7609 done:
7610         end_synchronized_op(sc, 0);
7611         return (rc);
7612 }
7613
7614 static int
7615 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
7616 {
7617         uint32_t addr, off, remaining, i, n;
7618         uint32_t *buf, *b;
7619         uint32_t mw_base, mw_aperture;
7620         int rc;
7621         uint8_t *dst;
7622
7623         rc = validate_mem_range(sc, mr->addr, mr->len);
7624         if (rc != 0)
7625                 return (rc);
7626
7627         memwin_info(sc, win, &mw_base, &mw_aperture);
7628         buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
7629         addr = mr->addr;
7630         remaining = mr->len;
7631         dst = (void *)mr->data;
7632
7633         while (remaining) {
7634                 off = position_memwin(sc, win, addr);
7635
7636                 /* number of bytes that we'll copy in the inner loop */
7637                 n = min(remaining, mw_aperture - off);
7638                 for (i = 0; i < n; i += 4)
7639                         *b++ = t4_read_reg(sc, mw_base + off + i);
7640
7641                 rc = copyout(buf, dst, n);
7642                 if (rc != 0)
7643                         break;
7644
7645                 b = buf;
7646                 dst += n;
7647                 remaining -= n;
7648                 addr += n;
7649         }
7650
7651         free(buf, M_CXGBE);
7652         return (rc);
7653 }
7654
7655 static int
7656 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
7657 {
7658         int rc;
7659
7660         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
7661                 return (EINVAL);
7662
7663         if (i2cd->len > sizeof(i2cd->data))
7664                 return (EFBIG);
7665
7666         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
7667         if (rc)
7668                 return (rc);
7669         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
7670             i2cd->offset, i2cd->len, &i2cd->data[0]);
7671         end_synchronized_op(sc, 0);
7672
7673         return (rc);
7674 }
7675
7676 static int
7677 in_range(int val, int lo, int hi)
7678 {
7679
7680         return (val < 0 || (val <= hi && val >= lo));
7681 }
7682
7683 static int
7684 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
7685 {
7686         int fw_subcmd, fw_type, rc;
7687
7688         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
7689         if (rc)
7690                 return (rc);
7691
7692         if (!(sc->flags & FULL_INIT_DONE)) {
7693                 rc = EAGAIN;
7694                 goto done;
7695         }
7696
7697         /*
7698          * Translate the cxgbetool parameters into T4 firmware parameters.  (The
7699          * sub-command and type are in common locations.)
7700          */
7701         if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
7702                 fw_subcmd = FW_SCHED_SC_CONFIG;
7703         else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
7704                 fw_subcmd = FW_SCHED_SC_PARAMS;
7705         else {
7706                 rc = EINVAL;
7707                 goto done;
7708         }
7709         if (p->type == SCHED_CLASS_TYPE_PACKET)
7710                 fw_type = FW_SCHED_TYPE_PKTSCHED;
7711         else {
7712                 rc = EINVAL;
7713                 goto done;
7714         }
7715
7716         if (fw_subcmd == FW_SCHED_SC_CONFIG) {
7717                 /* Vet our parameters ..*/
7718                 if (p->u.config.minmax < 0) {
7719                         rc = EINVAL;
7720                         goto done;
7721                 }
7722
7723                 /* And pass the request to the firmware ...*/
7724                 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1);
7725                 goto done;
7726         }
7727
7728         if (fw_subcmd == FW_SCHED_SC_PARAMS) {
7729                 int fw_level;
7730                 int fw_mode;
7731                 int fw_rateunit;
7732                 int fw_ratemode;
7733
7734                 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
7735                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
7736                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
7737                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
7738                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
7739                         fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
7740                 else {
7741                         rc = EINVAL;
7742                         goto done;
7743                 }
7744
7745                 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
7746                         fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
7747                 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
7748                         fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
7749                 else {
7750                         rc = EINVAL;
7751                         goto done;
7752                 }
7753
7754                 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
7755                         fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
7756                 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
7757                         fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
7758                 else {
7759                         rc = EINVAL;
7760                         goto done;
7761                 }
7762
7763                 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
7764                         fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
7765                 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
7766                         fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
7767                 else {
7768                         rc = EINVAL;
7769                         goto done;
7770                 }
7771
7772                 /* Vet our parameters ... */
7773                 if (!in_range(p->u.params.channel, 0, 3) ||
7774                     !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
7775                     !in_range(p->u.params.minrate, 0, 10000000) ||
7776                     !in_range(p->u.params.maxrate, 0, 10000000) ||
7777                     !in_range(p->u.params.weight, 0, 100)) {
7778                         rc = ERANGE;
7779                         goto done;
7780                 }
7781
7782                 /*
7783                  * Translate any unset parameters into the firmware's
7784                  * nomenclature and/or fail the call if the parameters
7785                  * are required ...
7786                  */
7787                 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
7788                     p->u.params.channel < 0 || p->u.params.cl < 0) {
7789                         rc = EINVAL;
7790                         goto done;
7791                 }
7792                 if (p->u.params.minrate < 0)
7793                         p->u.params.minrate = 0;
7794                 if (p->u.params.maxrate < 0) {
7795                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7796                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7797                                 rc = EINVAL;
7798                                 goto done;
7799                         } else
7800                                 p->u.params.maxrate = 0;
7801                 }
7802                 if (p->u.params.weight < 0) {
7803                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
7804                                 rc = EINVAL;
7805                                 goto done;
7806                         } else
7807                                 p->u.params.weight = 0;
7808                 }
7809                 if (p->u.params.pktsize < 0) {
7810                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7811                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7812                                 rc = EINVAL;
7813                                 goto done;
7814                         } else
7815                                 p->u.params.pktsize = 0;
7816                 }
7817
7818                 /* See what the firmware thinks of the request ... */
7819                 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
7820                     fw_rateunit, fw_ratemode, p->u.params.channel,
7821                     p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
7822                     p->u.params.weight, p->u.params.pktsize, 1);
7823                 goto done;
7824         }
7825
7826         rc = EINVAL;
7827 done:
7828         end_synchronized_op(sc, 0);
7829         return (rc);
7830 }
7831
7832 static int
7833 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
7834 {
7835         struct port_info *pi = NULL;
7836         struct sge_txq *txq;
7837         uint32_t fw_mnem, fw_queue, fw_class;
7838         int i, rc;
7839
7840         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
7841         if (rc)
7842                 return (rc);
7843
7844         if (!(sc->flags & FULL_INIT_DONE)) {
7845                 rc = EAGAIN;
7846                 goto done;
7847         }
7848
7849         if (p->port >= sc->params.nports) {
7850                 rc = EINVAL;
7851                 goto done;
7852         }
7853
7854         pi = sc->port[p->port];
7855         if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
7856                 rc = EINVAL;
7857                 goto done;
7858         }
7859
7860         /*
7861          * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
7862          * Scheduling Class in this case).
7863          */
7864         fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
7865             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
7866         fw_class = p->cl < 0 ? 0xffffffff : p->cl;
7867
7868         /*
7869          * If op.queue is non-negative, then we're only changing the scheduling
7870          * on a single specified TX queue.
7871          */
7872         if (p->queue >= 0) {
7873                 txq = &sc->sge.txq[pi->first_txq + p->queue];
7874                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7875                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7876                     &fw_class);
7877                 goto done;
7878         }
7879
7880         /*
7881          * Change the scheduling on all the TX queues for the
7882          * interface.
7883          */
7884         for_each_txq(pi, i, txq) {
7885                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7886                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7887                     &fw_class);
7888                 if (rc)
7889                         goto done;
7890         }
7891
7892         rc = 0;
7893 done:
7894         end_synchronized_op(sc, 0);
7895         return (rc);
7896 }
7897
7898 int
7899 t4_os_find_pci_capability(struct adapter *sc, int cap)
7900 {
7901         int i;
7902
7903         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
7904 }
7905
7906 int
7907 t4_os_pci_save_state(struct adapter *sc)
7908 {
7909         device_t dev;
7910         struct pci_devinfo *dinfo;
7911
7912         dev = sc->dev;
7913         dinfo = device_get_ivars(dev);
7914
7915         pci_cfg_save(dev, dinfo, 0);
7916         return (0);
7917 }
7918
7919 int
7920 t4_os_pci_restore_state(struct adapter *sc)
7921 {
7922         device_t dev;
7923         struct pci_devinfo *dinfo;
7924
7925         dev = sc->dev;
7926         dinfo = device_get_ivars(dev);
7927
7928         pci_cfg_restore(dev, dinfo);
7929         return (0);
7930 }
7931
7932 void
7933 t4_os_portmod_changed(const struct adapter *sc, int idx)
7934 {
7935         struct port_info *pi = sc->port[idx];
7936         static const char *mod_str[] = {
7937                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
7938         };
7939
7940         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
7941                 if_printf(pi->ifp, "transceiver unplugged.\n");
7942         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
7943                 if_printf(pi->ifp, "unknown transceiver inserted.\n");
7944         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
7945                 if_printf(pi->ifp, "unsupported transceiver inserted.\n");
7946         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
7947                 if_printf(pi->ifp, "%s transceiver inserted.\n",
7948                     mod_str[pi->mod_type]);
7949         } else {
7950                 if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
7951                     pi->mod_type);
7952         }
7953 }
7954
7955 void
7956 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
7957 {
7958         struct port_info *pi = sc->port[idx];
7959         struct ifnet *ifp = pi->ifp;
7960
7961         if (link_stat) {
7962                 pi->linkdnrc = -1;
7963                 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
7964                 if_link_state_change(ifp, LINK_STATE_UP);
7965         } else {
7966                 if (reason >= 0)
7967                         pi->linkdnrc = reason;
7968                 if_link_state_change(ifp, LINK_STATE_DOWN);
7969         }
7970 }
7971
7972 void
7973 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
7974 {
7975         struct adapter *sc;
7976
7977         sx_slock(&t4_list_lock);
7978         SLIST_FOREACH(sc, &t4_list, link) {
7979                 /*
7980                  * func should not make any assumptions about what state sc is
7981                  * in - the only guarantee is that sc->sc_lock is a valid lock.
7982                  */
7983                 func(sc, arg);
7984         }
7985         sx_sunlock(&t4_list_lock);
7986 }
7987
7988 static int
7989 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
7990 {
7991        return (0);
7992 }
7993
7994 static int
7995 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
7996 {
7997        return (0);
7998 }
7999
8000 static int
8001 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
8002     struct thread *td)
8003 {
8004         int rc;
8005         struct adapter *sc = dev->si_drv1;
8006
8007         rc = priv_check(td, PRIV_DRIVER);
8008         if (rc != 0)
8009                 return (rc);
8010
8011         switch (cmd) {
8012         case CHELSIO_T4_GETREG: {
8013                 struct t4_reg *edata = (struct t4_reg *)data;
8014
8015                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8016                         return (EFAULT);
8017
8018                 if (edata->size == 4)
8019                         edata->val = t4_read_reg(sc, edata->addr);
8020                 else if (edata->size == 8)
8021                         edata->val = t4_read_reg64(sc, edata->addr);
8022                 else
8023                         return (EINVAL);
8024
8025                 break;
8026         }
8027         case CHELSIO_T4_SETREG: {
8028                 struct t4_reg *edata = (struct t4_reg *)data;
8029
8030                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8031                         return (EFAULT);
8032
8033                 if (edata->size == 4) {
8034                         if (edata->val & 0xffffffff00000000)
8035                                 return (EINVAL);
8036                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
8037                 } else if (edata->size == 8)
8038                         t4_write_reg64(sc, edata->addr, edata->val);
8039                 else
8040                         return (EINVAL);
8041                 break;
8042         }
8043         case CHELSIO_T4_REGDUMP: {
8044                 struct t4_regdump *regs = (struct t4_regdump *)data;
8045                 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
8046                 uint8_t *buf;
8047
8048                 if (regs->len < reglen) {
8049                         regs->len = reglen; /* hint to the caller */
8050                         return (ENOBUFS);
8051                 }
8052
8053                 regs->len = reglen;
8054                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
8055                 t4_get_regs(sc, regs, buf);
8056                 rc = copyout(buf, regs->data, reglen);
8057                 free(buf, M_CXGBE);
8058                 break;
8059         }
8060         case CHELSIO_T4_GET_FILTER_MODE:
8061                 rc = get_filter_mode(sc, (uint32_t *)data);
8062                 break;
8063         case CHELSIO_T4_SET_FILTER_MODE:
8064                 rc = set_filter_mode(sc, *(uint32_t *)data);
8065                 break;
8066         case CHELSIO_T4_GET_FILTER:
8067                 rc = get_filter(sc, (struct t4_filter *)data);
8068                 break;
8069         case CHELSIO_T4_SET_FILTER:
8070                 rc = set_filter(sc, (struct t4_filter *)data);
8071                 break;
8072         case CHELSIO_T4_DEL_FILTER:
8073                 rc = del_filter(sc, (struct t4_filter *)data);
8074                 break;
8075         case CHELSIO_T4_GET_SGE_CONTEXT:
8076                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
8077                 break;
8078         case CHELSIO_T4_LOAD_FW:
8079                 rc = load_fw(sc, (struct t4_data *)data);
8080                 break;
8081         case CHELSIO_T4_GET_MEM:
8082                 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
8083                 break;
8084         case CHELSIO_T4_GET_I2C:
8085                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
8086                 break;
8087         case CHELSIO_T4_CLEAR_STATS: {
8088                 int i;
8089                 u_int port_id = *(uint32_t *)data;
8090                 struct port_info *pi;
8091
8092                 if (port_id >= sc->params.nports)
8093                         return (EINVAL);
8094                 pi = sc->port[port_id];
8095
8096                 /* MAC stats */
8097                 t4_clr_port_stats(sc, pi->tx_chan);
8098
8099                 if (pi->flags & PORT_INIT_DONE) {
8100                         struct sge_rxq *rxq;
8101                         struct sge_txq *txq;
8102                         struct sge_wrq *wrq;
8103
8104                         for_each_rxq(pi, i, rxq) {
8105 #if defined(INET) || defined(INET6)
8106                                 rxq->lro.lro_queued = 0;
8107                                 rxq->lro.lro_flushed = 0;
8108 #endif
8109                                 rxq->rxcsum = 0;
8110                                 rxq->vlan_extraction = 0;
8111                         }
8112
8113                         for_each_txq(pi, i, txq) {
8114                                 txq->txcsum = 0;
8115                                 txq->tso_wrs = 0;
8116                                 txq->vlan_insertion = 0;
8117                                 txq->imm_wrs = 0;
8118                                 txq->sgl_wrs = 0;
8119                                 txq->txpkt_wrs = 0;
8120                                 txq->txpkts_wrs = 0;
8121                                 txq->txpkts_pkts = 0;
8122                                 txq->br->br_drops = 0;
8123                                 txq->no_dmamap = 0;
8124                                 txq->no_desc = 0;
8125                         }
8126
8127 #ifdef TCP_OFFLOAD
8128                         /* nothing to clear for each ofld_rxq */
8129
8130                         for_each_ofld_txq(pi, i, wrq) {
8131                                 wrq->tx_wrs = 0;
8132                                 wrq->no_desc = 0;
8133                         }
8134 #endif
8135                         wrq = &sc->sge.ctrlq[pi->port_id];
8136                         wrq->tx_wrs = 0;
8137                         wrq->no_desc = 0;
8138                 }
8139                 break;
8140         }
8141         case CHELSIO_T4_SCHED_CLASS:
8142                 rc = set_sched_class(sc, (struct t4_sched_params *)data);
8143                 break;
8144         case CHELSIO_T4_SCHED_QUEUE:
8145                 rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
8146                 break;
8147         case CHELSIO_T4_GET_TRACER:
8148                 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
8149                 break;
8150         case CHELSIO_T4_SET_TRACER:
8151                 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
8152                 break;
8153         default:
8154                 rc = EINVAL;
8155         }
8156
8157         return (rc);
8158 }
8159
8160 #ifdef TCP_OFFLOAD
8161 void
8162 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask,
8163     const unsigned int *pgsz_order)
8164 {
8165         struct port_info *pi = ifp->if_softc;
8166         struct adapter *sc = pi->adapter;
8167
8168         t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
8169         t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |
8170                 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) |
8171                 V_HPZ3(pgsz_order[3]));
8172 }
8173
8174 static int
8175 toe_capability(struct port_info *pi, int enable)
8176 {
8177         int rc;
8178         struct adapter *sc = pi->adapter;
8179
8180         ASSERT_SYNCHRONIZED_OP(sc);
8181
8182         if (!is_offload(sc))
8183                 return (ENODEV);
8184
8185         if (enable) {
8186                 if (!(sc->flags & FULL_INIT_DONE)) {
8187                         rc = cxgbe_init_synchronized(pi);
8188                         if (rc)
8189                                 return (rc);
8190                 }
8191
8192                 if (isset(&sc->offload_map, pi->port_id))
8193                         return (0);
8194
8195                 if (!(sc->flags & TOM_INIT_DONE)) {
8196                         rc = t4_activate_uld(sc, ULD_TOM);
8197                         if (rc == EAGAIN) {
8198                                 log(LOG_WARNING,
8199                                     "You must kldload t4_tom.ko before trying "
8200                                     "to enable TOE on a cxgbe interface.\n");
8201                         }
8202                         if (rc != 0)
8203                                 return (rc);
8204                         KASSERT(sc->tom_softc != NULL,
8205                             ("%s: TOM activated but softc NULL", __func__));
8206                         KASSERT(sc->flags & TOM_INIT_DONE,
8207                             ("%s: TOM activated but flag not set", __func__));
8208                 }
8209
8210                 setbit(&sc->offload_map, pi->port_id);
8211         } else {
8212                 if (!isset(&sc->offload_map, pi->port_id))
8213                         return (0);
8214
8215                 KASSERT(sc->flags & TOM_INIT_DONE,
8216                     ("%s: TOM never initialized?", __func__));
8217                 clrbit(&sc->offload_map, pi->port_id);
8218         }
8219
8220         return (0);
8221 }
8222
8223 /*
8224  * Add an upper layer driver to the global list.
8225  */
8226 int
8227 t4_register_uld(struct uld_info *ui)
8228 {
8229         int rc = 0;
8230         struct uld_info *u;
8231
8232         sx_xlock(&t4_uld_list_lock);
8233         SLIST_FOREACH(u, &t4_uld_list, link) {
8234             if (u->uld_id == ui->uld_id) {
8235                     rc = EEXIST;
8236                     goto done;
8237             }
8238         }
8239
8240         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
8241         ui->refcount = 0;
8242 done:
8243         sx_xunlock(&t4_uld_list_lock);
8244         return (rc);
8245 }
8246
8247 int
8248 t4_unregister_uld(struct uld_info *ui)
8249 {
8250         int rc = EINVAL;
8251         struct uld_info *u;
8252
8253         sx_xlock(&t4_uld_list_lock);
8254
8255         SLIST_FOREACH(u, &t4_uld_list, link) {
8256             if (u == ui) {
8257                     if (ui->refcount > 0) {
8258                             rc = EBUSY;
8259                             goto done;
8260                     }
8261
8262                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
8263                     rc = 0;
8264                     goto done;
8265             }
8266         }
8267 done:
8268         sx_xunlock(&t4_uld_list_lock);
8269         return (rc);
8270 }
8271
8272 int
8273 t4_activate_uld(struct adapter *sc, int id)
8274 {
8275         int rc = EAGAIN;
8276         struct uld_info *ui;
8277
8278         ASSERT_SYNCHRONIZED_OP(sc);
8279
8280         sx_slock(&t4_uld_list_lock);
8281
8282         SLIST_FOREACH(ui, &t4_uld_list, link) {
8283                 if (ui->uld_id == id) {
8284                         rc = ui->activate(sc);
8285                         if (rc == 0)
8286                                 ui->refcount++;
8287                         goto done;
8288                 }
8289         }
8290 done:
8291         sx_sunlock(&t4_uld_list_lock);
8292
8293         return (rc);
8294 }
8295
8296 int
8297 t4_deactivate_uld(struct adapter *sc, int id)
8298 {
8299         int rc = EINVAL;
8300         struct uld_info *ui;
8301
8302         ASSERT_SYNCHRONIZED_OP(sc);
8303
8304         sx_slock(&t4_uld_list_lock);
8305
8306         SLIST_FOREACH(ui, &t4_uld_list, link) {
8307                 if (ui->uld_id == id) {
8308                         rc = ui->deactivate(sc);
8309                         if (rc == 0)
8310                                 ui->refcount--;
8311                         goto done;
8312                 }
8313         }
8314 done:
8315         sx_sunlock(&t4_uld_list_lock);
8316
8317         return (rc);
8318 }
8319 #endif
8320
8321 /*
8322  * Come up with reasonable defaults for some of the tunables, provided they're
8323  * not set by the user (in which case we'll use the values as is).
8324  */
8325 static void
8326 tweak_tunables(void)
8327 {
8328         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
8329
8330         if (t4_ntxq10g < 1)
8331                 t4_ntxq10g = min(nc, NTXQ_10G);
8332
8333         if (t4_ntxq1g < 1)
8334                 t4_ntxq1g = min(nc, NTXQ_1G);
8335
8336         if (t4_nrxq10g < 1)
8337                 t4_nrxq10g = min(nc, NRXQ_10G);
8338
8339         if (t4_nrxq1g < 1)
8340                 t4_nrxq1g = min(nc, NRXQ_1G);
8341
8342 #ifdef TCP_OFFLOAD
8343         if (t4_nofldtxq10g < 1)
8344                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
8345
8346         if (t4_nofldtxq1g < 1)
8347                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
8348
8349         if (t4_nofldrxq10g < 1)
8350                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
8351
8352         if (t4_nofldrxq1g < 1)
8353                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
8354
8355         if (t4_toecaps_allowed == -1)
8356                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
8357 #else
8358         if (t4_toecaps_allowed == -1)
8359                 t4_toecaps_allowed = 0;
8360 #endif
8361
8362 #ifdef DEV_NETMAP
8363         if (t4_nnmtxq10g < 1)
8364                 t4_nnmtxq10g = min(nc, NNMTXQ_10G);
8365
8366         if (t4_nnmtxq1g < 1)
8367                 t4_nnmtxq1g = min(nc, NNMTXQ_1G);
8368
8369         if (t4_nnmrxq10g < 1)
8370                 t4_nnmrxq10g = min(nc, NNMRXQ_10G);
8371
8372         if (t4_nnmrxq1g < 1)
8373                 t4_nnmrxq1g = min(nc, NNMRXQ_1G);
8374 #endif
8375
8376         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
8377                 t4_tmr_idx_10g = TMR_IDX_10G;
8378
8379         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
8380                 t4_pktc_idx_10g = PKTC_IDX_10G;
8381
8382         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
8383                 t4_tmr_idx_1g = TMR_IDX_1G;
8384
8385         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
8386                 t4_pktc_idx_1g = PKTC_IDX_1G;
8387
8388         if (t4_qsize_txq < 128)
8389                 t4_qsize_txq = 128;
8390
8391         if (t4_qsize_rxq < 128)
8392                 t4_qsize_rxq = 128;
8393         while (t4_qsize_rxq & 7)
8394                 t4_qsize_rxq++;
8395
8396         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
8397 }
8398
8399 static struct sx mlu;   /* mod load unload */
8400 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
8401
8402 static int
8403 mod_event(module_t mod, int cmd, void *arg)
8404 {
8405         int rc = 0;
8406         static int loaded = 0;
8407
8408         switch (cmd) {
8409         case MOD_LOAD:
8410                 sx_xlock(&mlu);
8411                 if (loaded++ == 0) {
8412                         t4_sge_modload();
8413                         sx_init(&t4_list_lock, "T4/T5 adapters");
8414                         SLIST_INIT(&t4_list);
8415 #ifdef TCP_OFFLOAD
8416                         sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
8417                         SLIST_INIT(&t4_uld_list);
8418 #endif
8419                         t4_tracer_modload();
8420                         tweak_tunables();
8421                 }
8422                 sx_xunlock(&mlu);
8423                 break;
8424
8425         case MOD_UNLOAD:
8426                 sx_xlock(&mlu);
8427                 if (--loaded == 0) {
8428                         int tries;
8429
8430                         sx_slock(&t4_list_lock);
8431                         if (!SLIST_EMPTY(&t4_list)) {
8432                                 rc = EBUSY;
8433                                 sx_sunlock(&t4_list_lock);
8434                                 goto done_unload;
8435                         }
8436 #ifdef TCP_OFFLOAD
8437                         sx_slock(&t4_uld_list_lock);
8438                         if (!SLIST_EMPTY(&t4_uld_list)) {
8439                                 rc = EBUSY;
8440                                 sx_sunlock(&t4_uld_list_lock);
8441                                 sx_sunlock(&t4_list_lock);
8442                                 goto done_unload;
8443                         }
8444 #endif
8445                         tries = 0;
8446                         while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
8447                                 uprintf("%ju clusters with custom free routine "
8448                                     "still is use.\n", t4_sge_extfree_refs());
8449                                 pause("t4unload", 2 * hz);
8450                         }
8451 #ifdef TCP_OFFLOAD
8452                         sx_sunlock(&t4_uld_list_lock);
8453 #endif
8454                         sx_sunlock(&t4_list_lock);
8455
8456                         if (t4_sge_extfree_refs() == 0) {
8457                                 t4_tracer_modunload();
8458 #ifdef TCP_OFFLOAD
8459                                 sx_destroy(&t4_uld_list_lock);
8460 #endif
8461                                 sx_destroy(&t4_list_lock);
8462                                 t4_sge_modunload();
8463                                 loaded = 0;
8464                         } else {
8465                                 rc = EBUSY;
8466                                 loaded++;       /* undo earlier decrement */
8467                         }
8468                 }
8469 done_unload:
8470                 sx_xunlock(&mlu);
8471                 break;
8472         }
8473
8474         return (rc);
8475 }
8476
8477 static devclass_t t4_devclass, t5_devclass;
8478 static devclass_t cxgbe_devclass, cxl_devclass;
8479
8480 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
8481 MODULE_VERSION(t4nex, 1);
8482 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
8483
8484 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
8485 MODULE_VERSION(t5nex, 1);
8486 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
8487
8488 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
8489 MODULE_VERSION(cxgbe, 1);
8490
8491 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
8492 MODULE_VERSION(cxl, 1);