sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/conf.h>
  36 #include <sys/priv.h>
  37 #include <sys/kernel.h>
  38 #include <sys/bus.h>
  39 #include <sys/module.h>
  40 #include <sys/malloc.h>
  41 #include <sys/queue.h>
  42 #include <sys/taskqueue.h>
  43 #include <sys/pciio.h>
  44 #include <dev/pci/pcireg.h>
  45 #include <dev/pci/pcivar.h>
  46 #include <dev/pci/pci_private.h>
  47 #include <sys/firmware.h>
  48 #include <sys/sbuf.h>
  49 #include <sys/smp.h>
  50 #include <sys/socket.h>
  51 #include <sys/sockio.h>
  52 #include <sys/sysctl.h>
  53 #include <net/ethernet.h>
  54 #include <net/if.h>
  55 #include <net/if_types.h>
  56 #include <net/if_dl.h>
  57 #include <net/if_vlan_var.h>
  58 #if defined(__i386__) || defined(__amd64__)
  59 #include <vm/vm.h>
  60 #include <vm/pmap.h>
  61 #endif
  62
  63 #include "common/common.h"
  64 #include "common/t4_msg.h"
  65 #include "common/t4_regs.h"
  66 #include "common/t4_regs_values.h"
  67 #include "t4_ioctl.h"
  68 #include "t4_l2t.h"
  69
  70 /* T4 bus driver interface */
  71 static int t4_probe(device_t);
  72 static int t4_attach(device_t);
  73 static int t4_detach(device_t);
  74 static device_method_t t4_methods[] = {
  75         DEVMETHOD(device_probe,         t4_probe),
  76         DEVMETHOD(device_attach,        t4_attach),
  77         DEVMETHOD(device_detach,        t4_detach),
  78
  79         DEVMETHOD_END
  80 };
  81 static driver_t t4_driver = {
  82         "t4nex",
  83         t4_methods,
  84         sizeof(struct adapter)
  85 };
  86
  87
  88 /* T4 port (cxgbe) interface */
  89 static int cxgbe_probe(device_t);
  90 static int cxgbe_attach(device_t);
  91 static int cxgbe_detach(device_t);
  92 static device_method_t cxgbe_methods[] = {
  93         DEVMETHOD(device_probe,         cxgbe_probe),
  94         DEVMETHOD(device_attach,        cxgbe_attach),
  95         DEVMETHOD(device_detach,        cxgbe_detach),
  96         { 0, 0 }
  97 };
  98 static driver_t cxgbe_driver = {
  99         "cxgbe",
 100         cxgbe_methods,
 101         sizeof(struct port_info)
 102 };
 103
 104 static d_ioctl_t t4_ioctl;
 105 static d_open_t t4_open;
 106 static d_close_t t4_close;
 107
 108 static struct cdevsw t4_cdevsw = {
 109        .d_version = D_VERSION,
 110        .d_flags = 0,
 111        .d_open = t4_open,
 112        .d_close = t4_close,
 113        .d_ioctl = t4_ioctl,
 114        .d_name = "t4nex",
 115 };
 116
 117 /* T5 bus driver interface */
 118 static int t5_probe(device_t);
 119 static device_method_t t5_methods[] = {
 120         DEVMETHOD(device_probe,         t5_probe),
 121         DEVMETHOD(device_attach,        t4_attach),
 122         DEVMETHOD(device_detach,        t4_detach),
 123
 124         DEVMETHOD_END
 125 };
 126 static driver_t t5_driver = {
 127         "t5nex",
 128         t5_methods,
 129         sizeof(struct adapter)
 130 };
 131
 132
 133 /* T5 port (cxl) interface */
 134 static driver_t cxl_driver = {
 135         "cxl",
 136         cxgbe_methods,
 137         sizeof(struct port_info)
 138 };
 139
 140 static struct cdevsw t5_cdevsw = {
 141        .d_version = D_VERSION,
 142        .d_flags = 0,
 143        .d_open = t4_open,
 144        .d_close = t4_close,
 145        .d_ioctl = t4_ioctl,
 146        .d_name = "t5nex",
 147 };
 148
 149 /* ifnet + media interface */
 150 static void cxgbe_init(void *);
 151 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 152 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 153 static void cxgbe_qflush(struct ifnet *);
 154 static int cxgbe_media_change(struct ifnet *);
 155 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 156
 157 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
 158
 159 /*
 160  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 161  * then ADAPTER_LOCK, then t4_uld_list_lock.
 162  */
 163 static struct sx t4_list_lock;
 164 SLIST_HEAD(, adapter) t4_list;
 165 #ifdef TCP_OFFLOAD
 166 static struct sx t4_uld_list_lock;
 167 SLIST_HEAD(, uld_info) t4_uld_list;
 168 #endif
 169
 170 /*
 171  * Tunables.  See tweak_tunables() too.
 172  *
 173  * Each tunable is set to a default value here if it's known at compile-time.
 174  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
 175  * provide a reasonable default when the driver is loaded.
 176  *
 177  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
 178  * T5 are under hw.cxl.
 179  */
 180
 181 /*
 182  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 183  */
 184 #define NTXQ_10G 16
 185 static int t4_ntxq10g = -1;
 186 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 187
 188 #define NRXQ_10G 8
 189 static int t4_nrxq10g = -1;
 190 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 191
 192 #define NTXQ_1G 4
 193 static int t4_ntxq1g = -1;
 194 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 195
 196 #define NRXQ_1G 2
 197 static int t4_nrxq1g = -1;
 198 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 199
 200 static int t4_rsrv_noflowq = 0;
 201 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
 202
 203 #ifdef TCP_OFFLOAD
 204 #define NOFLDTXQ_10G 8
 205 static int t4_nofldtxq10g = -1;
 206 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 207
 208 #define NOFLDRXQ_10G 2
 209 static int t4_nofldrxq10g = -1;
 210 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 211
 212 #define NOFLDTXQ_1G 2
 213 static int t4_nofldtxq1g = -1;
 214 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 215
 216 #define NOFLDRXQ_1G 1
 217 static int t4_nofldrxq1g = -1;
 218 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 219 #endif
 220
 221 #ifdef DEV_NETMAP
 222 #define NNMTXQ_10G 2
 223 static int t4_nnmtxq10g = -1;
 224 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g);
 225
 226 #define NNMRXQ_10G 2
 227 static int t4_nnmrxq10g = -1;
 228 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g);
 229
 230 #define NNMTXQ_1G 1
 231 static int t4_nnmtxq1g = -1;
 232 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g);
 233
 234 #define NNMRXQ_1G 1
 235 static int t4_nnmrxq1g = -1;
 236 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g);
 237 #endif
 238
 239 /*
 240  * Holdoff parameters for 10G and 1G ports.
 241  */
 242 #define TMR_IDX_10G 1
 243 static int t4_tmr_idx_10g = TMR_IDX_10G;
 244 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 245
 246 #define PKTC_IDX_10G (-1)
 247 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 248 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 249
 250 #define TMR_IDX_1G 1
 251 static int t4_tmr_idx_1g = TMR_IDX_1G;
 252 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 253
 254 #define PKTC_IDX_1G (-1)
 255 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 256 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 257
 258 /*
 259  * Size (# of entries) of each tx and rx queue.
 260  */
 261 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 262 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 263
 264 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 265 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 266
 267 /*
 268  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 269  */
 270 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 271 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 272
 273 /*
 274  * Configuration file.
 275  */
 276 #define DEFAULT_CF      "default"
 277 #define FLASH_CF        "flash"
 278 #define UWIRE_CF        "uwire"
 279 #define FPGA_CF         "fpga"
 280 static char t4_cfg_file[32] = DEFAULT_CF;
 281 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 282
 283 /*
 284  * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively).
 285  * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
 286  * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
 287  *            mark or when signalled to do so, 0 to never emit PAUSE.
 288  */
 289 static int t4_pause_settings = PAUSE_TX | PAUSE_RX;
 290 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings);
 291
 292 /*
 293  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
 294  * encouraged respectively).
 295  */
 296 static unsigned int t4_fw_install = 1;
 297 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 298
 299 /*
 300  * ASIC features that will be used.  Disable the ones you don't want so that the
 301  * chip resources aren't wasted on features that will not be used.
 302  */
 303 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 304 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 305
 306 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 307 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 308
 309 static int t4_toecaps_allowed = -1;
 310 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 311
 312 static int t4_rdmacaps_allowed = 0;
 313 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 314
 315 static int t4_iscsicaps_allowed = 0;
 316 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 317
 318 static int t4_fcoecaps_allowed = 0;
 319 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 320
 321 static int t5_write_combine = 0;
 322 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 323
 324 struct intrs_and_queues {
 325         uint16_t intr_type;     /* INTx, MSI, or MSI-X */
 326         uint16_t nirq;          /* Total # of vectors */
 327         uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
 328         uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */
 329         uint16_t ntxq10g;       /* # of NIC txq's for each 10G port */
 330         uint16_t nrxq10g;       /* # of NIC rxq's for each 10G port */
 331         uint16_t ntxq1g;        /* # of NIC txq's for each 1G port */
 332         uint16_t nrxq1g;        /* # of NIC rxq's for each 1G port */
 333         uint16_t rsrv_noflowq;  /* Flag whether to reserve queue 0 */
 334 #ifdef TCP_OFFLOAD
 335         uint16_t nofldtxq10g;   /* # of TOE txq's for each 10G port */
 336         uint16_t nofldrxq10g;   /* # of TOE rxq's for each 10G port */
 337         uint16_t nofldtxq1g;    /* # of TOE txq's for each 1G port */
 338         uint16_t nofldrxq1g;    /* # of TOE rxq's for each 1G port */
 339 #endif
 340 #ifdef DEV_NETMAP
 341         uint16_t nnmtxq10g;     /* # of netmap txq's for each 10G port */
 342         uint16_t nnmrxq10g;     /* # of netmap rxq's for each 10G port */
 343         uint16_t nnmtxq1g;      /* # of netmap txq's for each 1G port */
 344         uint16_t nnmrxq1g;      /* # of netmap rxq's for each 1G port */
 345 #endif
 346 };
 347
 348 struct filter_entry {
 349         uint32_t valid:1;       /* filter allocated and valid */
 350         uint32_t locked:1;      /* filter is administratively locked */
 351         uint32_t pending:1;     /* filter action is pending firmware reply */
 352         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 353         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 354
 355         struct t4_filter_specification fs;
 356 };
 357
 358 static int map_bars_0_and_4(struct adapter *);
 359 static int map_bar_2(struct adapter *);
 360 static void setup_memwin(struct adapter *);
 361 static int validate_mem_range(struct adapter *, uint32_t, int);
 362 static int fwmtype_to_hwmtype(int);
 363 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
 364     uint32_t *);
 365 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
 366 static uint32_t position_memwin(struct adapter *, int, uint32_t);
 367 static int cfg_itype_and_nqueues(struct adapter *, int, int,
 368     struct intrs_and_queues *);
 369 static int prep_firmware(struct adapter *);
 370 static int partition_resources(struct adapter *, const struct firmware *,
 371     const char *);
 372 static int get_params__pre_init(struct adapter *);
 373 static int get_params__post_init(struct adapter *);
 374 static int set_params__post_init(struct adapter *);
 375 static void t4_set_desc(struct adapter *);
 376 static void build_medialist(struct port_info *, struct ifmedia *);
 377 static int cxgbe_init_synchronized(struct port_info *);
 378 static int cxgbe_uninit_synchronized(struct port_info *);
 379 static int setup_intr_handlers(struct adapter *);
 380 static void quiesce_eq(struct adapter *, struct sge_eq *);
 381 static void quiesce_iq(struct adapter *, struct sge_iq *);
 382 static void quiesce_fl(struct adapter *, struct sge_fl *);
 383 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 384     driver_intr_t *, void *, char *);
 385 static int t4_free_irq(struct adapter *, struct irq *);
 386 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
 387     unsigned int);
 388 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 389 static void cxgbe_refresh_stats(struct adapter *, struct port_info *);
 390 static void cxgbe_tick(void *);
 391 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 392 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
 393     struct mbuf *);
 394 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 395 static int fw_msg_not_handled(struct adapter *, const __be64 *);
 396 static int t4_sysctls(struct adapter *);
 397 static int cxgbe_sysctls(struct port_info *);
 398 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 399 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 400 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
 401 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 402 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 403 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 404 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 405 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 406 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
 407 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 408 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 409 #ifdef SBUF_DRAIN
 410 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 411 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 412 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 413 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
 414 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
 415 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 416 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 417 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 418 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 419 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 420 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 421 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 422 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
 423 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 424 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
 425 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 426 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 427 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 428 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 429 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 430 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 431 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
 432 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 433 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 434 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 435 #endif
 436 static inline void txq_start(struct ifnet *, struct sge_txq *);
 437 static uint32_t fconf_to_mode(uint32_t);
 438 static uint32_t mode_to_fconf(uint32_t);
 439 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 440 static int get_filter_mode(struct adapter *, uint32_t *);
 441 static int set_filter_mode(struct adapter *, uint32_t);
 442 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 443 static int get_filter(struct adapter *, struct t4_filter *);
 444 static int set_filter(struct adapter *, struct t4_filter *);
 445 static int del_filter(struct adapter *, struct t4_filter *);
 446 static void clear_filter(struct filter_entry *);
 447 static int set_filter_wr(struct adapter *, int);
 448 static int del_filter_wr(struct adapter *, int);
 449 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 450 static int load_fw(struct adapter *, struct t4_data *);
 451 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 452 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 453 static int set_sched_class(struct adapter *, struct t4_sched_params *);
 454 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
 455 #ifdef TCP_OFFLOAD
 456 static int toe_capability(struct port_info *, int);
 457 #endif
 458 static int mod_event(module_t, int, void *);
 459
 460 struct {
 461         uint16_t device;
 462         char *desc;
 463 } t4_pciids[] = {
 464         {0xa000, "Chelsio Terminator 4 FPGA"},
 465         {0x4400, "Chelsio T440-dbg"},
 466         {0x4401, "Chelsio T420-CR"},
 467         {0x4402, "Chelsio T422-CR"},
 468         {0x4403, "Chelsio T440-CR"},
 469         {0x4404, "Chelsio T420-BCH"},
 470         {0x4405, "Chelsio T440-BCH"},
 471         {0x4406, "Chelsio T440-CH"},
 472         {0x4407, "Chelsio T420-SO"},
 473         {0x4408, "Chelsio T420-CX"},
 474         {0x4409, "Chelsio T420-BT"},
 475         {0x440a, "Chelsio T404-BT"},
 476         {0x440e, "Chelsio T440-LP-CR"},
 477 }, t5_pciids[] = {
 478         {0xb000, "Chelsio Terminator 5 FPGA"},
 479         {0x5400, "Chelsio T580-dbg"},
 480         {0x5401,  "Chelsio T520-CR"},           /* 2 x 10G */
 481         {0x5402,  "Chelsio T522-CR"},           /* 2 x 10G, 2 X 1G */
 482         {0x5403,  "Chelsio T540-CR"},           /* 4 x 10G */
 483         {0x5407,  "Chelsio T520-SO"},           /* 2 x 10G, nomem */
 484         {0x5409,  "Chelsio T520-BT"},           /* 2 x 10GBaseT */
 485         {0x540a,  "Chelsio T504-BT"},           /* 4 x 1G */
 486         {0x540d,  "Chelsio T580-CR"},           /* 2 x 40G */
 487         {0x540e,  "Chelsio T540-LP-CR"},        /* 4 x 10G */
 488         {0x5410,  "Chelsio T580-LP-CR"},        /* 2 x 40G */
 489         {0x5411,  "Chelsio T520-LL-CR"},        /* 2 x 10G */
 490         {0x5412,  "Chelsio T560-CR"},           /* 1 x 40G, 2 x 10G */
 491         {0x5414,  "Chelsio T580-LP-SO-CR"},     /* 2 x 40G, nomem */
 492         {0x5415,  "Chelsio T502-BT"},           /* 2 x 1G */
 493 #ifdef notyet
 494         {0x5404,  "Chelsio T520-BCH"},
 495         {0x5405,  "Chelsio T540-BCH"},
 496         {0x5406,  "Chelsio T540-CH"},
 497         {0x5408,  "Chelsio T520-CX"},
 498         {0x540b,  "Chelsio B520-SR"},
 499         {0x540c,  "Chelsio B504-BT"},
 500         {0x540f,  "Chelsio Amsterdam"},
 501         {0x5413,  "Chelsio T580-CHR"},
 502 #endif
 503 };
 504
 505 #ifdef TCP_OFFLOAD
 506 /*
 507  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 508  * exactly the same for both rxq and ofld_rxq.
 509  */
 510 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 511 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 512 #endif
 513
 514 /* No easy way to include t4_msg.h before adapter.h so we check this way */
 515 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
 516 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
 517
 518 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
 519
 520 static int
 521 t4_probe(device_t dev)
 522 {
 523         int i;
 524         uint16_t v = pci_get_vendor(dev);
 525         uint16_t d = pci_get_device(dev);
 526         uint8_t f = pci_get_function(dev);
 527
 528         if (v != PCI_VENDOR_ID_CHELSIO)
 529                 return (ENXIO);
 530
 531         /* Attach only to PF0 of the FPGA */
 532         if (d == 0xa000 && f != 0)
 533                 return (ENXIO);
 534
 535         for (i = 0; i < nitems(t4_pciids); i++) {
 536                 if (d == t4_pciids[i].device) {
 537                         device_set_desc(dev, t4_pciids[i].desc);
 538                         return (BUS_PROBE_DEFAULT);
 539                 }
 540         }
 541
 542         return (ENXIO);
 543 }
 544
 545 static int
 546 t5_probe(device_t dev)
 547 {
 548         int i;
 549         uint16_t v = pci_get_vendor(dev);
 550         uint16_t d = pci_get_device(dev);
 551         uint8_t f = pci_get_function(dev);
 552
 553         if (v != PCI_VENDOR_ID_CHELSIO)
 554                 return (ENXIO);
 555
 556         /* Attach only to PF0 of the FPGA */
 557         if (d == 0xb000 && f != 0)
 558                 return (ENXIO);
 559
 560         for (i = 0; i < nitems(t5_pciids); i++) {
 561                 if (d == t5_pciids[i].device) {
 562                         device_set_desc(dev, t5_pciids[i].desc);
 563                         return (BUS_PROBE_DEFAULT);
 564                 }
 565         }
 566
 567         return (ENXIO);
 568 }
 569
 570 static int
 571 t4_attach(device_t dev)
 572 {
 573         struct adapter *sc;
 574         int rc = 0, i, n10g, n1g, rqidx, tqidx;
 575         struct intrs_and_queues iaq;
 576         struct sge *s;
 577 #ifdef TCP_OFFLOAD
 578         int ofld_rqidx, ofld_tqidx;
 579 #endif
 580 #ifdef DEV_NETMAP
 581         int nm_rqidx, nm_tqidx;
 582 #endif
 583
 584         sc = device_get_softc(dev);
 585         sc->dev = dev;
 586
 587         pci_enable_busmaster(dev);
 588         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 589                 uint32_t v;
 590
 591                 pci_set_max_read_req(dev, 4096);
 592                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 593                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 594                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 595
 596                 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
 597         }
 598
 599         sc->traceq = -1;
 600         mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
 601         snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
 602             device_get_nameunit(dev));
 603
 604         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 605             device_get_nameunit(dev));
 606         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 607         sx_xlock(&t4_list_lock);
 608         SLIST_INSERT_HEAD(&t4_list, sc, link);
 609         sx_xunlock(&t4_list_lock);
 610
 611         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 612         TAILQ_INIT(&sc->sfl);
 613         callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
 614
 615         mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF);
 616
 617         rc = map_bars_0_and_4(sc);
 618         if (rc != 0)
 619                 goto done; /* error message displayed already */
 620
 621         /*
 622          * This is the real PF# to which we're attaching.  Works from within PCI
 623          * passthrough environments too, where pci_get_function() could return a
 624          * different PF# depending on the passthrough configuration.  We need to
 625          * use the real PF# in all our communication with the firmware.
 626          */
 627         sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 628         sc->mbox = sc->pf;
 629
 630         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 631         sc->an_handler = an_not_handled;
 632         for (i = 0; i < nitems(sc->cpl_handler); i++)
 633                 sc->cpl_handler[i] = cpl_not_handled;
 634         for (i = 0; i < nitems(sc->fw_msg_handler); i++)
 635                 sc->fw_msg_handler[i] = fw_msg_not_handled;
 636         t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 637         t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
 638         t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt);
 639         t4_init_sge_cpl_handlers(sc);
 640
 641         /* Prepare the adapter for operation */
 642         rc = -t4_prep_adapter(sc);
 643         if (rc != 0) {
 644                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 645                 goto done;
 646         }
 647
 648         /*
 649          * Do this really early, with the memory windows set up even before the
 650          * character device.  The userland tool's register i/o and mem read
 651          * will work even in "recovery mode".
 652          */
 653         setup_memwin(sc);
 654         sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
 655             device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
 656             device_get_nameunit(dev));
 657         if (sc->cdev == NULL)
 658                 device_printf(dev, "failed to create nexus char device.\n");
 659         else
 660                 sc->cdev->si_drv1 = sc;
 661
 662         /* Go no further if recovery mode has been requested. */
 663         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 664                 device_printf(dev, "recovery mode.\n");
 665                 goto done;
 666         }
 667
 668         /* Prepare the firmware for operation */
 669         rc = prep_firmware(sc);
 670         if (rc != 0)
 671                 goto done; /* error message displayed already */
 672
 673         rc = get_params__post_init(sc);
 674         if (rc != 0)
 675                 goto done; /* error message displayed already */
 676
 677         rc = set_params__post_init(sc);
 678         if (rc != 0)
 679                 goto done; /* error message displayed already */
 680
 681         rc = map_bar_2(sc);
 682         if (rc != 0)
 683                 goto done; /* error message displayed already */
 684
 685         rc = t4_create_dma_tag(sc);
 686         if (rc != 0)
 687                 goto done; /* error message displayed already */
 688
 689         /*
 690          * First pass over all the ports - allocate VIs and initialize some
 691          * basic parameters like mac address, port type, etc.  We also figure
 692          * out whether a port is 10G or 1G and use that information when
 693          * calculating how many interrupts to attempt to allocate.
 694          */
 695         n10g = n1g = 0;
 696         for_each_port(sc, i) {
 697                 struct port_info *pi;
 698
 699                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 700                 sc->port[i] = pi;
 701
 702                 /* These must be set before t4_port_init */
 703                 pi->adapter = sc;
 704                 pi->port_id = i;
 705
 706                 /* Allocate the vi and initialize parameters like mac addr */
 707                 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 708                 if (rc != 0) {
 709                         device_printf(dev, "unable to initialize port %d: %d\n",
 710                             i, rc);
 711                         free(pi, M_CXGBE);
 712                         sc->port[i] = NULL;
 713                         goto done;
 714                 }
 715
 716                 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX);
 717                 pi->link_cfg.requested_fc |= t4_pause_settings;
 718                 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX);
 719                 pi->link_cfg.fc |= t4_pause_settings;
 720
 721                 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
 722                 if (rc != 0) {
 723                         device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
 724                         free(pi, M_CXGBE);
 725                         sc->port[i] = NULL;
 726                         goto done;
 727                 }
 728
 729                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 730                     device_get_nameunit(dev), i);
 731                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 732                 sc->chan_map[pi->tx_chan] = i;
 733
 734                 if (is_10G_port(pi) || is_40G_port(pi)) {
 735                         n10g++;
 736                         pi->tmr_idx = t4_tmr_idx_10g;
 737                         pi->pktc_idx = t4_pktc_idx_10g;
 738                 } else {
 739                         n1g++;
 740                         pi->tmr_idx = t4_tmr_idx_1g;
 741                         pi->pktc_idx = t4_pktc_idx_1g;
 742                 }
 743
 744                 pi->xact_addr_filt = -1;
 745                 pi->linkdnrc = -1;
 746
 747                 pi->qsize_rxq = t4_qsize_rxq;
 748                 pi->qsize_txq = t4_qsize_txq;
 749
 750                 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
 751                 if (pi->dev == NULL) {
 752                         device_printf(dev,
 753                             "failed to add device for port %d.\n", i);
 754                         rc = ENXIO;
 755                         goto done;
 756                 }
 757                 device_set_softc(pi->dev, pi);
 758         }
 759
 760         /*
 761          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 762          */
 763         rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
 764         if (rc != 0)
 765                 goto done; /* error message displayed already */
 766
 767         sc->intr_type = iaq.intr_type;
 768         sc->intr_count = iaq.nirq;
 769
 770         s = &sc->sge;
 771         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 772         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 773         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
 774         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 775         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
 776 #ifdef TCP_OFFLOAD
 777         if (is_offload(sc)) {
 778                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 779                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 780                 s->neq += s->nofldtxq + s->nofldrxq;
 781                 s->niq += s->nofldrxq;
 782
 783                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 784                     M_CXGBE, M_ZERO | M_WAITOK);
 785                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 786                     M_CXGBE, M_ZERO | M_WAITOK);
 787         }
 788 #endif
 789 #ifdef DEV_NETMAP
 790         s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g;
 791         s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g;
 792         s->neq += s->nnmtxq + s->nnmrxq;
 793         s->niq += s->nnmrxq;
 794
 795         s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
 796             M_CXGBE, M_ZERO | M_WAITOK);
 797         s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
 798             M_CXGBE, M_ZERO | M_WAITOK);
 799 #endif
 800
 801         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 802             M_ZERO | M_WAITOK);
 803         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 804             M_ZERO | M_WAITOK);
 805         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 806             M_ZERO | M_WAITOK);
 807         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 808             M_ZERO | M_WAITOK);
 809         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 810             M_ZERO | M_WAITOK);
 811
 812         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 813             M_ZERO | M_WAITOK);
 814
 815         t4_init_l2t(sc, M_WAITOK);
 816
 817         /*
 818          * Second pass over the ports.  This time we know the number of rx and
 819          * tx queues that each port should get.
 820          */
 821         rqidx = tqidx = 0;
 822 #ifdef TCP_OFFLOAD
 823         ofld_rqidx = ofld_tqidx = 0;
 824 #endif
 825 #ifdef DEV_NETMAP
 826         nm_rqidx = nm_tqidx = 0;
 827 #endif
 828         for_each_port(sc, i) {
 829                 struct port_info *pi = sc->port[i];
 830
 831                 if (pi == NULL)
 832                         continue;
 833
 834                 pi->first_rxq = rqidx;
 835                 pi->first_txq = tqidx;
 836                 if (is_10G_port(pi) || is_40G_port(pi)) {
 837                         pi->flags |= iaq.intr_flags_10g;
 838                         pi->nrxq = iaq.nrxq10g;
 839                         pi->ntxq = iaq.ntxq10g;
 840                 } else {
 841                         pi->flags |= iaq.intr_flags_1g;
 842                         pi->nrxq = iaq.nrxq1g;
 843                         pi->ntxq = iaq.ntxq1g;
 844                 }
 845
 846                 if (pi->ntxq > 1)
 847                         pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
 848                 else
 849                         pi->rsrv_noflowq = 0;
 850
 851                 rqidx += pi->nrxq;
 852                 tqidx += pi->ntxq;
 853 #ifdef TCP_OFFLOAD
 854                 if (is_offload(sc)) {
 855                         pi->first_ofld_rxq = ofld_rqidx;
 856                         pi->first_ofld_txq = ofld_tqidx;
 857                         if (is_10G_port(pi) || is_40G_port(pi)) {
 858                                 pi->nofldrxq = iaq.nofldrxq10g;
 859                                 pi->nofldtxq = iaq.nofldtxq10g;
 860                         } else {
 861                                 pi->nofldrxq = iaq.nofldrxq1g;
 862                                 pi->nofldtxq = iaq.nofldtxq1g;
 863                         }
 864                         ofld_rqidx += pi->nofldrxq;
 865                         ofld_tqidx += pi->nofldtxq;
 866                 }
 867 #endif
 868 #ifdef DEV_NETMAP
 869                 pi->first_nm_rxq = nm_rqidx;
 870                 pi->first_nm_txq = nm_tqidx;
 871                 if (is_10G_port(pi) || is_40G_port(pi)) {
 872                         pi->nnmrxq = iaq.nnmrxq10g;
 873                         pi->nnmtxq = iaq.nnmtxq10g;
 874                 } else {
 875                         pi->nnmrxq = iaq.nnmrxq1g;
 876                         pi->nnmtxq = iaq.nnmtxq1g;
 877                 }
 878                 nm_rqidx += pi->nnmrxq;
 879                 nm_tqidx += pi->nnmtxq;
 880 #endif
 881         }
 882
 883         rc = setup_intr_handlers(sc);
 884         if (rc != 0) {
 885                 device_printf(dev,
 886                     "failed to setup interrupt handlers: %d\n", rc);
 887                 goto done;
 888         }
 889
 890         rc = bus_generic_attach(dev);
 891         if (rc != 0) {
 892                 device_printf(dev,
 893                     "failed to attach all child ports: %d\n", rc);
 894                 goto done;
 895         }
 896
 897         device_printf(dev,
 898             "PCIe x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
 899             sc->params.pci.width, sc->params.nports, sc->intr_count,
 900             sc->intr_type == INTR_MSIX ? "MSI-X" :
 901             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
 902             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
 903
 904         t4_set_desc(sc);
 905
 906 done:
 907         if (rc != 0 && sc->cdev) {
 908                 /* cdev was created and so cxgbetool works; recover that way. */
 909                 device_printf(dev,
 910                     "error during attach, adapter is now in recovery mode.\n");
 911                 rc = 0;
 912         }
 913
 914         if (rc != 0)
 915                 t4_detach(dev);
 916         else
 917                 t4_sysctls(sc);
 918
 919         return (rc);
 920 }
 921
 922 /*
 923  * Idempotent
 924  */
 925 static int
 926 t4_detach(device_t dev)
 927 {
 928         struct adapter *sc;
 929         struct port_info *pi;
 930         int i, rc;
 931
 932         sc = device_get_softc(dev);
 933
 934         if (sc->flags & FULL_INIT_DONE)
 935                 t4_intr_disable(sc);
 936
 937         if (sc->cdev) {
 938                 destroy_dev(sc->cdev);
 939                 sc->cdev = NULL;
 940         }
 941
 942         rc = bus_generic_detach(dev);
 943         if (rc) {
 944                 device_printf(dev,
 945                     "failed to detach child devices: %d\n", rc);
 946                 return (rc);
 947         }
 948
 949         for (i = 0; i < sc->intr_count; i++)
 950                 t4_free_irq(sc, &sc->irq[i]);
 951
 952         for (i = 0; i < MAX_NPORTS; i++) {
 953                 pi = sc->port[i];
 954                 if (pi) {
 955                         t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid);
 956                         if (pi->dev)
 957                                 device_delete_child(dev, pi->dev);
 958
 959                         mtx_destroy(&pi->pi_lock);
 960                         free(pi, M_CXGBE);
 961                 }
 962         }
 963
 964         if (sc->flags & FULL_INIT_DONE)
 965                 adapter_full_uninit(sc);
 966
 967         if (sc->flags & FW_OK)
 968                 t4_fw_bye(sc, sc->mbox);
 969
 970         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
 971                 pci_release_msi(dev);
 972
 973         if (sc->regs_res)
 974                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
 975                     sc->regs_res);
 976
 977         if (sc->udbs_res)
 978                 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
 979                     sc->udbs_res);
 980
 981         if (sc->msix_res)
 982                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
 983                     sc->msix_res);
 984
 985         if (sc->l2t)
 986                 t4_free_l2t(sc->l2t);
 987
 988 #ifdef TCP_OFFLOAD
 989         free(sc->sge.ofld_rxq, M_CXGBE);
 990         free(sc->sge.ofld_txq, M_CXGBE);
 991 #endif
 992 #ifdef DEV_NETMAP
 993         free(sc->sge.nm_rxq, M_CXGBE);
 994         free(sc->sge.nm_txq, M_CXGBE);
 995 #endif
 996         free(sc->irq, M_CXGBE);
 997         free(sc->sge.rxq, M_CXGBE);
 998         free(sc->sge.txq, M_CXGBE);
 999         free(sc->sge.ctrlq, M_CXGBE);
1000         free(sc->sge.iqmap, M_CXGBE);
1001         free(sc->sge.eqmap, M_CXGBE);
1002         free(sc->tids.ftid_tab, M_CXGBE);
1003         t4_destroy_dma_tag(sc);
1004         if (mtx_initialized(&sc->sc_lock)) {
1005                 sx_xlock(&t4_list_lock);
1006                 SLIST_REMOVE(&t4_list, sc, adapter, link);
1007                 sx_xunlock(&t4_list_lock);
1008                 mtx_destroy(&sc->sc_lock);
1009         }
1010
1011         if (mtx_initialized(&sc->tids.ftid_lock))
1012                 mtx_destroy(&sc->tids.ftid_lock);
1013         if (mtx_initialized(&sc->sfl_lock))
1014                 mtx_destroy(&sc->sfl_lock);
1015         if (mtx_initialized(&sc->ifp_lock))
1016                 mtx_destroy(&sc->ifp_lock);
1017         if (mtx_initialized(&sc->regwin_lock))
1018                 mtx_destroy(&sc->regwin_lock);
1019
1020         bzero(sc, sizeof(*sc));
1021
1022         return (0);
1023 }
1024
1025 static int
1026 cxgbe_probe(device_t dev)
1027 {
1028         char buf[128];
1029         struct port_info *pi = device_get_softc(dev);
1030
1031         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
1032         device_set_desc_copy(dev, buf);
1033
1034         return (BUS_PROBE_DEFAULT);
1035 }
1036
1037 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
1038     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1039     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
1040 #define T4_CAP_ENABLE (T4_CAP)
1041
1042 static int
1043 cxgbe_attach(device_t dev)
1044 {
1045         struct port_info *pi = device_get_softc(dev);
1046         struct ifnet *ifp;
1047         char *s;
1048         int n, o;
1049
1050         /* Allocate an ifnet and set it up */
1051         ifp = if_alloc(IFT_ETHER);
1052         if (ifp == NULL) {
1053                 device_printf(dev, "Cannot allocate ifnet\n");
1054                 return (ENOMEM);
1055         }
1056         pi->ifp = ifp;
1057         ifp->if_softc = pi;
1058
1059         callout_init(&pi->tick, CALLOUT_MPSAFE);
1060
1061         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1062         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1063
1064         ifp->if_init = cxgbe_init;
1065         ifp->if_ioctl = cxgbe_ioctl;
1066         ifp->if_transmit = cxgbe_transmit;
1067         ifp->if_qflush = cxgbe_qflush;
1068
1069         ifp->if_capabilities = T4_CAP;
1070 #ifdef TCP_OFFLOAD
1071         if (is_offload(pi->adapter))
1072                 ifp->if_capabilities |= IFCAP_TOE;
1073 #endif
1074         ifp->if_capenable = T4_CAP_ENABLE;
1075         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1076             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1077
1078         ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1079         ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS;
1080         ifp->if_hw_tsomaxsegsize = 65536;
1081
1082         /* Initialize ifmedia for this port */
1083         ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1084             cxgbe_media_status);
1085         build_medialist(pi, &pi->media);
1086
1087         pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1088             EVENTHANDLER_PRI_ANY);
1089
1090         ether_ifattach(ifp, pi->hw_addr);
1091
1092         n = 128;
1093         s = malloc(n, M_CXGBE, M_WAITOK);
1094         o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq);
1095         MPASS(n > o);
1096 #ifdef TCP_OFFLOAD
1097         if (is_offload(pi->adapter)) {
1098                 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)",
1099                     pi->nofldtxq, pi->nofldrxq);
1100                 MPASS(n > o);
1101         }
1102 #endif
1103 #ifdef DEV_NETMAP
1104         o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq,
1105             pi->nnmrxq);
1106         MPASS(n > o);
1107 #endif
1108         device_printf(dev, "%s\n", s);
1109         free(s, M_CXGBE);
1110
1111 #ifdef DEV_NETMAP
1112         /* nm_media handled here to keep implementation private to this file */
1113         ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change,
1114             cxgbe_media_status);
1115         build_medialist(pi, &pi->nm_media);
1116         create_netmap_ifnet(pi);        /* logs errors it something fails */
1117 #endif
1118         cxgbe_sysctls(pi);
1119
1120         return (0);
1121 }
1122
1123 static int
1124 cxgbe_detach(device_t dev)
1125 {
1126         struct port_info *pi = device_get_softc(dev);
1127         struct adapter *sc = pi->adapter;
1128         struct ifnet *ifp = pi->ifp;
1129
1130         /* Tell if_ioctl and if_init that the port is going away */
1131         ADAPTER_LOCK(sc);
1132         SET_DOOMED(pi);
1133         wakeup(&sc->flags);
1134         while (IS_BUSY(sc))
1135                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
1136         SET_BUSY(sc);
1137 #ifdef INVARIANTS
1138         sc->last_op = "t4detach";
1139         sc->last_op_thr = curthread;
1140 #endif
1141         ADAPTER_UNLOCK(sc);
1142
1143         if (pi->flags & HAS_TRACEQ) {
1144                 sc->traceq = -1;        /* cloner should not create ifnet */
1145                 t4_tracer_port_detach(sc);
1146         }
1147
1148         if (pi->vlan_c)
1149                 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
1150
1151         PORT_LOCK(pi);
1152         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1153         callout_stop(&pi->tick);
1154         PORT_UNLOCK(pi);
1155         callout_drain(&pi->tick);
1156
1157         /* Let detach proceed even if these fail. */
1158         cxgbe_uninit_synchronized(pi);
1159         port_full_uninit(pi);
1160
1161         ifmedia_removeall(&pi->media);
1162         ether_ifdetach(pi->ifp);
1163         if_free(pi->ifp);
1164
1165 #ifdef DEV_NETMAP
1166         /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */
1167         destroy_netmap_ifnet(pi);
1168 #endif
1169
1170         ADAPTER_LOCK(sc);
1171         CLR_BUSY(sc);
1172         wakeup(&sc->flags);
1173         ADAPTER_UNLOCK(sc);
1174
1175         return (0);
1176 }
1177
1178 static void
1179 cxgbe_init(void *arg)
1180 {
1181         struct port_info *pi = arg;
1182         struct adapter *sc = pi->adapter;
1183
1184         if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
1185                 return;
1186         cxgbe_init_synchronized(pi);
1187         end_synchronized_op(sc, 0);
1188 }
1189
1190 static int
1191 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1192 {
1193         int rc = 0, mtu, flags, can_sleep;
1194         struct port_info *pi = ifp->if_softc;
1195         struct adapter *sc = pi->adapter;
1196         struct ifreq *ifr = (struct ifreq *)data;
1197         uint32_t mask;
1198
1199         switch (cmd) {
1200         case SIOCSIFMTU:
1201                 mtu = ifr->ifr_mtu;
1202                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1203                         return (EINVAL);
1204
1205                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
1206                 if (rc)
1207                         return (rc);
1208                 ifp->if_mtu = mtu;
1209                 if (pi->flags & PORT_INIT_DONE) {
1210                         t4_update_fl_bufsize(ifp);
1211                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1212                                 rc = update_mac_settings(ifp, XGMAC_MTU);
1213                 }
1214                 end_synchronized_op(sc, 0);
1215                 break;
1216
1217         case SIOCSIFFLAGS:
1218                 can_sleep = 0;
1219 redo_sifflags:
1220                 rc = begin_synchronized_op(sc, pi,
1221                     can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg");
1222                 if (rc)
1223                         return (rc);
1224
1225                 if (ifp->if_flags & IFF_UP) {
1226                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1227                                 flags = pi->if_flags;
1228                                 if ((ifp->if_flags ^ flags) &
1229                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1230                                         if (can_sleep == 1) {
1231                                                 end_synchronized_op(sc, 0);
1232                                                 can_sleep = 0;
1233                                                 goto redo_sifflags;
1234                                         }
1235                                         rc = update_mac_settings(ifp,
1236                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1237                                 }
1238                         } else {
1239                                 if (can_sleep == 0) {
1240                                         end_synchronized_op(sc, LOCK_HELD);
1241                                         can_sleep = 1;
1242                                         goto redo_sifflags;
1243                                 }
1244                                 rc = cxgbe_init_synchronized(pi);
1245                         }
1246                         pi->if_flags = ifp->if_flags;
1247                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1248                         if (can_sleep == 0) {
1249                                 end_synchronized_op(sc, LOCK_HELD);
1250                                 can_sleep = 1;
1251                                 goto redo_sifflags;
1252                         }
1253                         rc = cxgbe_uninit_synchronized(pi);
1254                 }
1255                 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD);
1256                 break;
1257
1258         case SIOCADDMULTI:
1259         case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1260                 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
1261                 if (rc)
1262                         return (rc);
1263                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1264                         rc = update_mac_settings(ifp, XGMAC_MCADDRS);
1265                 end_synchronized_op(sc, LOCK_HELD);
1266                 break;
1267
1268         case SIOCSIFCAP:
1269                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
1270                 if (rc)
1271                         return (rc);
1272
1273                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1274                 if (mask & IFCAP_TXCSUM) {
1275                         ifp->if_capenable ^= IFCAP_TXCSUM;
1276                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1277
1278                         if (IFCAP_TSO4 & ifp->if_capenable &&
1279                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1280                                 ifp->if_capenable &= ~IFCAP_TSO4;
1281                                 if_printf(ifp,
1282                                     "tso4 disabled due to -txcsum.\n");
1283                         }
1284                 }
1285                 if (mask & IFCAP_TXCSUM_IPV6) {
1286                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1287                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1288
1289                         if (IFCAP_TSO6 & ifp->if_capenable &&
1290                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1291                                 ifp->if_capenable &= ~IFCAP_TSO6;
1292                                 if_printf(ifp,
1293                                     "tso6 disabled due to -txcsum6.\n");
1294                         }
1295                 }
1296                 if (mask & IFCAP_RXCSUM)
1297                         ifp->if_capenable ^= IFCAP_RXCSUM;
1298                 if (mask & IFCAP_RXCSUM_IPV6)
1299                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1300
1301                 /*
1302                  * Note that we leave CSUM_TSO alone (it is always set).  The
1303                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1304                  * sending a TSO request our way, so it's sufficient to toggle
1305                  * IFCAP_TSOx only.
1306                  */
1307                 if (mask & IFCAP_TSO4) {
1308                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1309                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1310                                 if_printf(ifp, "enable txcsum first.\n");
1311                                 rc = EAGAIN;
1312                                 goto fail;
1313                         }
1314                         ifp->if_capenable ^= IFCAP_TSO4;
1315                 }
1316                 if (mask & IFCAP_TSO6) {
1317                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1318                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1319                                 if_printf(ifp, "enable txcsum6 first.\n");
1320                                 rc = EAGAIN;
1321                                 goto fail;
1322                         }
1323                         ifp->if_capenable ^= IFCAP_TSO6;
1324                 }
1325                 if (mask & IFCAP_LRO) {
1326 #if defined(INET) || defined(INET6)
1327                         int i;
1328                         struct sge_rxq *rxq;
1329
1330                         ifp->if_capenable ^= IFCAP_LRO;
1331                         for_each_rxq(pi, i, rxq) {
1332                                 if (ifp->if_capenable & IFCAP_LRO)
1333                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1334                                 else
1335                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1336                         }
1337 #endif
1338                 }
1339 #ifdef TCP_OFFLOAD
1340                 if (mask & IFCAP_TOE) {
1341                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1342
1343                         rc = toe_capability(pi, enable);
1344                         if (rc != 0)
1345                                 goto fail;
1346
1347                         ifp->if_capenable ^= mask;
1348                 }
1349 #endif
1350                 if (mask & IFCAP_VLAN_HWTAGGING) {
1351                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1352                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1353                                 rc = update_mac_settings(ifp, XGMAC_VLANEX);
1354                 }
1355                 if (mask & IFCAP_VLAN_MTU) {
1356                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1357
1358                         /* Need to find out how to disable auto-mtu-inflation */
1359                 }
1360                 if (mask & IFCAP_VLAN_HWTSO)
1361                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1362                 if (mask & IFCAP_VLAN_HWCSUM)
1363                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1364
1365 #ifdef VLAN_CAPABILITIES
1366                 VLAN_CAPABILITIES(ifp);
1367 #endif
1368 fail:
1369                 end_synchronized_op(sc, 0);
1370                 break;
1371
1372         case SIOCSIFMEDIA:
1373         case SIOCGIFMEDIA:
1374                 ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1375                 break;
1376
1377         default:
1378                 rc = ether_ioctl(ifp, cmd, data);
1379         }
1380
1381         return (rc);
1382 }
1383
1384 static int
1385 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1386 {
1387         struct port_info *pi = ifp->if_softc;
1388         struct adapter *sc = pi->adapter;
1389         struct sge_txq *txq = &sc->sge.txq[pi->first_txq];
1390         struct buf_ring *br;
1391         int rc;
1392
1393         M_ASSERTPKTHDR(m);
1394
1395         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1396                 m_freem(m);
1397                 return (ENETDOWN);
1398         }
1399
1400         /* check if flowid is set */
1401         if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
1402                 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq))
1403                     + pi->rsrv_noflowq);
1404         br = txq->br;
1405
1406         if (TXQ_TRYLOCK(txq) == 0) {
1407                 struct sge_eq *eq = &txq->eq;
1408
1409                 /*
1410                  * It is possible that t4_eth_tx finishes up and releases the
1411                  * lock between the TRYLOCK above and the drbr_enqueue here.  We
1412                  * need to make sure that this mbuf doesn't just sit there in
1413                  * the drbr.
1414                  */
1415
1416                 rc = drbr_enqueue(ifp, br, m);
1417                 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 &&
1418                     !(eq->flags & EQ_DOOMED))
1419                         callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq);
1420                 return (rc);
1421         }
1422
1423         /*
1424          * txq->m is the mbuf that is held up due to a temporary shortage of
1425          * resources and it should be put on the wire first.  Then what's in
1426          * drbr and finally the mbuf that was just passed in to us.
1427          *
1428          * Return code should indicate the fate of the mbuf that was passed in
1429          * this time.
1430          */
1431
1432         TXQ_LOCK_ASSERT_OWNED(txq);
1433         if (drbr_needs_enqueue(ifp, br) || txq->m) {
1434
1435                 /* Queued for transmission. */
1436
1437                 rc = drbr_enqueue(ifp, br, m);
1438                 m = txq->m ? txq->m : drbr_dequeue(ifp, br);
1439                 (void) t4_eth_tx(ifp, txq, m);
1440                 TXQ_UNLOCK(txq);
1441                 return (rc);
1442         }
1443
1444         /* Direct transmission. */
1445         rc = t4_eth_tx(ifp, txq, m);
1446         if (rc != 0 && txq->m)
1447                 rc = 0; /* held, will be transmitted soon (hopefully) */
1448
1449         TXQ_UNLOCK(txq);
1450         return (rc);
1451 }
1452
1453 static void
1454 cxgbe_qflush(struct ifnet *ifp)
1455 {
1456         struct port_info *pi = ifp->if_softc;
1457         struct sge_txq *txq;
1458         int i;
1459         struct mbuf *m;
1460
1461         /* queues do not exist if !PORT_INIT_DONE. */
1462         if (pi->flags & PORT_INIT_DONE) {
1463                 for_each_txq(pi, i, txq) {
1464                         TXQ_LOCK(txq);
1465                         m_freem(txq->m);
1466                         txq->m = NULL;
1467                         while ((m = buf_ring_dequeue_sc(txq->br)) != NULL)
1468                                 m_freem(m);
1469                         TXQ_UNLOCK(txq);
1470                 }
1471         }
1472         if_qflush(ifp);
1473 }
1474
1475 static int
1476 cxgbe_media_change(struct ifnet *ifp)
1477 {
1478         struct port_info *pi = ifp->if_softc;
1479
1480         device_printf(pi->dev, "%s unimplemented.\n", __func__);
1481
1482         return (EOPNOTSUPP);
1483 }
1484
1485 static void
1486 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1487 {
1488         struct port_info *pi = ifp->if_softc;
1489         struct ifmedia *media = NULL;
1490         struct ifmedia_entry *cur;
1491         int speed = pi->link_cfg.speed;
1492 #ifdef INVARIANTS
1493         int data = (pi->port_type << 8) | pi->mod_type;
1494 #endif
1495
1496         if (ifp == pi->ifp)
1497                 media = &pi->media;
1498 #ifdef DEV_NETMAP
1499         else if (ifp == pi->nm_ifp)
1500                 media = &pi->nm_media;
1501 #endif
1502         MPASS(media != NULL);
1503
1504         cur = media->ifm_cur;
1505         MPASS(cur->ifm_data == data);
1506
1507         ifmr->ifm_status = IFM_AVALID;
1508         if (!pi->link_cfg.link_ok)
1509                 return;
1510
1511         ifmr->ifm_status |= IFM_ACTIVE;
1512
1513         /* active and current will differ iff current media is autoselect. */
1514         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1515                 return;
1516
1517         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1518         if (speed == SPEED_10000)
1519                 ifmr->ifm_active |= IFM_10G_T;
1520         else if (speed == SPEED_1000)
1521                 ifmr->ifm_active |= IFM_1000_T;
1522         else if (speed == SPEED_100)
1523                 ifmr->ifm_active |= IFM_100_TX;
1524         else if (speed == SPEED_10)
1525                 ifmr->ifm_active |= IFM_10_T;
1526         else
1527                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1528                             speed));
1529 }
1530
1531 void
1532 t4_fatal_err(struct adapter *sc)
1533 {
1534         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1535         t4_intr_disable(sc);
1536         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1537             device_get_nameunit(sc->dev));
1538 }
1539
1540 static int
1541 map_bars_0_and_4(struct adapter *sc)
1542 {
1543         sc->regs_rid = PCIR_BAR(0);
1544         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1545             &sc->regs_rid, RF_ACTIVE);
1546         if (sc->regs_res == NULL) {
1547                 device_printf(sc->dev, "cannot map registers.\n");
1548                 return (ENXIO);
1549         }
1550         sc->bt = rman_get_bustag(sc->regs_res);
1551         sc->bh = rman_get_bushandle(sc->regs_res);
1552         sc->mmio_len = rman_get_size(sc->regs_res);
1553         setbit(&sc->doorbells, DOORBELL_KDB);
1554
1555         sc->msix_rid = PCIR_BAR(4);
1556         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1557             &sc->msix_rid, RF_ACTIVE);
1558         if (sc->msix_res == NULL) {
1559                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1560                 return (ENXIO);
1561         }
1562
1563         return (0);
1564 }
1565
1566 static int
1567 map_bar_2(struct adapter *sc)
1568 {
1569
1570         /*
1571          * T4: only iWARP driver uses the userspace doorbells.  There is no need
1572          * to map it if RDMA is disabled.
1573          */
1574         if (is_t4(sc) && sc->rdmacaps == 0)
1575                 return (0);
1576
1577         sc->udbs_rid = PCIR_BAR(2);
1578         sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1579             &sc->udbs_rid, RF_ACTIVE);
1580         if (sc->udbs_res == NULL) {
1581                 device_printf(sc->dev, "cannot map doorbell BAR.\n");
1582                 return (ENXIO);
1583         }
1584         sc->udbs_base = rman_get_virtual(sc->udbs_res);
1585
1586         if (is_t5(sc)) {
1587                 setbit(&sc->doorbells, DOORBELL_UDB);
1588 #if defined(__i386__) || defined(__amd64__)
1589                 if (t5_write_combine) {
1590                         int rc;
1591
1592                         /*
1593                          * Enable write combining on BAR2.  This is the
1594                          * userspace doorbell BAR and is split into 128B
1595                          * (UDBS_SEG_SIZE) doorbell regions, each associated
1596                          * with an egress queue.  The first 64B has the doorbell
1597                          * and the second 64B can be used to submit a tx work
1598                          * request with an implicit doorbell.
1599                          */
1600
1601                         rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
1602                             rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
1603                         if (rc == 0) {
1604                                 clrbit(&sc->doorbells, DOORBELL_UDB);
1605                                 setbit(&sc->doorbells, DOORBELL_WCWR);
1606                                 setbit(&sc->doorbells, DOORBELL_UDBWC);
1607                         } else {
1608                                 device_printf(sc->dev,
1609                                     "couldn't enable write combining: %d\n",
1610                                     rc);
1611                         }
1612
1613                         t4_write_reg(sc, A_SGE_STAT_CFG,
1614                             V_STATSOURCE_T5(7) | V_STATMODE(0));
1615                 }
1616 #endif
1617         }
1618
1619         return (0);
1620 }
1621
1622 static const struct memwin t4_memwin[] = {
1623         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1624         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1625         { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
1626 };
1627
1628 static const struct memwin t5_memwin[] = {
1629         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1630         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1631         { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1632 };
1633
1634 static void
1635 setup_memwin(struct adapter *sc)
1636 {
1637         const struct memwin *mw;
1638         int i, n;
1639         uint32_t bar0;
1640
1641         if (is_t4(sc)) {
1642                 /*
1643                  * Read low 32b of bar0 indirectly via the hardware backdoor
1644                  * mechanism.  Works from within PCI passthrough environments
1645                  * too, where rman_get_start() can return a different value.  We
1646                  * need to program the T4 memory window decoders with the actual
1647                  * addresses that will be coming across the PCIe link.
1648                  */
1649                 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1650                 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1651
1652                 mw = &t4_memwin[0];
1653                 n = nitems(t4_memwin);
1654         } else {
1655                 /* T5 uses the relative offset inside the PCIe BAR */
1656                 bar0 = 0;
1657
1658                 mw = &t5_memwin[0];
1659                 n = nitems(t5_memwin);
1660         }
1661
1662         for (i = 0; i < n; i++, mw++) {
1663                 t4_write_reg(sc,
1664                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
1665                     (mw->base + bar0) | V_BIR(0) |
1666                     V_WINDOW(ilog2(mw->aperture) - 10));
1667         }
1668
1669         /* flush */
1670         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1671 }
1672
1673 /*
1674  * Verify that the memory range specified by the addr/len pair is valid and lies
1675  * entirely within a single region (EDCx or MCx).
1676  */
1677 static int
1678 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
1679 {
1680         uint32_t em, addr_len, maddr, mlen;
1681
1682         /* Memory can only be accessed in naturally aligned 4 byte units */
1683         if (addr & 3 || len & 3 || len == 0)
1684                 return (EINVAL);
1685
1686         /* Enabled memories */
1687         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1688         if (em & F_EDRAM0_ENABLE) {
1689                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1690                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1691                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1692                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1693                     addr + len <= maddr + mlen)
1694                         return (0);
1695         }
1696         if (em & F_EDRAM1_ENABLE) {
1697                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1698                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1699                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1700                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1701                     addr + len <= maddr + mlen)
1702                         return (0);
1703         }
1704         if (em & F_EXT_MEM_ENABLE) {
1705                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1706                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1707                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1708                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1709                     addr + len <= maddr + mlen)
1710                         return (0);
1711         }
1712         if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
1713                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1714                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1715                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1716                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1717                     addr + len <= maddr + mlen)
1718                         return (0);
1719         }
1720
1721         return (EFAULT);
1722 }
1723
1724 static int
1725 fwmtype_to_hwmtype(int mtype)
1726 {
1727
1728         switch (mtype) {
1729         case FW_MEMTYPE_EDC0:
1730                 return (MEM_EDC0);
1731         case FW_MEMTYPE_EDC1:
1732                 return (MEM_EDC1);
1733         case FW_MEMTYPE_EXTMEM:
1734                 return (MEM_MC0);
1735         case FW_MEMTYPE_EXTMEM1:
1736                 return (MEM_MC1);
1737         default:
1738                 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
1739         }
1740 }
1741
1742 /*
1743  * Verify that the memory range specified by the memtype/offset/len pair is
1744  * valid and lies entirely within the memtype specified.  The global address of
1745  * the start of the range is returned in addr.
1746  */
1747 static int
1748 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1749     uint32_t *addr)
1750 {
1751         uint32_t em, addr_len, maddr, mlen;
1752
1753         /* Memory can only be accessed in naturally aligned 4 byte units */
1754         if (off & 3 || len & 3 || len == 0)
1755                 return (EINVAL);
1756
1757         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1758         switch (fwmtype_to_hwmtype(mtype)) {
1759         case MEM_EDC0:
1760                 if (!(em & F_EDRAM0_ENABLE))
1761                         return (EINVAL);
1762                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1763                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1764                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1765                 break;
1766         case MEM_EDC1:
1767                 if (!(em & F_EDRAM1_ENABLE))
1768                         return (EINVAL);
1769                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1770                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1771                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1772                 break;
1773         case MEM_MC:
1774                 if (!(em & F_EXT_MEM_ENABLE))
1775                         return (EINVAL);
1776                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1777                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1778                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1779                 break;
1780         case MEM_MC1:
1781                 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
1782                         return (EINVAL);
1783                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1784                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1785                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1786                 break;
1787         default:
1788                 return (EINVAL);
1789         }
1790
1791         if (mlen > 0 && off < mlen && off + len <= mlen) {
1792                 *addr = maddr + off;    /* global address */
1793                 return (0);
1794         }
1795
1796         return (EFAULT);
1797 }
1798
1799 static void
1800 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1801 {
1802         const struct memwin *mw;
1803
1804         if (is_t4(sc)) {
1805                 KASSERT(win >= 0 && win < nitems(t4_memwin),
1806                     ("%s: incorrect memwin# (%d)", __func__, win));
1807                 mw = &t4_memwin[win];
1808         } else {
1809                 KASSERT(win >= 0 && win < nitems(t5_memwin),
1810                     ("%s: incorrect memwin# (%d)", __func__, win));
1811                 mw = &t5_memwin[win];
1812         }
1813
1814         if (base != NULL)
1815                 *base = mw->base;
1816         if (aperture != NULL)
1817                 *aperture = mw->aperture;
1818 }
1819
1820 /*
1821  * Positions the memory window such that it can be used to access the specified
1822  * address in the chip's address space.  The return value is the offset of addr
1823  * from the start of the window.
1824  */
1825 static uint32_t
1826 position_memwin(struct adapter *sc, int n, uint32_t addr)
1827 {
1828         uint32_t start, pf;
1829         uint32_t reg;
1830
1831         KASSERT(n >= 0 && n <= 3,
1832             ("%s: invalid window %d.", __func__, n));
1833         KASSERT((addr & 3) == 0,
1834             ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
1835
1836         if (is_t4(sc)) {
1837                 pf = 0;
1838                 start = addr & ~0xf;    /* start must be 16B aligned */
1839         } else {
1840                 pf = V_PFNUM(sc->pf);
1841                 start = addr & ~0x7f;   /* start must be 128B aligned */
1842         }
1843         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1844
1845         t4_write_reg(sc, reg, start | pf);
1846         t4_read_reg(sc, reg);
1847
1848         return (addr - start);
1849 }
1850
1851 static int
1852 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1853     struct intrs_and_queues *iaq)
1854 {
1855         int rc, itype, navail, nrxq10g, nrxq1g, n;
1856         int nofldrxq10g = 0, nofldrxq1g = 0;
1857         int nnmrxq10g = 0, nnmrxq1g = 0;
1858
1859         bzero(iaq, sizeof(*iaq));
1860
1861         iaq->ntxq10g = t4_ntxq10g;
1862         iaq->ntxq1g = t4_ntxq1g;
1863         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1864         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1865         iaq->rsrv_noflowq = t4_rsrv_noflowq;
1866 #ifdef TCP_OFFLOAD
1867         if (is_offload(sc)) {
1868                 iaq->nofldtxq10g = t4_nofldtxq10g;
1869                 iaq->nofldtxq1g = t4_nofldtxq1g;
1870                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1871                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1872         }
1873 #endif
1874 #ifdef DEV_NETMAP
1875         iaq->nnmtxq10g = t4_nnmtxq10g;
1876         iaq->nnmtxq1g = t4_nnmtxq1g;
1877         iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g;
1878         iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g;
1879 #endif
1880
1881         for (itype = INTR_MSIX; itype; itype >>= 1) {
1882
1883                 if ((itype & t4_intr_types) == 0)
1884                         continue;       /* not allowed */
1885
1886                 if (itype == INTR_MSIX)
1887                         navail = pci_msix_count(sc->dev);
1888                 else if (itype == INTR_MSI)
1889                         navail = pci_msi_count(sc->dev);
1890                 else
1891                         navail = 1;
1892 restart:
1893                 if (navail == 0)
1894                         continue;
1895
1896                 iaq->intr_type = itype;
1897                 iaq->intr_flags_10g = 0;
1898                 iaq->intr_flags_1g = 0;
1899
1900                 /*
1901                  * Best option: an interrupt vector for errors, one for the
1902                  * firmware event queue, and one for every rxq (NIC, TOE, and
1903                  * netmap).
1904                  */
1905                 iaq->nirq = T4_EXTRA_INTR;
1906                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g);
1907                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g);
1908                 if (iaq->nirq <= navail &&
1909                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
1910                         iaq->intr_flags_10g = INTR_ALL;
1911                         iaq->intr_flags_1g = INTR_ALL;
1912                         goto allocate;
1913                 }
1914
1915                 /*
1916                  * Second best option: a vector for errors, one for the firmware
1917                  * event queue, and vectors for either all the NIC rx queues or
1918                  * all the TOE rx queues.  The queues that don't get vectors
1919                  * will forward their interrupts to those that do.
1920                  *
1921                  * Note: netmap rx queues cannot be created early and so they
1922                  * can't be setup to receive forwarded interrupts for others.
1923                  */
1924                 iaq->nirq = T4_EXTRA_INTR;
1925                 if (nrxq10g >= nofldrxq10g) {
1926                         iaq->intr_flags_10g = INTR_RXQ;
1927                         iaq->nirq += n10g * nrxq10g;
1928 #ifdef DEV_NETMAP
1929                         iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g);
1930 #endif
1931                 } else {
1932                         iaq->intr_flags_10g = INTR_OFLD_RXQ;
1933                         iaq->nirq += n10g * nofldrxq10g;
1934 #ifdef DEV_NETMAP
1935                         iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g);
1936 #endif
1937                 }
1938                 if (nrxq1g >= nofldrxq1g) {
1939                         iaq->intr_flags_1g = INTR_RXQ;
1940                         iaq->nirq += n1g * nrxq1g;
1941 #ifdef DEV_NETMAP
1942                         iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g);
1943 #endif
1944                 } else {
1945                         iaq->intr_flags_1g = INTR_OFLD_RXQ;
1946                         iaq->nirq += n1g * nofldrxq1g;
1947 #ifdef DEV_NETMAP
1948                         iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g);
1949 #endif
1950                 }
1951                 if (iaq->nirq <= navail &&
1952                     (itype != INTR_MSI || powerof2(iaq->nirq)))
1953                         goto allocate;
1954
1955                 /*
1956                  * Next best option: an interrupt vector for errors, one for the
1957                  * firmware event queue, and at least one per port.  At this
1958                  * point we know we'll have to downsize nrxq and/or nofldrxq
1959                  * and/or nnmrxq to fit what's available to us.
1960                  */
1961                 iaq->nirq = T4_EXTRA_INTR;
1962                 iaq->nirq += n10g + n1g;
1963                 if (iaq->nirq <= navail) {
1964                         int leftover = navail - iaq->nirq;
1965
1966                         if (n10g > 0) {
1967                                 int target = max(nrxq10g, nofldrxq10g);
1968
1969                                 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
1970                                     INTR_RXQ : INTR_OFLD_RXQ;
1971
1972                                 n = 1;
1973                                 while (n < target && leftover >= n10g) {
1974                                         leftover -= n10g;
1975                                         iaq->nirq += n10g;
1976                                         n++;
1977                                 }
1978                                 iaq->nrxq10g = min(n, nrxq10g);
1979 #ifdef TCP_OFFLOAD
1980                                 iaq->nofldrxq10g = min(n, nofldrxq10g);
1981 #endif
1982 #ifdef DEV_NETMAP
1983                                 iaq->nnmrxq10g = min(n, nnmrxq10g);
1984 #endif
1985                         }
1986
1987                         if (n1g > 0) {
1988                                 int target = max(nrxq1g, nofldrxq1g);
1989
1990                                 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
1991                                     INTR_RXQ : INTR_OFLD_RXQ;
1992
1993                                 n = 1;
1994                                 while (n < target && leftover >= n1g) {
1995                                         leftover -= n1g;
1996                                         iaq->nirq += n1g;
1997                                         n++;
1998                                 }
1999                                 iaq->nrxq1g = min(n, nrxq1g);
2000 #ifdef TCP_OFFLOAD
2001                                 iaq->nofldrxq1g = min(n, nofldrxq1g);
2002 #endif
2003 #ifdef DEV_NETMAP
2004                                 iaq->nnmrxq1g = min(n, nnmrxq1g);
2005 #endif
2006                         }
2007
2008                         if (itype != INTR_MSI || powerof2(iaq->nirq))
2009                                 goto allocate;
2010                 }
2011
2012                 /*
2013                  * Least desirable option: one interrupt vector for everything.
2014                  */
2015                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
2016                 iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
2017 #ifdef TCP_OFFLOAD
2018                 if (is_offload(sc))
2019                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2020 #endif
2021 #ifdef DEV_NETMAP
2022                 iaq->nnmrxq10g = iaq->nnmrxq1g = 1;
2023 #endif
2024
2025 allocate:
2026                 navail = iaq->nirq;
2027                 rc = 0;
2028                 if (itype == INTR_MSIX)
2029                         rc = pci_alloc_msix(sc->dev, &navail);
2030                 else if (itype == INTR_MSI)
2031                         rc = pci_alloc_msi(sc->dev, &navail);
2032
2033                 if (rc == 0) {
2034                         if (navail == iaq->nirq)
2035                                 return (0);
2036
2037                         /*
2038                          * Didn't get the number requested.  Use whatever number
2039                          * the kernel is willing to allocate (it's in navail).
2040                          */
2041                         device_printf(sc->dev, "fewer vectors than requested, "
2042                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
2043                             itype, iaq->nirq, navail);
2044                         pci_release_msi(sc->dev);
2045                         goto restart;
2046                 }
2047
2048                 device_printf(sc->dev,
2049                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
2050                     itype, rc, iaq->nirq, navail);
2051         }
2052
2053         device_printf(sc->dev,
2054             "failed to find a usable interrupt type.  "
2055             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
2056             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
2057
2058         return (ENXIO);
2059 }
2060
2061 #define FW_VERSION(chip) ( \
2062     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
2063     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
2064     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
2065     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
2066 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
2067
2068 struct fw_info {
2069         uint8_t chip;
2070         char *kld_name;
2071         char *fw_mod_name;
2072         struct fw_hdr fw_hdr;   /* XXX: waste of space, need a sparse struct */
2073 } fw_info[] = {
2074         {
2075                 .chip = CHELSIO_T4,
2076                 .kld_name = "t4fw_cfg",
2077                 .fw_mod_name = "t4fw",
2078                 .fw_hdr = {
2079                         .chip = FW_HDR_CHIP_T4,
2080                         .fw_ver = htobe32_const(FW_VERSION(T4)),
2081                         .intfver_nic = FW_INTFVER(T4, NIC),
2082                         .intfver_vnic = FW_INTFVER(T4, VNIC),
2083                         .intfver_ofld = FW_INTFVER(T4, OFLD),
2084                         .intfver_ri = FW_INTFVER(T4, RI),
2085                         .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
2086                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
2087                         .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
2088                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
2089                 },
2090         }, {
2091                 .chip = CHELSIO_T5,
2092                 .kld_name = "t5fw_cfg",
2093                 .fw_mod_name = "t5fw",
2094                 .fw_hdr = {
2095                         .chip = FW_HDR_CHIP_T5,
2096                         .fw_ver = htobe32_const(FW_VERSION(T5)),
2097                         .intfver_nic = FW_INTFVER(T5, NIC),
2098                         .intfver_vnic = FW_INTFVER(T5, VNIC),
2099                         .intfver_ofld = FW_INTFVER(T5, OFLD),
2100                         .intfver_ri = FW_INTFVER(T5, RI),
2101                         .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
2102                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
2103                         .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
2104                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
2105                 },
2106         }
2107 };
2108
2109 static struct fw_info *
2110 find_fw_info(int chip)
2111 {
2112         int i;
2113
2114         for (i = 0; i < nitems(fw_info); i++) {
2115                 if (fw_info[i].chip == chip)
2116                         return (&fw_info[i]);
2117         }
2118         return (NULL);
2119 }
2120
2121 /*
2122  * Is the given firmware API compatible with the one the driver was compiled
2123  * with?
2124  */
2125 static int
2126 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
2127 {
2128
2129         /* short circuit if it's the exact same firmware version */
2130         if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
2131                 return (1);
2132
2133         /*
2134          * XXX: Is this too conservative?  Perhaps I should limit this to the
2135          * features that are supported in the driver.
2136          */
2137 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
2138         if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
2139             SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
2140             SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
2141                 return (1);
2142 #undef SAME_INTF
2143
2144         return (0);
2145 }
2146
2147 /*
2148  * The firmware in the KLD is usable, but should it be installed?  This routine
2149  * explains itself in detail if it indicates the KLD firmware should be
2150  * installed.
2151  */
2152 static int
2153 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
2154 {
2155         const char *reason;
2156
2157         if (!card_fw_usable) {
2158                 reason = "incompatible or unusable";
2159                 goto install;
2160         }
2161
2162         if (k > c) {
2163                 reason = "older than the version bundled with this driver";
2164                 goto install;
2165         }
2166
2167         if (t4_fw_install == 2 && k != c) {
2168                 reason = "different than the version bundled with this driver";
2169                 goto install;
2170         }
2171
2172         return (0);
2173
2174 install:
2175         if (t4_fw_install == 0) {
2176                 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2177                     "but the driver is prohibited from installing a different "
2178                     "firmware on the card.\n",
2179                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2180                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2181
2182                 return (0);
2183         }
2184
2185         device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2186             "installing firmware %u.%u.%u.%u on card.\n",
2187             G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2188             G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2189             G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2190             G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2191
2192         return (1);
2193 }
2194 /*
2195  * Establish contact with the firmware and determine if we are the master driver
2196  * or not, and whether we are responsible for chip initialization.
2197  */
2198 static int
2199 prep_firmware(struct adapter *sc)
2200 {
2201         const struct firmware *fw = NULL, *default_cfg;
2202         int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2203         enum dev_state state;
2204         struct fw_info *fw_info;
2205         struct fw_hdr *card_fw;         /* fw on the card */
2206         const struct fw_hdr *kld_fw;    /* fw in the KLD */
2207         const struct fw_hdr *drv_fw;    /* fw header the driver was compiled
2208                                            against */
2209
2210         /* Contact firmware. */
2211         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2212         if (rc < 0 || state == DEV_STATE_ERR) {
2213                 rc = -rc;
2214                 device_printf(sc->dev,
2215                     "failed to connect to the firmware: %d, %d.\n", rc, state);
2216                 return (rc);
2217         }
2218         pf = rc;
2219         if (pf == sc->mbox)
2220                 sc->flags |= MASTER_PF;
2221         else if (state == DEV_STATE_UNINIT) {
2222                 /*
2223                  * We didn't get to be the master so we definitely won't be
2224                  * configuring the chip.  It's a bug if someone else hasn't
2225                  * configured it already.
2226                  */
2227                 device_printf(sc->dev, "couldn't be master(%d), "
2228                     "device not already initialized either(%d).\n", rc, state);
2229                 return (EDOOFUS);
2230         }
2231
2232         /* This is the firmware whose headers the driver was compiled against */
2233         fw_info = find_fw_info(chip_id(sc));
2234         if (fw_info == NULL) {
2235                 device_printf(sc->dev,
2236                     "unable to look up firmware information for chip %d.\n",
2237                     chip_id(sc));
2238                 return (EINVAL);
2239         }
2240         drv_fw = &fw_info->fw_hdr;
2241
2242         /*
2243          * The firmware KLD contains many modules.  The KLD name is also the
2244          * name of the module that contains the default config file.
2245          */
2246         default_cfg = firmware_get(fw_info->kld_name);
2247
2248         /* Read the header of the firmware on the card */
2249         card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2250         rc = -t4_read_flash(sc, FLASH_FW_START,
2251             sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2252         if (rc == 0)
2253                 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2254         else {
2255                 device_printf(sc->dev,
2256                     "Unable to read card's firmware header: %d\n", rc);
2257                 card_fw_usable = 0;
2258         }
2259
2260         /* This is the firmware in the KLD */
2261         fw = firmware_get(fw_info->fw_mod_name);
2262         if (fw != NULL) {
2263                 kld_fw = (const void *)fw->data;
2264                 kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2265         } else {
2266                 kld_fw = NULL;
2267                 kld_fw_usable = 0;
2268         }
2269
2270         if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2271             (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2272                 /*
2273                  * Common case: the firmware on the card is an exact match and
2274                  * the KLD is an exact match too, or the KLD is
2275                  * absent/incompatible.  Note that t4_fw_install = 2 is ignored
2276                  * here -- use cxgbetool loadfw if you want to reinstall the
2277                  * same firmware as the one on the card.
2278                  */
2279         } else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2280             should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2281             be32toh(card_fw->fw_ver))) {
2282
2283                 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
2284                 if (rc != 0) {
2285                         device_printf(sc->dev,
2286                             "failed to install firmware: %d\n", rc);
2287                         goto done;
2288                 }
2289
2290                 /* Installed successfully, update the cached header too. */
2291                 memcpy(card_fw, kld_fw, sizeof(*card_fw));
2292                 card_fw_usable = 1;
2293                 need_fw_reset = 0;      /* already reset as part of load_fw */
2294         }
2295
2296         if (!card_fw_usable) {
2297                 uint32_t d, c, k;
2298
2299                 d = ntohl(drv_fw->fw_ver);
2300                 c = ntohl(card_fw->fw_ver);
2301                 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
2302
2303                 device_printf(sc->dev, "Cannot find a usable firmware: "
2304                     "fw_install %d, chip state %d, "
2305                     "driver compiled with %d.%d.%d.%d, "
2306                     "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
2307                     t4_fw_install, state,
2308                     G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
2309                     G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
2310                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2311                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
2312                     G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2313                     G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2314                 rc = EINVAL;
2315                 goto done;
2316         }
2317
2318         /* We're using whatever's on the card and it's known to be good. */
2319         sc->params.fw_vers = ntohl(card_fw->fw_ver);
2320         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
2321             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2322             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2323             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2324             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2325         t4_get_tp_version(sc, &sc->params.tp_vers);
2326
2327         /* Reset device */
2328         if (need_fw_reset &&
2329             (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
2330                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
2331                 if (rc != ETIMEDOUT && rc != EIO)
2332                         t4_fw_bye(sc, sc->mbox);
2333                 goto done;
2334         }
2335         sc->flags |= FW_OK;
2336
2337         rc = get_params__pre_init(sc);
2338         if (rc != 0)
2339                 goto done; /* error message displayed already */
2340
2341         /* Partition adapter resources as specified in the config file. */
2342         if (state == DEV_STATE_UNINIT) {
2343
2344                 KASSERT(sc->flags & MASTER_PF,
2345                     ("%s: trying to change chip settings when not master.",
2346                     __func__));
2347
2348                 rc = partition_resources(sc, default_cfg, fw_info->kld_name);
2349                 if (rc != 0)
2350                         goto done;      /* error message displayed already */
2351
2352                 t4_tweak_chip_settings(sc);
2353
2354                 /* get basic stuff going */
2355                 rc = -t4_fw_initialize(sc, sc->mbox);
2356                 if (rc != 0) {
2357                         device_printf(sc->dev, "fw init failed: %d.\n", rc);
2358                         goto done;
2359                 }
2360         } else {
2361                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
2362                 sc->cfcsum = 0;
2363         }
2364
2365 done:
2366         free(card_fw, M_CXGBE);
2367         if (fw != NULL)
2368                 firmware_put(fw, FIRMWARE_UNLOAD);
2369         if (default_cfg != NULL)
2370                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
2371
2372         return (rc);
2373 }
2374
2375 #define FW_PARAM_DEV(param) \
2376         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2377          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2378 #define FW_PARAM_PFVF(param) \
2379         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2380          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2381
2382 /*
2383  * Partition chip resources for use between various PFs, VFs, etc.
2384  */
2385 static int
2386 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
2387     const char *name_prefix)
2388 {
2389         const struct firmware *cfg = NULL;
2390         int rc = 0;
2391         struct fw_caps_config_cmd caps;
2392         uint32_t mtype, moff, finicsum, cfcsum;
2393
2394         /*
2395          * Figure out what configuration file to use.  Pick the default config
2396          * file for the card if the user hasn't specified one explicitly.
2397          */
2398         snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
2399         if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
2400                 /* Card specific overrides go here. */
2401                 if (pci_get_device(sc->dev) == 0x440a)
2402                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
2403                 if (is_fpga(sc))
2404                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
2405         }
2406
2407         /*
2408          * We need to load another module if the profile is anything except
2409          * "default" or "flash".
2410          */
2411         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
2412             strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2413                 char s[32];
2414
2415                 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
2416                 cfg = firmware_get(s);
2417                 if (cfg == NULL) {
2418                         if (default_cfg != NULL) {
2419                                 device_printf(sc->dev,
2420                                     "unable to load module \"%s\" for "
2421                                     "configuration profile \"%s\", will use "
2422                                     "the default config file instead.\n",
2423                                     s, sc->cfg_file);
2424                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2425                                     "%s", DEFAULT_CF);
2426                         } else {
2427                                 device_printf(sc->dev,
2428                                     "unable to load module \"%s\" for "
2429                                     "configuration profile \"%s\", will use "
2430                                     "the config file on the card's flash "
2431                                     "instead.\n", s, sc->cfg_file);
2432                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2433                                     "%s", FLASH_CF);
2434                         }
2435                 }
2436         }
2437
2438         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
2439             default_cfg == NULL) {
2440                 device_printf(sc->dev,
2441                     "default config file not available, will use the config "
2442                     "file on the card's flash instead.\n");
2443                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
2444         }
2445
2446         if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2447                 u_int cflen, i, n;
2448                 const uint32_t *cfdata;
2449                 uint32_t param, val, addr, off, mw_base, mw_aperture;
2450
2451                 KASSERT(cfg != NULL || default_cfg != NULL,
2452                     ("%s: no config to upload", __func__));
2453
2454                 /*
2455                  * Ask the firmware where it wants us to upload the config file.
2456                  */
2457                 param = FW_PARAM_DEV(CF);
2458                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2459                 if (rc != 0) {
2460                         /* No support for config file?  Shouldn't happen. */
2461                         device_printf(sc->dev,
2462                             "failed to query config file location: %d.\n", rc);
2463                         goto done;
2464                 }
2465                 mtype = G_FW_PARAMS_PARAM_Y(val);
2466                 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
2467
2468                 /*
2469                  * XXX: sheer laziness.  We deliberately added 4 bytes of
2470                  * useless stuffing/comments at the end of the config file so
2471                  * it's ok to simply throw away the last remaining bytes when
2472                  * the config file is not an exact multiple of 4.  This also
2473                  * helps with the validate_mt_off_len check.
2474                  */
2475                 if (cfg != NULL) {
2476                         cflen = cfg->datasize & ~3;
2477                         cfdata = cfg->data;
2478                 } else {
2479                         cflen = default_cfg->datasize & ~3;
2480                         cfdata = default_cfg->data;
2481                 }
2482
2483                 if (cflen > FLASH_CFG_MAX_SIZE) {
2484                         device_printf(sc->dev,
2485                             "config file too long (%d, max allowed is %d).  "
2486                             "Will try to use the config on the card, if any.\n",
2487                             cflen, FLASH_CFG_MAX_SIZE);
2488                         goto use_config_on_flash;
2489                 }
2490
2491                 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
2492                 if (rc != 0) {
2493                         device_printf(sc->dev,
2494                             "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
2495                             "Will try to use the config on the card, if any.\n",
2496                             __func__, mtype, moff, cflen, rc);
2497                         goto use_config_on_flash;
2498                 }
2499
2500                 memwin_info(sc, 2, &mw_base, &mw_aperture);
2501                 while (cflen) {
2502                         off = position_memwin(sc, 2, addr);
2503                         n = min(cflen, mw_aperture - off);
2504                         for (i = 0; i < n; i += 4)
2505                                 t4_write_reg(sc, mw_base + off + i, *cfdata++);
2506                         cflen -= n;
2507                         addr += n;
2508                 }
2509         } else {
2510 use_config_on_flash:
2511                 mtype = FW_MEMTYPE_FLASH;
2512                 moff = t4_flash_cfg_addr(sc);
2513         }
2514
2515         bzero(&caps, sizeof(caps));
2516         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2517             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2518         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
2519             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
2520             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
2521         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2522         if (rc != 0) {
2523                 device_printf(sc->dev,
2524                     "failed to pre-process config file: %d "
2525                     "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
2526                 goto done;
2527         }
2528
2529         finicsum = be32toh(caps.finicsum);
2530         cfcsum = be32toh(caps.cfcsum);
2531         if (finicsum != cfcsum) {
2532                 device_printf(sc->dev,
2533                     "WARNING: config file checksum mismatch: %08x %08x\n",
2534                     finicsum, cfcsum);
2535         }
2536         sc->cfcsum = cfcsum;
2537
2538 #define LIMIT_CAPS(x) do { \
2539         caps.x &= htobe16(t4_##x##_allowed); \
2540 } while (0)
2541
2542         /*
2543          * Let the firmware know what features will (not) be used so it can tune
2544          * things accordingly.
2545          */
2546         LIMIT_CAPS(linkcaps);
2547         LIMIT_CAPS(niccaps);
2548         LIMIT_CAPS(toecaps);
2549         LIMIT_CAPS(rdmacaps);
2550         LIMIT_CAPS(iscsicaps);
2551         LIMIT_CAPS(fcoecaps);
2552 #undef LIMIT_CAPS
2553
2554         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2555             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
2556         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2557         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
2558         if (rc != 0) {
2559                 device_printf(sc->dev,
2560                     "failed to process config file: %d.\n", rc);
2561         }
2562 done:
2563         if (cfg != NULL)
2564                 firmware_put(cfg, FIRMWARE_UNLOAD);
2565         return (rc);
2566 }
2567
2568 /*
2569  * Retrieve parameters that are needed (or nice to have) very early.
2570  */
2571 static int
2572 get_params__pre_init(struct adapter *sc)
2573 {
2574         int rc;
2575         uint32_t param[2], val[2];
2576         struct fw_devlog_cmd cmd;
2577         struct devlog_params *dlog = &sc->params.devlog;
2578
2579         param[0] = FW_PARAM_DEV(PORTVEC);
2580         param[1] = FW_PARAM_DEV(CCLK);
2581         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2582         if (rc != 0) {
2583                 device_printf(sc->dev,
2584                     "failed to query parameters (pre_init): %d.\n", rc);
2585                 return (rc);
2586         }
2587
2588         sc->params.portvec = val[0];
2589         sc->params.nports = bitcount32(val[0]);
2590         sc->params.vpd.cclk = val[1];
2591
2592         /* Read device log parameters. */
2593         bzero(&cmd, sizeof(cmd));
2594         cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
2595             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2596         cmd.retval_len16 = htobe32(FW_LEN16(cmd));
2597         rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
2598         if (rc != 0) {
2599                 device_printf(sc->dev,
2600                     "failed to get devlog parameters: %d.\n", rc);
2601                 bzero(dlog, sizeof (*dlog));
2602                 rc = 0; /* devlog isn't critical for device operation */
2603         } else {
2604                 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
2605                 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
2606                 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
2607                 dlog->size = be32toh(cmd.memsize_devlog);
2608         }
2609
2610         return (rc);
2611 }
2612
2613 /*
2614  * Retrieve various parameters that are of interest to the driver.  The device
2615  * has been initialized by the firmware at this point.
2616  */
2617 static int
2618 get_params__post_init(struct adapter *sc)
2619 {
2620         int rc;
2621         uint32_t param[7], val[7];
2622         struct fw_caps_config_cmd caps;
2623
2624         param[0] = FW_PARAM_PFVF(IQFLINT_START);
2625         param[1] = FW_PARAM_PFVF(EQ_START);
2626         param[2] = FW_PARAM_PFVF(FILTER_START);
2627         param[3] = FW_PARAM_PFVF(FILTER_END);
2628         param[4] = FW_PARAM_PFVF(L2T_START);
2629         param[5] = FW_PARAM_PFVF(L2T_END);
2630         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2631         if (rc != 0) {
2632                 device_printf(sc->dev,
2633                     "failed to query parameters (post_init): %d.\n", rc);
2634                 return (rc);
2635         }
2636
2637         sc->sge.iq_start = val[0];
2638         sc->sge.eq_start = val[1];
2639         sc->tids.ftid_base = val[2];
2640         sc->tids.nftids = val[3] - val[2] + 1;
2641         sc->params.ftid_min = val[2];
2642         sc->params.ftid_max = val[3];
2643         sc->vres.l2t.start = val[4];
2644         sc->vres.l2t.size = val[5] - val[4] + 1;
2645         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
2646             ("%s: L2 table size (%u) larger than expected (%u)",
2647             __func__, sc->vres.l2t.size, L2T_SIZE));
2648
2649         /* get capabilites */
2650         bzero(&caps, sizeof(caps));
2651         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2652             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2653         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2654         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2655         if (rc != 0) {
2656                 device_printf(sc->dev,
2657                     "failed to get card capabilities: %d.\n", rc);
2658                 return (rc);
2659         }
2660
2661 #define READ_CAPS(x) do { \
2662         sc->x = htobe16(caps.x); \
2663 } while (0)
2664         READ_CAPS(linkcaps);
2665         READ_CAPS(niccaps);
2666         READ_CAPS(toecaps);
2667         READ_CAPS(rdmacaps);
2668         READ_CAPS(iscsicaps);
2669         READ_CAPS(fcoecaps);
2670
2671         if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
2672                 param[0] = FW_PARAM_PFVF(ETHOFLD_START);
2673                 param[1] = FW_PARAM_PFVF(ETHOFLD_END);
2674                 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2675                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
2676                 if (rc != 0) {
2677                         device_printf(sc->dev,
2678                             "failed to query NIC parameters: %d.\n", rc);
2679                         return (rc);
2680                 }
2681                 sc->tids.etid_base = val[0];
2682                 sc->params.etid_min = val[0];
2683                 sc->tids.netids = val[1] - val[0] + 1;
2684                 sc->params.netids = sc->tids.netids;
2685                 sc->params.eo_wr_cred = val[2];
2686                 sc->params.ethoffload = 1;
2687         }
2688
2689         if (sc->toecaps) {
2690                 /* query offload-related parameters */
2691                 param[0] = FW_PARAM_DEV(NTID);
2692                 param[1] = FW_PARAM_PFVF(SERVER_START);
2693                 param[2] = FW_PARAM_PFVF(SERVER_END);
2694                 param[3] = FW_PARAM_PFVF(TDDP_START);
2695                 param[4] = FW_PARAM_PFVF(TDDP_END);
2696                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2697                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2698                 if (rc != 0) {
2699                         device_printf(sc->dev,
2700                             "failed to query TOE parameters: %d.\n", rc);
2701                         return (rc);
2702                 }
2703                 sc->tids.ntids = val[0];
2704                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
2705                 sc->tids.stid_base = val[1];
2706                 sc->tids.nstids = val[2] - val[1] + 1;
2707                 sc->vres.ddp.start = val[3];
2708                 sc->vres.ddp.size = val[4] - val[3] + 1;
2709                 sc->params.ofldq_wr_cred = val[5];
2710                 sc->params.offload = 1;
2711         }
2712         if (sc->rdmacaps) {
2713                 param[0] = FW_PARAM_PFVF(STAG_START);
2714                 param[1] = FW_PARAM_PFVF(STAG_END);
2715                 param[2] = FW_PARAM_PFVF(RQ_START);
2716                 param[3] = FW_PARAM_PFVF(RQ_END);
2717                 param[4] = FW_PARAM_PFVF(PBL_START);
2718                 param[5] = FW_PARAM_PFVF(PBL_END);
2719                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2720                 if (rc != 0) {
2721                         device_printf(sc->dev,
2722                             "failed to query RDMA parameters(1): %d.\n", rc);
2723                         return (rc);
2724                 }
2725                 sc->vres.stag.start = val[0];
2726                 sc->vres.stag.size = val[1] - val[0] + 1;
2727                 sc->vres.rq.start = val[2];
2728                 sc->vres.rq.size = val[3] - val[2] + 1;
2729                 sc->vres.pbl.start = val[4];
2730                 sc->vres.pbl.size = val[5] - val[4] + 1;
2731
2732                 param[0] = FW_PARAM_PFVF(SQRQ_START);
2733                 param[1] = FW_PARAM_PFVF(SQRQ_END);
2734                 param[2] = FW_PARAM_PFVF(CQ_START);
2735                 param[3] = FW_PARAM_PFVF(CQ_END);
2736                 param[4] = FW_PARAM_PFVF(OCQ_START);
2737                 param[5] = FW_PARAM_PFVF(OCQ_END);
2738                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2739                 if (rc != 0) {
2740                         device_printf(sc->dev,
2741                             "failed to query RDMA parameters(2): %d.\n", rc);
2742                         return (rc);
2743                 }
2744                 sc->vres.qp.start = val[0];
2745                 sc->vres.qp.size = val[1] - val[0] + 1;
2746                 sc->vres.cq.start = val[2];
2747                 sc->vres.cq.size = val[3] - val[2] + 1;
2748                 sc->vres.ocq.start = val[4];
2749                 sc->vres.ocq.size = val[5] - val[4] + 1;
2750         }
2751         if (sc->iscsicaps) {
2752                 param[0] = FW_PARAM_PFVF(ISCSI_START);
2753                 param[1] = FW_PARAM_PFVF(ISCSI_END);
2754                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2755                 if (rc != 0) {
2756                         device_printf(sc->dev,
2757                             "failed to query iSCSI parameters: %d.\n", rc);
2758                         return (rc);
2759                 }
2760                 sc->vres.iscsi.start = val[0];
2761                 sc->vres.iscsi.size = val[1] - val[0] + 1;
2762         }
2763
2764         /*
2765          * We've got the params we wanted to query via the firmware.  Now grab
2766          * some others directly from the chip.
2767          */
2768         rc = t4_read_chip_settings(sc);
2769
2770         return (rc);
2771 }
2772
2773 static int
2774 set_params__post_init(struct adapter *sc)
2775 {
2776         uint32_t param, val;
2777
2778         /* ask for encapsulated CPLs */
2779         param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
2780         val = 1;
2781         (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2782
2783         return (0);
2784 }
2785
2786 #undef FW_PARAM_PFVF
2787 #undef FW_PARAM_DEV
2788
2789 static void
2790 t4_set_desc(struct adapter *sc)
2791 {
2792         char buf[128];
2793         struct adapter_params *p = &sc->params;
2794
2795         snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
2796             "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
2797             chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
2798
2799         device_set_desc_copy(sc->dev, buf);
2800 }
2801
2802 static void
2803 build_medialist(struct port_info *pi, struct ifmedia *media)
2804 {
2805         int data, m;
2806
2807         PORT_LOCK(pi);
2808
2809         ifmedia_removeall(media);
2810
2811         m = IFM_ETHER | IFM_FDX;
2812         data = (pi->port_type << 8) | pi->mod_type;
2813
2814         switch(pi->port_type) {
2815         case FW_PORT_TYPE_BT_XFI:
2816         case FW_PORT_TYPE_BT_XAUI:
2817                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2818                 /* fall through */
2819
2820         case FW_PORT_TYPE_BT_SGMII:
2821                 ifmedia_add(media, m | IFM_1000_T, data, NULL);
2822                 ifmedia_add(media, m | IFM_100_TX, data, NULL);
2823                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL);
2824                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2825                 break;
2826
2827         case FW_PORT_TYPE_CX4:
2828                 ifmedia_add(media, m | IFM_10G_CX4, data, NULL);
2829                 ifmedia_set(media, m | IFM_10G_CX4);
2830                 break;
2831
2832         case FW_PORT_TYPE_QSFP_10G:
2833         case FW_PORT_TYPE_SFP:
2834         case FW_PORT_TYPE_FIBER_XFI:
2835         case FW_PORT_TYPE_FIBER_XAUI:
2836                 switch (pi->mod_type) {
2837
2838                 case FW_PORT_MOD_TYPE_LR:
2839                         ifmedia_add(media, m | IFM_10G_LR, data, NULL);
2840                         ifmedia_set(media, m | IFM_10G_LR);
2841                         break;
2842
2843                 case FW_PORT_MOD_TYPE_SR:
2844                         ifmedia_add(media, m | IFM_10G_SR, data, NULL);
2845                         ifmedia_set(media, m | IFM_10G_SR);
2846                         break;
2847
2848                 case FW_PORT_MOD_TYPE_LRM:
2849                         ifmedia_add(media, m | IFM_10G_LRM, data, NULL);
2850                         ifmedia_set(media, m | IFM_10G_LRM);
2851                         break;
2852
2853                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2854                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2855                         ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL);
2856                         ifmedia_set(media, m | IFM_10G_TWINAX);
2857                         break;
2858
2859                 case FW_PORT_MOD_TYPE_NONE:
2860                         m &= ~IFM_FDX;
2861                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2862                         ifmedia_set(media, m | IFM_NONE);
2863                         break;
2864
2865                 case FW_PORT_MOD_TYPE_NA:
2866                 case FW_PORT_MOD_TYPE_ER:
2867                 default:
2868                         device_printf(pi->dev,
2869                             "unknown port_type (%d), mod_type (%d)\n",
2870                             pi->port_type, pi->mod_type);
2871                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2872                         ifmedia_set(media, m | IFM_UNKNOWN);
2873                         break;
2874                 }
2875                 break;
2876
2877         case FW_PORT_TYPE_QSFP:
2878                 switch (pi->mod_type) {
2879
2880                 case FW_PORT_MOD_TYPE_LR:
2881                         ifmedia_add(media, m | IFM_40G_LR4, data, NULL);
2882                         ifmedia_set(media, m | IFM_40G_LR4);
2883                         break;
2884
2885                 case FW_PORT_MOD_TYPE_SR:
2886                         ifmedia_add(media, m | IFM_40G_SR4, data, NULL);
2887                         ifmedia_set(media, m | IFM_40G_SR4);
2888                         break;
2889
2890                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2891                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2892                         ifmedia_add(media, m | IFM_40G_CR4, data, NULL);
2893                         ifmedia_set(media, m | IFM_40G_CR4);
2894                         break;
2895
2896                 case FW_PORT_MOD_TYPE_NONE:
2897                         m &= ~IFM_FDX;
2898                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2899                         ifmedia_set(media, m | IFM_NONE);
2900                         break;
2901
2902                 default:
2903                         device_printf(pi->dev,
2904                             "unknown port_type (%d), mod_type (%d)\n",
2905                             pi->port_type, pi->mod_type);
2906                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2907                         ifmedia_set(media, m | IFM_UNKNOWN);
2908                         break;
2909                 }
2910                 break;
2911
2912         default:
2913                 device_printf(pi->dev,
2914                     "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
2915                     pi->mod_type);
2916                 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2917                 ifmedia_set(media, m | IFM_UNKNOWN);
2918                 break;
2919         }
2920
2921         PORT_UNLOCK(pi);
2922 }
2923
2924 #define FW_MAC_EXACT_CHUNK      7
2925
2926 /*
2927  * Program the port's XGMAC based on parameters in ifnet.  The caller also
2928  * indicates which parameters should be programmed (the rest are left alone).
2929  */
2930 int
2931 update_mac_settings(struct ifnet *ifp, int flags)
2932 {
2933         int rc = 0;
2934         struct port_info *pi = ifp->if_softc;
2935         struct adapter *sc = pi->adapter;
2936         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
2937         uint16_t viid = 0xffff;
2938         int16_t *xact_addr_filt = NULL;
2939
2940         ASSERT_SYNCHRONIZED_OP(sc);
2941         KASSERT(flags, ("%s: not told what to update.", __func__));
2942
2943         if (ifp == pi->ifp) {
2944                 viid = pi->viid;
2945                 xact_addr_filt = &pi->xact_addr_filt;
2946         }
2947 #ifdef DEV_NETMAP
2948         else if (ifp == pi->nm_ifp) {
2949                 viid = pi->nm_viid;
2950                 xact_addr_filt = &pi->nm_xact_addr_filt;
2951         }
2952 #endif
2953         if (flags & XGMAC_MTU)
2954                 mtu = ifp->if_mtu;
2955
2956         if (flags & XGMAC_PROMISC)
2957                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
2958
2959         if (flags & XGMAC_ALLMULTI)
2960                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
2961
2962         if (flags & XGMAC_VLANEX)
2963                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
2964
2965         if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
2966                 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti,
2967                     1, vlanex, false);
2968                 if (rc) {
2969                         if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
2970                             rc);
2971                         return (rc);
2972                 }
2973         }
2974
2975         if (flags & XGMAC_UCADDR) {
2976                 uint8_t ucaddr[ETHER_ADDR_LEN];
2977
2978                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
2979                 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr,
2980                     true, true);
2981                 if (rc < 0) {
2982                         rc = -rc;
2983                         if_printf(ifp, "change_mac failed: %d\n", rc);
2984                         return (rc);
2985                 } else {
2986                         *xact_addr_filt = rc;
2987                         rc = 0;
2988                 }
2989         }
2990
2991         if (flags & XGMAC_MCADDRS) {
2992                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
2993                 int del = 1;
2994                 uint64_t hash = 0;
2995                 struct ifmultiaddr *ifma;
2996                 int i = 0, j;
2997
2998                 if_maddr_rlock(ifp);
2999                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
3000                         if (ifma->ifma_addr->sa_family != AF_LINK)
3001                                 continue;
3002                         mcaddr[i] =
3003                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
3004                         MPASS(ETHER_IS_MULTICAST(mcaddr[i]));
3005                         i++;
3006
3007                         if (i == FW_MAC_EXACT_CHUNK) {
3008                                 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del,
3009                                     i, mcaddr, NULL, &hash, 0);
3010                                 if (rc < 0) {
3011                                         rc = -rc;
3012                                         for (j = 0; j < i; j++) {
3013                                                 if_printf(ifp,
3014                                                     "failed to add mc address"
3015                                                     " %02x:%02x:%02x:"
3016                                                     "%02x:%02x:%02x rc=%d\n",
3017                                                     mcaddr[j][0], mcaddr[j][1],
3018                                                     mcaddr[j][2], mcaddr[j][3],
3019                                                     mcaddr[j][4], mcaddr[j][5],
3020                                                     rc);
3021                                         }
3022                                         goto mcfail;
3023                                 }
3024                                 del = 0;
3025                                 i = 0;
3026                         }
3027                 }
3028                 if (i > 0) {
3029                         rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i,
3030                             mcaddr, NULL, &hash, 0);
3031                         if (rc < 0) {
3032                                 rc = -rc;
3033                                 for (j = 0; j < i; j++) {
3034                                         if_printf(ifp,
3035                                             "failed to add mc address"
3036                                             " %02x:%02x:%02x:"
3037                                             "%02x:%02x:%02x rc=%d\n",
3038                                             mcaddr[j][0], mcaddr[j][1],
3039                                             mcaddr[j][2], mcaddr[j][3],
3040                                             mcaddr[j][4], mcaddr[j][5],
3041                                             rc);
3042                                 }
3043                                 goto mcfail;
3044                         }
3045                 }
3046
3047                 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0);
3048                 if (rc != 0)
3049                         if_printf(ifp, "failed to set mc address hash: %d", rc);
3050 mcfail:
3051                 if_maddr_runlock(ifp);
3052         }
3053
3054         return (rc);
3055 }
3056
3057 /*
3058  * {begin|end}_synchronized_op must be called from the same thread.
3059  */
3060 int
3061 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
3062     char *wmesg)
3063 {
3064         int rc, pri;
3065
3066 #ifdef WITNESS
3067         /* the caller thinks it's ok to sleep, but is it really? */
3068         if (flags & SLEEP_OK)
3069                 pause("t4slptst", 1);
3070 #endif
3071
3072         if (INTR_OK)
3073                 pri = PCATCH;
3074         else
3075                 pri = 0;
3076
3077         ADAPTER_LOCK(sc);
3078         for (;;) {
3079
3080                 if (pi && IS_DOOMED(pi)) {
3081                         rc = ENXIO;
3082                         goto done;
3083                 }
3084
3085                 if (!IS_BUSY(sc)) {
3086                         rc = 0;
3087                         break;
3088                 }
3089
3090                 if (!(flags & SLEEP_OK)) {
3091                         rc = EBUSY;
3092                         goto done;
3093                 }
3094
3095                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
3096                         rc = EINTR;
3097                         goto done;
3098                 }
3099         }
3100
3101         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
3102         SET_BUSY(sc);
3103 #ifdef INVARIANTS
3104         sc->last_op = wmesg;
3105         sc->last_op_thr = curthread;
3106 #endif
3107
3108 done:
3109         if (!(flags & HOLD_LOCK) || rc)
3110                 ADAPTER_UNLOCK(sc);
3111
3112         return (rc);
3113 }
3114
3115 /*
3116  * {begin|end}_synchronized_op must be called from the same thread.
3117  */
3118 void
3119 end_synchronized_op(struct adapter *sc, int flags)
3120 {
3121
3122         if (flags & LOCK_HELD)
3123                 ADAPTER_LOCK_ASSERT_OWNED(sc);
3124         else
3125                 ADAPTER_LOCK(sc);
3126
3127         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
3128         CLR_BUSY(sc);
3129         wakeup(&sc->flags);
3130         ADAPTER_UNLOCK(sc);
3131 }
3132
3133 static int
3134 cxgbe_init_synchronized(struct port_info *pi)
3135 {
3136         struct adapter *sc = pi->adapter;
3137         struct ifnet *ifp = pi->ifp;
3138         int rc = 0;
3139
3140         ASSERT_SYNCHRONIZED_OP(sc);
3141
3142         if (isset(&sc->open_device_map, pi->port_id)) {
3143                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
3144                     ("mismatch between open_device_map and if_drv_flags"));
3145                 return (0);     /* already running */
3146         }
3147
3148         if (!(sc->flags & FULL_INIT_DONE) &&
3149             ((rc = adapter_full_init(sc)) != 0))
3150                 return (rc);    /* error message displayed already */
3151
3152         if (!(pi->flags & PORT_INIT_DONE) &&
3153             ((rc = port_full_init(pi)) != 0))
3154                 return (rc); /* error message displayed already */
3155
3156         rc = update_mac_settings(ifp, XGMAC_ALL);
3157         if (rc)
3158                 goto done;      /* error message displayed already */
3159
3160         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
3161         if (rc != 0) {
3162                 if_printf(ifp, "enable_vi failed: %d\n", rc);
3163                 goto done;
3164         }
3165
3166         /*
3167          * The first iq of the first port to come up is used for tracing.
3168          */
3169         if (sc->traceq < 0) {
3170                 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
3171                 t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
3172                     A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
3173                     V_QUEUENUMBER(sc->traceq));
3174                 pi->flags |= HAS_TRACEQ;
3175         }
3176
3177         /* all ok */
3178         setbit(&sc->open_device_map, pi->port_id);
3179         PORT_LOCK(pi);
3180         ifp->if_drv_flags |= IFF_DRV_RUNNING;
3181         PORT_UNLOCK(pi);
3182
3183         callout_reset(&pi->tick, hz, cxgbe_tick, pi);
3184 done:
3185         if (rc != 0)
3186                 cxgbe_uninit_synchronized(pi);
3187
3188         return (rc);
3189 }
3190
3191 /*
3192  * Idempotent.
3193  */
3194 static int
3195 cxgbe_uninit_synchronized(struct port_info *pi)
3196 {
3197         struct adapter *sc = pi->adapter;
3198         struct ifnet *ifp = pi->ifp;
3199         int rc;
3200
3201         ASSERT_SYNCHRONIZED_OP(sc);
3202
3203         /*
3204          * Disable the VI so that all its data in either direction is discarded
3205          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
3206          * tick) intact as the TP can deliver negative advice or data that it's
3207          * holding in its RAM (for an offloaded connection) even after the VI is
3208          * disabled.
3209          */
3210         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
3211         if (rc) {
3212                 if_printf(ifp, "disable_vi failed: %d\n", rc);
3213                 return (rc);
3214         }
3215
3216         clrbit(&sc->open_device_map, pi->port_id);
3217         PORT_LOCK(pi);
3218         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3219         PORT_UNLOCK(pi);
3220
3221         pi->link_cfg.link_ok = 0;
3222         pi->link_cfg.speed = 0;
3223         pi->linkdnrc = -1;
3224         t4_os_link_changed(sc, pi->port_id, 0, -1);
3225
3226         return (0);
3227 }
3228
3229 /*
3230  * It is ok for this function to fail midway and return right away.  t4_detach
3231  * will walk the entire sc->irq list and clean up whatever is valid.
3232  */
3233 static int
3234 setup_intr_handlers(struct adapter *sc)
3235 {
3236         int rc, rid, p, q;
3237         char s[8];
3238         struct irq *irq;
3239         struct port_info *pi;
3240         struct sge_rxq *rxq;
3241 #ifdef TCP_OFFLOAD
3242         struct sge_ofld_rxq *ofld_rxq;
3243 #endif
3244 #ifdef DEV_NETMAP
3245         struct sge_nm_rxq *nm_rxq;
3246 #endif
3247
3248         /*
3249          * Setup interrupts.
3250          */
3251         irq = &sc->irq[0];
3252         rid = sc->intr_type == INTR_INTX ? 0 : 1;
3253         if (sc->intr_count == 1)
3254                 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
3255
3256         /* Multiple interrupts. */
3257         KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
3258             ("%s: too few intr.", __func__));
3259
3260         /* The first one is always error intr */
3261         rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
3262         if (rc != 0)
3263                 return (rc);
3264         irq++;
3265         rid++;
3266
3267         /* The second one is always the firmware event queue */
3268         rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
3269         if (rc != 0)
3270                 return (rc);
3271         irq++;
3272         rid++;
3273
3274         for_each_port(sc, p) {
3275                 pi = sc->port[p];
3276
3277                 if (pi->flags & INTR_RXQ) {
3278                         for_each_rxq(pi, q, rxq) {
3279                                 snprintf(s, sizeof(s), "%d.%d", p, q);
3280                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
3281                                     s);
3282                                 if (rc != 0)
3283                                         return (rc);
3284                                 irq++;
3285                                 rid++;
3286                         }
3287                 }
3288 #ifdef TCP_OFFLOAD
3289                 if (pi->flags & INTR_OFLD_RXQ) {
3290                         for_each_ofld_rxq(pi, q, ofld_rxq) {
3291                                 snprintf(s, sizeof(s), "%d,%d", p, q);
3292                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr,
3293                                     ofld_rxq, s);
3294                                 if (rc != 0)
3295                                         return (rc);
3296                                 irq++;
3297                                 rid++;
3298                         }
3299                 }
3300 #endif
3301 #ifdef DEV_NETMAP
3302                 if (pi->flags & INTR_NM_RXQ) {
3303                         for_each_nm_rxq(pi, q, nm_rxq) {
3304                                 snprintf(s, sizeof(s), "%d-%d", p, q);
3305                                 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr,
3306                                     nm_rxq, s);
3307                                 if (rc != 0)
3308                                         return (rc);
3309                                 irq++;
3310                                 rid++;
3311                         }
3312                 }
3313 #endif
3314         }
3315         MPASS(irq == &sc->irq[sc->intr_count]);
3316
3317         return (0);
3318 }
3319
3320 int
3321 adapter_full_init(struct adapter *sc)
3322 {
3323         int rc, i;
3324
3325         ASSERT_SYNCHRONIZED_OP(sc);
3326         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3327         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
3328             ("%s: FULL_INIT_DONE already", __func__));
3329
3330         /*
3331          * queues that belong to the adapter (not any particular port).
3332          */
3333         rc = t4_setup_adapter_queues(sc);
3334         if (rc != 0)
3335                 goto done;
3336
3337         for (i = 0; i < nitems(sc->tq); i++) {
3338                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
3339                     taskqueue_thread_enqueue, &sc->tq[i]);
3340                 if (sc->tq[i] == NULL) {
3341                         device_printf(sc->dev,
3342                             "failed to allocate task queue %d\n", i);
3343                         rc = ENOMEM;
3344                         goto done;
3345                 }
3346                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
3347                     device_get_nameunit(sc->dev), i);
3348         }
3349
3350         t4_intr_enable(sc);
3351         sc->flags |= FULL_INIT_DONE;
3352 done:
3353         if (rc != 0)
3354                 adapter_full_uninit(sc);
3355
3356         return (rc);
3357 }
3358
3359 int
3360 adapter_full_uninit(struct adapter *sc)
3361 {
3362         int i;
3363
3364         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3365
3366         t4_teardown_adapter_queues(sc);
3367
3368         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
3369                 taskqueue_free(sc->tq[i]);
3370                 sc->tq[i] = NULL;
3371         }
3372
3373         sc->flags &= ~FULL_INIT_DONE;
3374
3375         return (0);
3376 }
3377
3378 int
3379 port_full_init(struct port_info *pi)
3380 {
3381         struct adapter *sc = pi->adapter;
3382         struct ifnet *ifp = pi->ifp;
3383         uint16_t *rss;
3384         struct sge_rxq *rxq;
3385         int rc, i, j;
3386
3387         ASSERT_SYNCHRONIZED_OP(sc);
3388         KASSERT((pi->flags & PORT_INIT_DONE) == 0,
3389             ("%s: PORT_INIT_DONE already", __func__));
3390
3391         sysctl_ctx_init(&pi->ctx);
3392         pi->flags |= PORT_SYSCTL_CTX;
3393
3394         /*
3395          * Allocate tx/rx/fl queues for this port.
3396          */
3397         rc = t4_setup_port_queues(pi);
3398         if (rc != 0)
3399                 goto done;      /* error message displayed already */
3400
3401         /*
3402          * Setup RSS for this port.  Save a copy of the RSS table for later use.
3403          */
3404         rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
3405         for (i = 0; i < pi->rss_size;) {
3406                 for_each_rxq(pi, j, rxq) {
3407                         rss[i++] = rxq->iq.abs_id;
3408                         if (i == pi->rss_size)
3409                                 break;
3410                 }
3411         }
3412
3413         rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
3414             pi->rss_size);
3415         if (rc != 0) {
3416                 if_printf(ifp, "rss_config failed: %d\n", rc);
3417                 goto done;
3418         }
3419
3420         pi->rss = rss;
3421         pi->flags |= PORT_INIT_DONE;
3422 done:
3423         if (rc != 0)
3424                 port_full_uninit(pi);
3425
3426         return (rc);
3427 }
3428
3429 /*
3430  * Idempotent.
3431  */
3432 int
3433 port_full_uninit(struct port_info *pi)
3434 {
3435         struct adapter *sc = pi->adapter;
3436         int i;
3437         struct sge_rxq *rxq;
3438         struct sge_txq *txq;
3439 #ifdef TCP_OFFLOAD
3440         struct sge_ofld_rxq *ofld_rxq;
3441         struct sge_wrq *ofld_txq;
3442 #endif
3443
3444         if (pi->flags & PORT_INIT_DONE) {
3445
3446                 /* Need to quiesce queues.  XXX: ctrl queues? */
3447
3448                 for_each_txq(pi, i, txq) {
3449                         quiesce_eq(sc, &txq->eq);
3450                 }
3451
3452 #ifdef TCP_OFFLOAD
3453                 for_each_ofld_txq(pi, i, ofld_txq) {
3454                         quiesce_eq(sc, &ofld_txq->eq);
3455                 }
3456 #endif
3457
3458                 for_each_rxq(pi, i, rxq) {
3459                         quiesce_iq(sc, &rxq->iq);
3460                         quiesce_fl(sc, &rxq->fl);
3461                 }
3462
3463 #ifdef TCP_OFFLOAD
3464                 for_each_ofld_rxq(pi, i, ofld_rxq) {
3465                         quiesce_iq(sc, &ofld_rxq->iq);
3466                         quiesce_fl(sc, &ofld_rxq->fl);
3467                 }
3468 #endif
3469                 free(pi->rss, M_CXGBE);
3470         }
3471
3472         t4_teardown_port_queues(pi);
3473         pi->flags &= ~PORT_INIT_DONE;
3474
3475         return (0);
3476 }
3477
3478 static void
3479 quiesce_eq(struct adapter *sc, struct sge_eq *eq)
3480 {
3481         EQ_LOCK(eq);
3482         eq->flags |= EQ_DOOMED;
3483
3484         /*
3485          * Wait for the response to a credit flush if one's
3486          * pending.
3487          */
3488         while (eq->flags & EQ_CRFLUSHED)
3489                 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0);
3490         EQ_UNLOCK(eq);
3491
3492         callout_drain(&eq->tx_callout); /* XXX: iffy */
3493         pause("callout", 10);           /* Still iffy */
3494
3495         taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task);
3496 }
3497
3498 static void
3499 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
3500 {
3501         (void) sc;      /* unused */
3502
3503         /* Synchronize with the interrupt handler */
3504         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
3505                 pause("iqfree", 1);
3506 }
3507
3508 static void
3509 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
3510 {
3511         mtx_lock(&sc->sfl_lock);
3512         FL_LOCK(fl);
3513         fl->flags |= FL_DOOMED;
3514         FL_UNLOCK(fl);
3515         mtx_unlock(&sc->sfl_lock);
3516
3517         callout_drain(&sc->sfl_callout);
3518         KASSERT((fl->flags & FL_STARVING) == 0,
3519             ("%s: still starving", __func__));
3520 }
3521
3522 static int
3523 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
3524     driver_intr_t *handler, void *arg, char *name)
3525 {
3526         int rc;
3527
3528         irq->rid = rid;
3529         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
3530             RF_SHAREABLE | RF_ACTIVE);
3531         if (irq->res == NULL) {
3532                 device_printf(sc->dev,
3533                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
3534                 return (ENOMEM);
3535         }
3536
3537         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
3538             NULL, handler, arg, &irq->tag);
3539         if (rc != 0) {
3540                 device_printf(sc->dev,
3541                     "failed to setup interrupt for rid %d, name %s: %d\n",
3542                     rid, name, rc);
3543         } else if (name)
3544                 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
3545
3546         return (rc);
3547 }
3548
3549 static int
3550 t4_free_irq(struct adapter *sc, struct irq *irq)
3551 {
3552         if (irq->tag)
3553                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
3554         if (irq->res)
3555                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
3556
3557         bzero(irq, sizeof(*irq));
3558
3559         return (0);
3560 }
3561
3562 static void
3563 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
3564     unsigned int end)
3565 {
3566         uint32_t *p = (uint32_t *)(buf + start);
3567
3568         for ( ; start <= end; start += sizeof(uint32_t))
3569                 *p++ = t4_read_reg(sc, start);
3570 }
3571
3572 static void
3573 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
3574 {
3575         int i, n;
3576         const unsigned int *reg_ranges;
3577         static const unsigned int t4_reg_ranges[] = {
3578                 0x1008, 0x1108,
3579                 0x1180, 0x11b4,
3580                 0x11fc, 0x123c,
3581                 0x1300, 0x173c,
3582                 0x1800, 0x18fc,
3583                 0x3000, 0x30d8,
3584                 0x30e0, 0x5924,
3585                 0x5960, 0x59d4,
3586                 0x5a00, 0x5af8,
3587                 0x6000, 0x6098,
3588                 0x6100, 0x6150,
3589                 0x6200, 0x6208,
3590                 0x6240, 0x6248,
3591                 0x6280, 0x6338,
3592                 0x6370, 0x638c,
3593                 0x6400, 0x643c,
3594                 0x6500, 0x6524,
3595                 0x6a00, 0x6a38,
3596                 0x6a60, 0x6a78,
3597                 0x6b00, 0x6b84,
3598                 0x6bf0, 0x6c84,
3599                 0x6cf0, 0x6d84,
3600                 0x6df0, 0x6e84,
3601                 0x6ef0, 0x6f84,
3602                 0x6ff0, 0x7084,
3603                 0x70f0, 0x7184,
3604                 0x71f0, 0x7284,
3605                 0x72f0, 0x7384,
3606                 0x73f0, 0x7450,
3607                 0x7500, 0x7530,
3608                 0x7600, 0x761c,
3609                 0x7680, 0x76cc,
3610                 0x7700, 0x7798,
3611                 0x77c0, 0x77fc,
3612                 0x7900, 0x79fc,
3613                 0x7b00, 0x7c38,
3614                 0x7d00, 0x7efc,
3615                 0x8dc0, 0x8e1c,
3616                 0x8e30, 0x8e78,
3617                 0x8ea0, 0x8f6c,
3618                 0x8fc0, 0x9074,
3619                 0x90fc, 0x90fc,
3620                 0x9400, 0x9458,
3621                 0x9600, 0x96bc,
3622                 0x9800, 0x9808,
3623                 0x9820, 0x983c,
3624                 0x9850, 0x9864,
3625                 0x9c00, 0x9c6c,
3626                 0x9c80, 0x9cec,
3627                 0x9d00, 0x9d6c,
3628                 0x9d80, 0x9dec,
3629                 0x9e00, 0x9e6c,
3630                 0x9e80, 0x9eec,
3631                 0x9f00, 0x9f6c,
3632                 0x9f80, 0x9fec,
3633                 0xd004, 0xd03c,
3634                 0xdfc0, 0xdfe0,
3635                 0xe000, 0xea7c,
3636                 0xf000, 0x11110,
3637                 0x11118, 0x11190,
3638                 0x19040, 0x1906c,
3639                 0x19078, 0x19080,
3640                 0x1908c, 0x19124,
3641                 0x19150, 0x191b0,
3642                 0x191d0, 0x191e8,
3643                 0x19238, 0x1924c,
3644                 0x193f8, 0x19474,
3645                 0x19490, 0x194f8,
3646                 0x19800, 0x19f30,
3647                 0x1a000, 0x1a06c,
3648                 0x1a0b0, 0x1a120,
3649                 0x1a128, 0x1a138,
3650                 0x1a190, 0x1a1c4,
3651                 0x1a1fc, 0x1a1fc,
3652                 0x1e040, 0x1e04c,
3653                 0x1e284, 0x1e28c,
3654                 0x1e2c0, 0x1e2c0,
3655                 0x1e2e0, 0x1e2e0,
3656                 0x1e300, 0x1e384,
3657                 0x1e3c0, 0x1e3c8,
3658                 0x1e440, 0x1e44c,
3659                 0x1e684, 0x1e68c,
3660                 0x1e6c0, 0x1e6c0,
3661                 0x1e6e0, 0x1e6e0,
3662                 0x1e700, 0x1e784,
3663                 0x1e7c0, 0x1e7c8,
3664                 0x1e840, 0x1e84c,
3665                 0x1ea84, 0x1ea8c,
3666                 0x1eac0, 0x1eac0,
3667                 0x1eae0, 0x1eae0,
3668                 0x1eb00, 0x1eb84,
3669                 0x1ebc0, 0x1ebc8,
3670                 0x1ec40, 0x1ec4c,
3671                 0x1ee84, 0x1ee8c,
3672                 0x1eec0, 0x1eec0,
3673                 0x1eee0, 0x1eee0,
3674                 0x1ef00, 0x1ef84,
3675                 0x1efc0, 0x1efc8,
3676                 0x1f040, 0x1f04c,
3677                 0x1f284, 0x1f28c,
3678                 0x1f2c0, 0x1f2c0,
3679                 0x1f2e0, 0x1f2e0,
3680                 0x1f300, 0x1f384,
3681                 0x1f3c0, 0x1f3c8,
3682                 0x1f440, 0x1f44c,
3683                 0x1f684, 0x1f68c,
3684                 0x1f6c0, 0x1f6c0,
3685                 0x1f6e0, 0x1f6e0,
3686                 0x1f700, 0x1f784,
3687                 0x1f7c0, 0x1f7c8,
3688                 0x1f840, 0x1f84c,
3689                 0x1fa84, 0x1fa8c,
3690                 0x1fac0, 0x1fac0,
3691                 0x1fae0, 0x1fae0,
3692                 0x1fb00, 0x1fb84,
3693                 0x1fbc0, 0x1fbc8,
3694                 0x1fc40, 0x1fc4c,
3695                 0x1fe84, 0x1fe8c,
3696                 0x1fec0, 0x1fec0,
3697                 0x1fee0, 0x1fee0,
3698                 0x1ff00, 0x1ff84,
3699                 0x1ffc0, 0x1ffc8,
3700                 0x20000, 0x2002c,
3701                 0x20100, 0x2013c,
3702                 0x20190, 0x201c8,
3703                 0x20200, 0x20318,
3704                 0x20400, 0x20528,
3705                 0x20540, 0x20614,
3706                 0x21000, 0x21040,
3707                 0x2104c, 0x21060,
3708                 0x210c0, 0x210ec,
3709                 0x21200, 0x21268,
3710                 0x21270, 0x21284,
3711                 0x212fc, 0x21388,
3712                 0x21400, 0x21404,
3713                 0x21500, 0x21518,
3714                 0x2152c, 0x2153c,
3715                 0x21550, 0x21554,
3716                 0x21600, 0x21600,
3717                 0x21608, 0x21628,
3718                 0x21630, 0x2163c,
3719                 0x21700, 0x2171c,
3720                 0x21780, 0x2178c,
3721                 0x21800, 0x21c38,
3722                 0x21c80, 0x21d7c,
3723                 0x21e00, 0x21e04,
3724                 0x22000, 0x2202c,
3725                 0x22100, 0x2213c,
3726                 0x22190, 0x221c8,
3727                 0x22200, 0x22318,
3728                 0x22400, 0x22528,
3729                 0x22540, 0x22614,
3730                 0x23000, 0x23040,
3731                 0x2304c, 0x23060,
3732                 0x230c0, 0x230ec,
3733                 0x23200, 0x23268,
3734                 0x23270, 0x23284,
3735                 0x232fc, 0x23388,
3736                 0x23400, 0x23404,
3737                 0x23500, 0x23518,
3738                 0x2352c, 0x2353c,
3739                 0x23550, 0x23554,
3740                 0x23600, 0x23600,
3741                 0x23608, 0x23628,
3742                 0x23630, 0x2363c,
3743                 0x23700, 0x2371c,
3744                 0x23780, 0x2378c,
3745                 0x23800, 0x23c38,
3746                 0x23c80, 0x23d7c,
3747                 0x23e00, 0x23e04,
3748                 0x24000, 0x2402c,
3749                 0x24100, 0x2413c,
3750                 0x24190, 0x241c8,
3751                 0x24200, 0x24318,
3752                 0x24400, 0x24528,
3753                 0x24540, 0x24614,
3754                 0x25000, 0x25040,
3755                 0x2504c, 0x25060,
3756                 0x250c0, 0x250ec,
3757                 0x25200, 0x25268,
3758                 0x25270, 0x25284,
3759                 0x252fc, 0x25388,
3760                 0x25400, 0x25404,
3761                 0x25500, 0x25518,
3762                 0x2552c, 0x2553c,
3763                 0x25550, 0x25554,
3764                 0x25600, 0x25600,
3765                 0x25608, 0x25628,
3766                 0x25630, 0x2563c,
3767                 0x25700, 0x2571c,
3768                 0x25780, 0x2578c,
3769                 0x25800, 0x25c38,
3770                 0x25c80, 0x25d7c,
3771                 0x25e00, 0x25e04,
3772                 0x26000, 0x2602c,
3773                 0x26100, 0x2613c,
3774                 0x26190, 0x261c8,
3775                 0x26200, 0x26318,
3776                 0x26400, 0x26528,
3777                 0x26540, 0x26614,
3778                 0x27000, 0x27040,
3779                 0x2704c, 0x27060,
3780                 0x270c0, 0x270ec,
3781                 0x27200, 0x27268,
3782                 0x27270, 0x27284,
3783                 0x272fc, 0x27388,
3784                 0x27400, 0x27404,
3785                 0x27500, 0x27518,
3786                 0x2752c, 0x2753c,
3787                 0x27550, 0x27554,
3788                 0x27600, 0x27600,
3789                 0x27608, 0x27628,
3790                 0x27630, 0x2763c,
3791                 0x27700, 0x2771c,
3792                 0x27780, 0x2778c,
3793                 0x27800, 0x27c38,
3794                 0x27c80, 0x27d7c,
3795                 0x27e00, 0x27e04
3796         };
3797         static const unsigned int t5_reg_ranges[] = {
3798                 0x1008, 0x1148,
3799                 0x1180, 0x11b4,
3800                 0x11fc, 0x123c,
3801                 0x1280, 0x173c,
3802                 0x1800, 0x18fc,
3803                 0x3000, 0x3028,
3804                 0x3060, 0x30d8,
3805                 0x30e0, 0x30fc,
3806                 0x3140, 0x357c,
3807                 0x35a8, 0x35cc,
3808                 0x35ec, 0x35ec,
3809                 0x3600, 0x5624,
3810                 0x56cc, 0x575c,
3811                 0x580c, 0x5814,
3812                 0x5890, 0x58bc,
3813                 0x5940, 0x59dc,
3814                 0x59fc, 0x5a18,
3815                 0x5a60, 0x5a9c,
3816                 0x5b94, 0x5bfc,
3817                 0x6000, 0x6040,
3818                 0x6058, 0x614c,
3819                 0x7700, 0x7798,
3820                 0x77c0, 0x78fc,
3821                 0x7b00, 0x7c54,
3822                 0x7d00, 0x7efc,
3823                 0x8dc0, 0x8de0,
3824                 0x8df8, 0x8e84,
3825                 0x8ea0, 0x8f84,
3826                 0x8fc0, 0x90f8,
3827                 0x9400, 0x9470,
3828                 0x9600, 0x96f4,
3829                 0x9800, 0x9808,
3830                 0x9820, 0x983c,
3831                 0x9850, 0x9864,
3832                 0x9c00, 0x9c6c,
3833                 0x9c80, 0x9cec,
3834                 0x9d00, 0x9d6c,
3835                 0x9d80, 0x9dec,
3836                 0x9e00, 0x9e6c,
3837                 0x9e80, 0x9eec,
3838                 0x9f00, 0x9f6c,
3839                 0x9f80, 0xa020,
3840                 0xd004, 0xd03c,
3841                 0xdfc0, 0xdfe0,
3842                 0xe000, 0x11088,
3843                 0x1109c, 0x11110,
3844                 0x11118, 0x1117c,
3845                 0x11190, 0x11204,
3846                 0x19040, 0x1906c,
3847                 0x19078, 0x19080,
3848                 0x1908c, 0x19124,
3849                 0x19150, 0x191b0,
3850                 0x191d0, 0x191e8,
3851                 0x19238, 0x19290,
3852                 0x193f8, 0x19474,
3853                 0x19490, 0x194cc,
3854                 0x194f0, 0x194f8,
3855                 0x19c00, 0x19c60,
3856                 0x19c94, 0x19e10,
3857                 0x19e50, 0x19f34,
3858                 0x19f40, 0x19f50,
3859                 0x19f90, 0x19fe4,
3860                 0x1a000, 0x1a06c,
3861                 0x1a0b0, 0x1a120,
3862                 0x1a128, 0x1a138,
3863                 0x1a190, 0x1a1c4,
3864                 0x1a1fc, 0x1a1fc,
3865                 0x1e008, 0x1e00c,
3866                 0x1e040, 0x1e04c,
3867                 0x1e284, 0x1e290,
3868                 0x1e2c0, 0x1e2c0,
3869                 0x1e2e0, 0x1e2e0,
3870                 0x1e300, 0x1e384,
3871                 0x1e3c0, 0x1e3c8,
3872                 0x1e408, 0x1e40c,
3873                 0x1e440, 0x1e44c,
3874                 0x1e684, 0x1e690,
3875                 0x1e6c0, 0x1e6c0,
3876                 0x1e6e0, 0x1e6e0,
3877                 0x1e700, 0x1e784,
3878                 0x1e7c0, 0x1e7c8,
3879                 0x1e808, 0x1e80c,
3880                 0x1e840, 0x1e84c,
3881                 0x1ea84, 0x1ea90,
3882                 0x1eac0, 0x1eac0,
3883                 0x1eae0, 0x1eae0,
3884                 0x1eb00, 0x1eb84,
3885                 0x1ebc0, 0x1ebc8,
3886                 0x1ec08, 0x1ec0c,
3887                 0x1ec40, 0x1ec4c,
3888                 0x1ee84, 0x1ee90,
3889                 0x1eec0, 0x1eec0,
3890                 0x1eee0, 0x1eee0,
3891                 0x1ef00, 0x1ef84,
3892                 0x1efc0, 0x1efc8,
3893                 0x1f008, 0x1f00c,
3894                 0x1f040, 0x1f04c,
3895                 0x1f284, 0x1f290,
3896                 0x1f2c0, 0x1f2c0,
3897                 0x1f2e0, 0x1f2e0,
3898                 0x1f300, 0x1f384,
3899                 0x1f3c0, 0x1f3c8,
3900                 0x1f408, 0x1f40c,
3901                 0x1f440, 0x1f44c,
3902                 0x1f684, 0x1f690,
3903                 0x1f6c0, 0x1f6c0,
3904                 0x1f6e0, 0x1f6e0,
3905                 0x1f700, 0x1f784,
3906                 0x1f7c0, 0x1f7c8,
3907                 0x1f808, 0x1f80c,
3908                 0x1f840, 0x1f84c,
3909                 0x1fa84, 0x1fa90,
3910                 0x1fac0, 0x1fac0,
3911                 0x1fae0, 0x1fae0,
3912                 0x1fb00, 0x1fb84,
3913                 0x1fbc0, 0x1fbc8,
3914                 0x1fc08, 0x1fc0c,
3915                 0x1fc40, 0x1fc4c,
3916                 0x1fe84, 0x1fe90,
3917                 0x1fec0, 0x1fec0,
3918                 0x1fee0, 0x1fee0,
3919                 0x1ff00, 0x1ff84,
3920                 0x1ffc0, 0x1ffc8,
3921                 0x30000, 0x30030,
3922                 0x30100, 0x30144,
3923                 0x30190, 0x301d0,
3924                 0x30200, 0x30318,
3925                 0x30400, 0x3052c,
3926                 0x30540, 0x3061c,
3927                 0x30800, 0x30834,
3928                 0x308c0, 0x30908,
3929                 0x30910, 0x309ac,
3930                 0x30a00, 0x30a2c,
3931                 0x30a44, 0x30a50,
3932                 0x30a74, 0x30c24,
3933                 0x30d00, 0x30d00,
3934                 0x30d08, 0x30d14,
3935                 0x30d1c, 0x30d20,
3936                 0x30d3c, 0x30d50,
3937                 0x31200, 0x3120c,
3938                 0x31220, 0x31220,
3939                 0x31240, 0x31240,
3940                 0x31600, 0x3160c,
3941                 0x31a00, 0x31a1c,
3942                 0x31e00, 0x31e20,
3943                 0x31e38, 0x31e3c,
3944                 0x31e80, 0x31e80,
3945                 0x31e88, 0x31ea8,
3946                 0x31eb0, 0x31eb4,
3947                 0x31ec8, 0x31ed4,
3948                 0x31fb8, 0x32004,
3949                 0x32200, 0x32200,
3950                 0x32208, 0x32240,
3951                 0x32248, 0x32280,
3952                 0x32288, 0x322c0,
3953                 0x322c8, 0x322fc,
3954                 0x32600, 0x32630,
3955                 0x32a00, 0x32abc,
3956                 0x32b00, 0x32b70,
3957                 0x33000, 0x33048,
3958                 0x33060, 0x3309c,
3959                 0x330f0, 0x33148,
3960                 0x33160, 0x3319c,
3961                 0x331f0, 0x332e4,
3962                 0x332f8, 0x333e4,
3963                 0x333f8, 0x33448,
3964                 0x33460, 0x3349c,
3965                 0x334f0, 0x33548,
3966                 0x33560, 0x3359c,
3967                 0x335f0, 0x336e4,
3968                 0x336f8, 0x337e4,
3969                 0x337f8, 0x337fc,
3970                 0x33814, 0x33814,
3971                 0x3382c, 0x3382c,
3972                 0x33880, 0x3388c,
3973                 0x338e8, 0x338ec,
3974                 0x33900, 0x33948,
3975                 0x33960, 0x3399c,
3976                 0x339f0, 0x33ae4,
3977                 0x33af8, 0x33b10,
3978                 0x33b28, 0x33b28,
3979                 0x33b3c, 0x33b50,
3980                 0x33bf0, 0x33c10,
3981                 0x33c28, 0x33c28,
3982                 0x33c3c, 0x33c50,
3983                 0x33cf0, 0x33cfc,
3984                 0x34000, 0x34030,
3985                 0x34100, 0x34144,
3986                 0x34190, 0x341d0,
3987                 0x34200, 0x34318,
3988                 0x34400, 0x3452c,
3989                 0x34540, 0x3461c,
3990                 0x34800, 0x34834,
3991                 0x348c0, 0x34908,
3992                 0x34910, 0x349ac,
3993                 0x34a00, 0x34a2c,
3994                 0x34a44, 0x34a50,
3995                 0x34a74, 0x34c24,
3996                 0x34d00, 0x34d00,
3997                 0x34d08, 0x34d14,
3998                 0x34d1c, 0x34d20,
3999                 0x34d3c, 0x34d50,
4000                 0x35200, 0x3520c,
4001                 0x35220, 0x35220,
4002                 0x35240, 0x35240,
4003                 0x35600, 0x3560c,
4004                 0x35a00, 0x35a1c,
4005                 0x35e00, 0x35e20,
4006                 0x35e38, 0x35e3c,
4007                 0x35e80, 0x35e80,
4008                 0x35e88, 0x35ea8,
4009                 0x35eb0, 0x35eb4,
4010                 0x35ec8, 0x35ed4,
4011                 0x35fb8, 0x36004,
4012                 0x36200, 0x36200,
4013                 0x36208, 0x36240,
4014                 0x36248, 0x36280,
4015                 0x36288, 0x362c0,
4016                 0x362c8, 0x362fc,
4017                 0x36600, 0x36630,
4018                 0x36a00, 0x36abc,
4019                 0x36b00, 0x36b70,
4020                 0x37000, 0x37048,
4021                 0x37060, 0x3709c,
4022                 0x370f0, 0x37148,
4023                 0x37160, 0x3719c,
4024                 0x371f0, 0x372e4,
4025                 0x372f8, 0x373e4,
4026                 0x373f8, 0x37448,
4027                 0x37460, 0x3749c,
4028                 0x374f0, 0x37548,
4029                 0x37560, 0x3759c,
4030                 0x375f0, 0x376e4,
4031                 0x376f8, 0x377e4,
4032                 0x377f8, 0x377fc,
4033                 0x37814, 0x37814,
4034                 0x3782c, 0x3782c,
4035                 0x37880, 0x3788c,
4036                 0x378e8, 0x378ec,
4037                 0x37900, 0x37948,
4038                 0x37960, 0x3799c,
4039                 0x379f0, 0x37ae4,
4040                 0x37af8, 0x37b10,
4041                 0x37b28, 0x37b28,
4042                 0x37b3c, 0x37b50,
4043                 0x37bf0, 0x37c10,
4044                 0x37c28, 0x37c28,
4045                 0x37c3c, 0x37c50,
4046                 0x37cf0, 0x37cfc,
4047                 0x38000, 0x38030,
4048                 0x38100, 0x38144,
4049                 0x38190, 0x381d0,
4050                 0x38200, 0x38318,
4051                 0x38400, 0x3852c,
4052                 0x38540, 0x3861c,
4053                 0x38800, 0x38834,
4054                 0x388c0, 0x38908,
4055                 0x38910, 0x389ac,
4056                 0x38a00, 0x38a2c,
4057                 0x38a44, 0x38a50,
4058                 0x38a74, 0x38c24,
4059                 0x38d00, 0x38d00,
4060                 0x38d08, 0x38d14,
4061                 0x38d1c, 0x38d20,
4062                 0x38d3c, 0x38d50,
4063                 0x39200, 0x3920c,
4064                 0x39220, 0x39220,
4065                 0x39240, 0x39240,
4066                 0x39600, 0x3960c,
4067                 0x39a00, 0x39a1c,
4068                 0x39e00, 0x39e20,
4069                 0x39e38, 0x39e3c,
4070                 0x39e80, 0x39e80,
4071                 0x39e88, 0x39ea8,
4072                 0x39eb0, 0x39eb4,
4073                 0x39ec8, 0x39ed4,
4074                 0x39fb8, 0x3a004,
4075                 0x3a200, 0x3a200,
4076                 0x3a208, 0x3a240,
4077                 0x3a248, 0x3a280,
4078                 0x3a288, 0x3a2c0,
4079                 0x3a2c8, 0x3a2fc,
4080                 0x3a600, 0x3a630,
4081                 0x3aa00, 0x3aabc,
4082                 0x3ab00, 0x3ab70,
4083                 0x3b000, 0x3b048,
4084                 0x3b060, 0x3b09c,
4085                 0x3b0f0, 0x3b148,
4086                 0x3b160, 0x3b19c,
4087                 0x3b1f0, 0x3b2e4,
4088                 0x3b2f8, 0x3b3e4,
4089                 0x3b3f8, 0x3b448,
4090                 0x3b460, 0x3b49c,
4091                 0x3b4f0, 0x3b548,
4092                 0x3b560, 0x3b59c,
4093                 0x3b5f0, 0x3b6e4,
4094                 0x3b6f8, 0x3b7e4,
4095                 0x3b7f8, 0x3b7fc,
4096                 0x3b814, 0x3b814,
4097                 0x3b82c, 0x3b82c,
4098                 0x3b880, 0x3b88c,
4099                 0x3b8e8, 0x3b8ec,
4100                 0x3b900, 0x3b948,
4101                 0x3b960, 0x3b99c,
4102                 0x3b9f0, 0x3bae4,
4103                 0x3baf8, 0x3bb10,
4104                 0x3bb28, 0x3bb28,
4105                 0x3bb3c, 0x3bb50,
4106                 0x3bbf0, 0x3bc10,
4107                 0x3bc28, 0x3bc28,
4108                 0x3bc3c, 0x3bc50,
4109                 0x3bcf0, 0x3bcfc,
4110                 0x3c000, 0x3c030,
4111                 0x3c100, 0x3c144,
4112                 0x3c190, 0x3c1d0,
4113                 0x3c200, 0x3c318,
4114                 0x3c400, 0x3c52c,
4115                 0x3c540, 0x3c61c,
4116                 0x3c800, 0x3c834,
4117                 0x3c8c0, 0x3c908,
4118                 0x3c910, 0x3c9ac,
4119                 0x3ca00, 0x3ca2c,
4120                 0x3ca44, 0x3ca50,
4121                 0x3ca74, 0x3cc24,
4122                 0x3cd00, 0x3cd00,
4123                 0x3cd08, 0x3cd14,
4124                 0x3cd1c, 0x3cd20,
4125                 0x3cd3c, 0x3cd50,
4126                 0x3d200, 0x3d20c,
4127                 0x3d220, 0x3d220,
4128                 0x3d240, 0x3d240,
4129                 0x3d600, 0x3d60c,
4130                 0x3da00, 0x3da1c,
4131                 0x3de00, 0x3de20,
4132                 0x3de38, 0x3de3c,
4133                 0x3de80, 0x3de80,
4134                 0x3de88, 0x3dea8,
4135                 0x3deb0, 0x3deb4,
4136                 0x3dec8, 0x3ded4,
4137                 0x3dfb8, 0x3e004,
4138                 0x3e200, 0x3e200,
4139                 0x3e208, 0x3e240,
4140                 0x3e248, 0x3e280,
4141                 0x3e288, 0x3e2c0,
4142                 0x3e2c8, 0x3e2fc,
4143                 0x3e600, 0x3e630,
4144                 0x3ea00, 0x3eabc,
4145                 0x3eb00, 0x3eb70,
4146                 0x3f000, 0x3f048,
4147                 0x3f060, 0x3f09c,
4148                 0x3f0f0, 0x3f148,
4149                 0x3f160, 0x3f19c,
4150                 0x3f1f0, 0x3f2e4,
4151                 0x3f2f8, 0x3f3e4,
4152                 0x3f3f8, 0x3f448,
4153                 0x3f460, 0x3f49c,
4154                 0x3f4f0, 0x3f548,
4155                 0x3f560, 0x3f59c,
4156                 0x3f5f0, 0x3f6e4,
4157                 0x3f6f8, 0x3f7e4,
4158                 0x3f7f8, 0x3f7fc,
4159                 0x3f814, 0x3f814,
4160                 0x3f82c, 0x3f82c,
4161                 0x3f880, 0x3f88c,
4162                 0x3f8e8, 0x3f8ec,
4163                 0x3f900, 0x3f948,
4164                 0x3f960, 0x3f99c,
4165                 0x3f9f0, 0x3fae4,
4166                 0x3faf8, 0x3fb10,
4167                 0x3fb28, 0x3fb28,
4168                 0x3fb3c, 0x3fb50,
4169                 0x3fbf0, 0x3fc10,
4170                 0x3fc28, 0x3fc28,
4171                 0x3fc3c, 0x3fc50,
4172                 0x3fcf0, 0x3fcfc,
4173                 0x40000, 0x4000c,
4174                 0x40040, 0x40068,
4175                 0x4007c, 0x40144,
4176                 0x40180, 0x4018c,
4177                 0x40200, 0x40298,
4178                 0x402ac, 0x4033c,
4179                 0x403f8, 0x403fc,
4180                 0x41304, 0x413c4,
4181                 0x41400, 0x4141c,
4182                 0x41480, 0x414d0,
4183                 0x44000, 0x44078,
4184                 0x440c0, 0x44278,
4185                 0x442c0, 0x44478,
4186                 0x444c0, 0x44678,
4187                 0x446c0, 0x44878,
4188                 0x448c0, 0x449fc,
4189                 0x45000, 0x45068,
4190                 0x45080, 0x45084,
4191                 0x450a0, 0x450b0,
4192                 0x45200, 0x45268,
4193                 0x45280, 0x45284,
4194                 0x452a0, 0x452b0,
4195                 0x460c0, 0x460e4,
4196                 0x47000, 0x4708c,
4197                 0x47200, 0x47250,
4198                 0x47400, 0x47420,
4199                 0x47600, 0x47618,
4200                 0x47800, 0x47814,
4201                 0x48000, 0x4800c,
4202                 0x48040, 0x48068,
4203                 0x4807c, 0x48144,
4204                 0x48180, 0x4818c,
4205                 0x48200, 0x48298,
4206                 0x482ac, 0x4833c,
4207                 0x483f8, 0x483fc,
4208                 0x49304, 0x493c4,
4209                 0x49400, 0x4941c,
4210                 0x49480, 0x494d0,
4211                 0x4c000, 0x4c078,
4212                 0x4c0c0, 0x4c278,
4213                 0x4c2c0, 0x4c478,
4214                 0x4c4c0, 0x4c678,
4215                 0x4c6c0, 0x4c878,
4216                 0x4c8c0, 0x4c9fc,
4217                 0x4d000, 0x4d068,
4218                 0x4d080, 0x4d084,
4219                 0x4d0a0, 0x4d0b0,
4220                 0x4d200, 0x4d268,
4221                 0x4d280, 0x4d284,
4222                 0x4d2a0, 0x4d2b0,
4223                 0x4e0c0, 0x4e0e4,
4224                 0x4f000, 0x4f08c,
4225                 0x4f200, 0x4f250,
4226                 0x4f400, 0x4f420,
4227                 0x4f600, 0x4f618,
4228                 0x4f800, 0x4f814,
4229                 0x50000, 0x500cc,
4230                 0x50400, 0x50400,
4231                 0x50800, 0x508cc,
4232                 0x50c00, 0x50c00,
4233                 0x51000, 0x5101c,
4234                 0x51300, 0x51308,
4235         };
4236
4237         if (is_t4(sc)) {
4238                 reg_ranges = &t4_reg_ranges[0];
4239                 n = nitems(t4_reg_ranges);
4240         } else {
4241                 reg_ranges = &t5_reg_ranges[0];
4242                 n = nitems(t5_reg_ranges);
4243         }
4244
4245         regs->version = chip_id(sc) | chip_rev(sc) << 10;
4246         for (i = 0; i < n; i += 2)
4247                 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
4248 }
4249
4250 static void
4251 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi)
4252 {
4253         struct ifnet *ifp = pi->ifp;
4254         struct sge_txq *txq;
4255         int i, drops;
4256         struct port_stats *s = &pi->stats;
4257         struct timeval tv;
4258         const struct timeval interval = {0, 250000};    /* 250ms */
4259
4260         getmicrotime(&tv);
4261         timevalsub(&tv, &interval);
4262         if (timevalcmp(&tv, &pi->last_refreshed, <))
4263                 return;
4264
4265         t4_get_port_stats(sc, pi->tx_chan, s);
4266
4267         ifp->if_opackets = s->tx_frames - s->tx_pause;
4268         ifp->if_ipackets = s->rx_frames - s->rx_pause;
4269         ifp->if_obytes = s->tx_octets - s->tx_pause * 64;
4270         ifp->if_ibytes = s->rx_octets - s->rx_pause * 64;
4271         ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause;
4272         ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause;
4273         ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
4274             s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
4275             s->rx_trunc3;
4276         for (i = 0; i < NCHAN; i++) {
4277                 if (pi->rx_chan_map & (1 << i)) {
4278                         uint32_t v;
4279
4280                         mtx_lock(&sc->regwin_lock);
4281                         t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
4282                             1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4283                         mtx_unlock(&sc->regwin_lock);
4284                         ifp->if_iqdrops += v;
4285                 }
4286         }
4287
4288         drops = s->tx_drop;
4289         for_each_txq(pi, i, txq)
4290                 drops += txq->br->br_drops;
4291         ifp->if_snd.ifq_drops = drops;
4292
4293         ifp->if_oerrors = s->tx_error_frames;
4294         ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long +
4295             s->rx_fcs_err + s->rx_len_err;
4296
4297         getmicrotime(&pi->last_refreshed);
4298 }
4299
4300 static void
4301 cxgbe_tick(void *arg)
4302 {
4303         struct port_info *pi = arg;
4304         struct adapter *sc = pi->adapter;
4305         struct ifnet *ifp = pi->ifp;
4306
4307         PORT_LOCK(pi);
4308         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4309                 PORT_UNLOCK(pi);
4310                 return; /* without scheduling another callout */
4311         }
4312
4313         cxgbe_refresh_stats(sc, pi);
4314
4315         callout_schedule(&pi->tick, hz);
4316         PORT_UNLOCK(pi);
4317 }
4318
4319 static void
4320 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4321 {
4322         struct ifnet *vlan;
4323
4324         if (arg != ifp || ifp->if_type != IFT_ETHER)
4325                 return;
4326
4327         vlan = VLAN_DEVAT(ifp, vid);
4328         VLAN_SETCOOKIE(vlan, ifp);
4329 }
4330
4331 static int
4332 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4333 {
4334
4335 #ifdef INVARIANTS
4336         panic("%s: opcode 0x%02x on iq %p with payload %p",
4337             __func__, rss->opcode, iq, m);
4338 #else
4339         log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
4340             __func__, rss->opcode, iq, m);
4341         m_freem(m);
4342 #endif
4343         return (EDOOFUS);
4344 }
4345
4346 int
4347 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
4348 {
4349         uintptr_t *loc, new;
4350
4351         if (opcode >= nitems(sc->cpl_handler))
4352                 return (EINVAL);
4353
4354         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
4355         loc = (uintptr_t *) &sc->cpl_handler[opcode];
4356         atomic_store_rel_ptr(loc, new);
4357
4358         return (0);
4359 }
4360
4361 static int
4362 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
4363 {
4364
4365 #ifdef INVARIANTS
4366         panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
4367 #else
4368         log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
4369             __func__, iq, ctrl);
4370 #endif
4371         return (EDOOFUS);
4372 }
4373
4374 int
4375 t4_register_an_handler(struct adapter *sc, an_handler_t h)
4376 {
4377         uintptr_t *loc, new;
4378
4379         new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
4380         loc = (uintptr_t *) &sc->an_handler;
4381         atomic_store_rel_ptr(loc, new);
4382
4383         return (0);
4384 }
4385
4386 static int
4387 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
4388 {
4389         const struct cpl_fw6_msg *cpl =
4390             __containerof(rpl, struct cpl_fw6_msg, data[0]);
4391
4392 #ifdef INVARIANTS
4393         panic("%s: fw_msg type %d", __func__, cpl->type);
4394 #else
4395         log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
4396 #endif
4397         return (EDOOFUS);
4398 }
4399
4400 int
4401 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
4402 {
4403         uintptr_t *loc, new;
4404
4405         if (type >= nitems(sc->fw_msg_handler))
4406                 return (EINVAL);
4407
4408         /*
4409          * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
4410          * handler dispatch table.  Reject any attempt to install a handler for
4411          * this subtype.
4412          */
4413         if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
4414                 return (EINVAL);
4415
4416         new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
4417         loc = (uintptr_t *) &sc->fw_msg_handler[type];
4418         atomic_store_rel_ptr(loc, new);
4419
4420         return (0);
4421 }
4422
4423 static int
4424 t4_sysctls(struct adapter *sc)
4425 {
4426         struct sysctl_ctx_list *ctx;
4427         struct sysctl_oid *oid;
4428         struct sysctl_oid_list *children, *c0;
4429         static char *caps[] = {
4430                 "\20\1PPP\2QFC\3DCBX",                  /* caps[0] linkcaps */
4431                 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL"        /* caps[1] niccaps */
4432                     "\6HASHFILTER\7ETHOFLD",
4433                 "\20\1TOE",                             /* caps[2] toecaps */
4434                 "\20\1RDDP\2RDMAC",                     /* caps[3] rdmacaps */
4435                 "\20\1INITIATOR_PDU\2TARGET_PDU"        /* caps[4] iscsicaps */
4436                     "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
4437                     "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
4438                 "\20\1INITIATOR\2TARGET\3CTRL_OFLD"     /* caps[5] fcoecaps */
4439                     "\4PO_INITIAOR\5PO_TARGET"
4440         };
4441         static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
4442
4443         ctx = device_get_sysctl_ctx(sc->dev);
4444
4445         /*
4446          * dev.t4nex.X.
4447          */
4448         oid = device_get_sysctl_tree(sc->dev);
4449         c0 = children = SYSCTL_CHILDREN(oid);
4450
4451         sc->sc_do_rxcopy = 1;
4452         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
4453             &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
4454
4455         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
4456             sc->params.nports, "# of ports");
4457
4458         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
4459             NULL, chip_rev(sc), "chip hardware revision");
4460
4461         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
4462             CTLFLAG_RD, sc->fw_version, 0, "firmware version");
4463
4464         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
4465             CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
4466
4467         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
4468             sc->cfcsum, "config file checksum");
4469
4470         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
4471             CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
4472             sysctl_bitfield, "A", "available doorbells");
4473
4474         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
4475             CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
4476             sysctl_bitfield, "A", "available link capabilities");
4477
4478         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
4479             CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
4480             sysctl_bitfield, "A", "available NIC capabilities");
4481
4482         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
4483             CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
4484             sysctl_bitfield, "A", "available TCP offload capabilities");
4485
4486         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
4487             CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
4488             sysctl_bitfield, "A", "available RDMA capabilities");
4489
4490         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
4491             CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
4492             sysctl_bitfield, "A", "available iSCSI capabilities");
4493
4494         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
4495             CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
4496             sysctl_bitfield, "A", "available FCoE capabilities");
4497
4498         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
4499             sc->params.vpd.cclk, "core clock frequency (in KHz)");
4500
4501         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
4502             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
4503             sizeof(sc->sge.timer_val), sysctl_int_array, "A",
4504             "interrupt holdoff timer values (us)");
4505
4506         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
4507             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
4508             sizeof(sc->sge.counter_val), sysctl_int_array, "A",
4509             "interrupt holdoff packet counter values");
4510
4511         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
4512             NULL, sc->tids.nftids, "number of filters");
4513
4514         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
4515             CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
4516             "chip temperature (in Celsius)");
4517
4518         t4_sge_sysctls(sc, ctx, children);
4519
4520         sc->lro_timeout = 100;
4521         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
4522             &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
4523
4524 #ifdef SBUF_DRAIN
4525         /*
4526          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
4527          */
4528         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
4529             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
4530             "logs and miscellaneous information");
4531         children = SYSCTL_CHILDREN(oid);
4532
4533         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
4534             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4535             sysctl_cctrl, "A", "congestion control");
4536
4537         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
4538             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4539             sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
4540
4541         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
4542             CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
4543             sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
4544
4545         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
4546             CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
4547             sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
4548
4549         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
4550             CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
4551             sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
4552
4553         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
4554             CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
4555             sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
4556
4557         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
4558             CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
4559             sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
4560
4561         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
4562             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4563             sysctl_cim_la, "A", "CIM logic analyzer");
4564
4565         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
4566             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4567             sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
4568
4569         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
4570             CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
4571             sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
4572
4573         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
4574             CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
4575             sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
4576
4577         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
4578             CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
4579             sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
4580
4581         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
4582             CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
4583             sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
4584
4585         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
4586             CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
4587             sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
4588
4589         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
4590             CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
4591             sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
4592
4593         if (is_t5(sc)) {
4594                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
4595                     CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
4596                     sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
4597
4598                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
4599                     CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
4600                     sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
4601         }
4602
4603         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
4604             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4605             sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
4606
4607         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
4608             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4609             sysctl_cim_qcfg, "A", "CIM queue configuration");
4610
4611         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
4612             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4613             sysctl_cpl_stats, "A", "CPL statistics");
4614
4615         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
4616             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4617             sysctl_ddp_stats, "A", "non-TCP DDP statistics");
4618
4619         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
4620             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4621             sysctl_devlog, "A", "firmware's device log");
4622
4623         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
4624             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4625             sysctl_fcoe_stats, "A", "FCoE statistics");
4626
4627         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
4628             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4629             sysctl_hw_sched, "A", "hardware scheduler ");
4630
4631         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
4632             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4633             sysctl_l2t, "A", "hardware L2 table");
4634
4635         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
4636             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4637             sysctl_lb_stats, "A", "loopback statistics");
4638
4639         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
4640             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4641             sysctl_meminfo, "A", "memory regions");
4642
4643         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
4644             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4645             sysctl_mps_tcam, "A", "MPS TCAM entries");
4646
4647         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
4648             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4649             sysctl_path_mtus, "A", "path MTUs");
4650
4651         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
4652             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4653             sysctl_pm_stats, "A", "PM statistics");
4654
4655         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
4656             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4657             sysctl_rdma_stats, "A", "RDMA statistics");
4658
4659         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
4660             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4661             sysctl_tcp_stats, "A", "TCP statistics");
4662
4663         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
4664             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4665             sysctl_tids, "A", "TID information");
4666
4667         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
4668             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4669             sysctl_tp_err_stats, "A", "TP error statistics");
4670
4671         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
4672             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4673             sysctl_tp_la, "A", "TP logic analyzer");
4674
4675         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
4676             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4677             sysctl_tx_rate, "A", "Tx rate");
4678
4679         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
4680             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4681             sysctl_ulprx_la, "A", "ULPRX logic analyzer");
4682
4683         if (is_t5(sc)) {
4684                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
4685                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4686                     sysctl_wcwr_stats, "A", "write combined work requests");
4687         }
4688 #endif
4689
4690 #ifdef TCP_OFFLOAD
4691         if (is_offload(sc)) {
4692                 /*
4693                  * dev.t4nex.X.toe.
4694                  */
4695                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
4696                     NULL, "TOE parameters");
4697                 children = SYSCTL_CHILDREN(oid);
4698
4699                 sc->tt.sndbuf = 256 * 1024;
4700                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
4701                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
4702
4703                 sc->tt.ddp = 0;
4704                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
4705                     &sc->tt.ddp, 0, "DDP allowed");
4706
4707                 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
4708                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
4709                     &sc->tt.indsz, 0, "DDP max indicate size allowed");
4710
4711                 sc->tt.ddp_thres =
4712                     G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
4713                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
4714                     &sc->tt.ddp_thres, 0, "DDP threshold");
4715
4716                 sc->tt.rx_coalesce = 1;
4717                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
4718                     CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
4719
4720                 sc->tt.tx_align = 1;
4721                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align",
4722                     CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload");
4723         }
4724 #endif
4725
4726
4727         return (0);
4728 }
4729
4730 static int
4731 cxgbe_sysctls(struct port_info *pi)
4732 {
4733         struct sysctl_ctx_list *ctx;
4734         struct sysctl_oid *oid;
4735         struct sysctl_oid_list *children;
4736         struct adapter *sc = pi->adapter;
4737
4738         ctx = device_get_sysctl_ctx(pi->dev);
4739
4740         /*
4741          * dev.cxgbe.X.
4742          */
4743         oid = device_get_sysctl_tree(pi->dev);
4744         children = SYSCTL_CHILDREN(oid);
4745
4746         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
4747            CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
4748         if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
4749                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
4750                     CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
4751                     "PHY temperature (in Celsius)");
4752                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
4753                     CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
4754                     "PHY firmware version");
4755         }
4756         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
4757             &pi->nrxq, 0, "# of rx queues");
4758         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
4759             &pi->ntxq, 0, "# of tx queues");
4760         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
4761             &pi->first_rxq, 0, "index of first rx queue");
4762         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
4763             &pi->first_txq, 0, "index of first tx queue");
4764         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
4765             CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU",
4766             "Reserve queue 0 for non-flowid packets");
4767
4768 #ifdef TCP_OFFLOAD
4769         if (is_offload(sc)) {
4770                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
4771                     &pi->nofldrxq, 0,
4772                     "# of rx queues for offloaded TCP connections");
4773                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
4774                     &pi->nofldtxq, 0,
4775                     "# of tx queues for offloaded TCP connections");
4776                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
4777                     CTLFLAG_RD, &pi->first_ofld_rxq, 0,
4778                     "index of first TOE rx queue");
4779                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
4780                     CTLFLAG_RD, &pi->first_ofld_txq, 0,
4781                     "index of first TOE tx queue");
4782         }
4783 #endif
4784 #ifdef DEV_NETMAP
4785         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
4786             &pi->nnmrxq, 0, "# of rx queues for netmap");
4787         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
4788             &pi->nnmtxq, 0, "# of tx queues for netmap");
4789         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
4790             CTLFLAG_RD, &pi->first_nm_rxq, 0,
4791             "index of first netmap rx queue");
4792         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
4793             CTLFLAG_RD, &pi->first_nm_txq, 0,
4794             "index of first netmap tx queue");
4795 #endif
4796
4797         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
4798             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
4799             "holdoff timer index");
4800         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
4801             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
4802             "holdoff packet counter index");
4803
4804         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
4805             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
4806             "rx queue size");
4807         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
4808             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
4809             "tx queue size");
4810
4811         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
4812             CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings,
4813             "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
4814
4815         /*
4816          * dev.cxgbe.X.stats.
4817          */
4818         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
4819             NULL, "port statistics");
4820         children = SYSCTL_CHILDREN(oid);
4821
4822 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
4823         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
4824             CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
4825             sysctl_handle_t4_reg64, "QU", desc)
4826
4827         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
4828             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
4829         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
4830             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
4831         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
4832             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
4833         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
4834             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
4835         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
4836             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
4837         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
4838             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
4839         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
4840             "# of tx frames in this range",
4841             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
4842         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
4843             "# of tx frames in this range",
4844             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
4845         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
4846             "# of tx frames in this range",
4847             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
4848         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
4849             "# of tx frames in this range",
4850             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
4851         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
4852             "# of tx frames in this range",
4853             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
4854         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
4855             "# of tx frames in this range",
4856             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
4857         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
4858             "# of tx frames in this range",
4859             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
4860         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
4861             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
4862         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
4863             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
4864         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
4865             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
4866         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
4867             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
4868         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
4869             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
4870         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
4871             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
4872         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
4873             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
4874         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
4875             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
4876         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
4877             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
4878         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
4879             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
4880
4881         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
4882             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
4883         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
4884             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
4885         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
4886             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
4887         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
4888             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
4889         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
4890             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
4891         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
4892             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
4893         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
4894             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
4895         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
4896             "# of frames received with bad FCS",
4897             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
4898         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
4899             "# of frames received with length error",
4900             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
4901         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
4902             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
4903         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
4904             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
4905         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
4906             "# of rx frames in this range",
4907             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
4908         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
4909             "# of rx frames in this range",
4910             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
4911         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
4912             "# of rx frames in this range",
4913             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
4914         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
4915             "# of rx frames in this range",
4916             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
4917         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
4918             "# of rx frames in this range",
4919             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
4920         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
4921             "# of rx frames in this range",
4922             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
4923         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
4924             "# of rx frames in this range",
4925             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
4926         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
4927             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
4928         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
4929             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
4930         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
4931             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
4932         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
4933             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
4934         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
4935             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
4936         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
4937             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
4938         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
4939             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
4940         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
4941             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
4942         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
4943             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
4944
4945 #undef SYSCTL_ADD_T4_REG64
4946
4947 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
4948         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
4949             &pi->stats.name, desc)
4950
4951         /* We get these from port_stats and they may be stale by upto 1s */
4952         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
4953             "# drops due to buffer-group 0 overflows");
4954         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
4955             "# drops due to buffer-group 1 overflows");
4956         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
4957             "# drops due to buffer-group 2 overflows");
4958         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
4959             "# drops due to buffer-group 3 overflows");
4960         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
4961             "# of buffer-group 0 truncated packets");
4962         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
4963             "# of buffer-group 1 truncated packets");
4964         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
4965             "# of buffer-group 2 truncated packets");
4966         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
4967             "# of buffer-group 3 truncated packets");
4968
4969 #undef SYSCTL_ADD_T4_PORTSTAT
4970
4971         return (0);
4972 }
4973
4974 static int
4975 sysctl_int_array(SYSCTL_HANDLER_ARGS)
4976 {
4977         int rc, *i, space = 0;
4978         struct sbuf sb;
4979
4980         sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
4981         for (i = arg1; arg2; arg2 -= sizeof(int), i++) {
4982                 if (space)
4983                         sbuf_printf(&sb, " ");
4984                 sbuf_printf(&sb, "%d", *i);
4985                 space = 1;
4986         }
4987         sbuf_finish(&sb);
4988         rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
4989         sbuf_delete(&sb);
4990         return (rc);
4991 }
4992
4993 static int
4994 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
4995 {
4996         int rc;
4997         struct sbuf *sb;
4998
4999         rc = sysctl_wire_old_buffer(req, 0);
5000         if (rc != 0)
5001                 return(rc);
5002
5003         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5004         if (sb == NULL)
5005                 return (ENOMEM);
5006
5007         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
5008         rc = sbuf_finish(sb);
5009         sbuf_delete(sb);
5010
5011         return (rc);
5012 }
5013
5014 static int
5015 sysctl_btphy(SYSCTL_HANDLER_ARGS)
5016 {
5017         struct port_info *pi = arg1;
5018         int op = arg2;
5019         struct adapter *sc = pi->adapter;
5020         u_int v;
5021         int rc;
5022
5023         rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt");
5024         if (rc)
5025                 return (rc);
5026         /* XXX: magic numbers */
5027         rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
5028             &v);
5029         end_synchronized_op(sc, 0);
5030         if (rc)
5031                 return (rc);
5032         if (op == 0)
5033                 v /= 256;
5034
5035         rc = sysctl_handle_int(oidp, &v, 0, req);
5036         return (rc);
5037 }
5038
5039 static int
5040 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
5041 {
5042         struct port_info *pi = arg1;
5043         int rc, val;
5044
5045         val = pi->rsrv_noflowq;
5046         rc = sysctl_handle_int(oidp, &val, 0, req);
5047         if (rc != 0 || req->newptr == NULL)
5048                 return (rc);
5049
5050         if ((val >= 1) && (pi->ntxq > 1))
5051                 pi->rsrv_noflowq = 1;
5052         else
5053                 pi->rsrv_noflowq = 0;
5054
5055         return (rc);
5056 }
5057
5058 static int
5059 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
5060 {
5061         struct port_info *pi = arg1;
5062         struct adapter *sc = pi->adapter;
5063         int idx, rc, i;
5064         struct sge_rxq *rxq;
5065 #ifdef TCP_OFFLOAD
5066         struct sge_ofld_rxq *ofld_rxq;
5067 #endif
5068         uint8_t v;
5069
5070         idx = pi->tmr_idx;
5071
5072         rc = sysctl_handle_int(oidp, &idx, 0, req);
5073         if (rc != 0 || req->newptr == NULL)
5074                 return (rc);
5075
5076         if (idx < 0 || idx >= SGE_NTIMERS)
5077                 return (EINVAL);
5078
5079         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5080             "t4tmr");
5081         if (rc)
5082                 return (rc);
5083
5084         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
5085         for_each_rxq(pi, i, rxq) {
5086 #ifdef atomic_store_rel_8
5087                 atomic_store_rel_8(&rxq->iq.intr_params, v);
5088 #else
5089                 rxq->iq.intr_params = v;
5090 #endif
5091         }
5092 #ifdef TCP_OFFLOAD
5093         for_each_ofld_rxq(pi, i, ofld_rxq) {
5094 #ifdef atomic_store_rel_8
5095                 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
5096 #else
5097                 ofld_rxq->iq.intr_params = v;
5098 #endif
5099         }
5100 #endif
5101         pi->tmr_idx = idx;
5102
5103         end_synchronized_op(sc, LOCK_HELD);
5104         return (0);
5105 }
5106
5107 static int
5108 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
5109 {
5110         struct port_info *pi = arg1;
5111         struct adapter *sc = pi->adapter;
5112         int idx, rc;
5113
5114         idx = pi->pktc_idx;
5115
5116         rc = sysctl_handle_int(oidp, &idx, 0, req);
5117         if (rc != 0 || req->newptr == NULL)
5118                 return (rc);
5119
5120         if (idx < -1 || idx >= SGE_NCOUNTERS)
5121                 return (EINVAL);
5122
5123         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5124             "t4pktc");
5125         if (rc)
5126                 return (rc);
5127
5128         if (pi->flags & PORT_INIT_DONE)
5129                 rc = EBUSY; /* cannot be changed once the queues are created */
5130         else
5131                 pi->pktc_idx = idx;
5132
5133         end_synchronized_op(sc, LOCK_HELD);
5134         return (rc);
5135 }
5136
5137 static int
5138 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
5139 {
5140         struct port_info *pi = arg1;
5141         struct adapter *sc = pi->adapter;
5142         int qsize, rc;
5143
5144         qsize = pi->qsize_rxq;
5145
5146         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5147         if (rc != 0 || req->newptr == NULL)
5148                 return (rc);
5149
5150         if (qsize < 128 || (qsize & 7))
5151                 return (EINVAL);
5152
5153         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5154             "t4rxqs");
5155         if (rc)
5156                 return (rc);
5157
5158         if (pi->flags & PORT_INIT_DONE)
5159                 rc = EBUSY; /* cannot be changed once the queues are created */
5160         else
5161                 pi->qsize_rxq = qsize;
5162
5163         end_synchronized_op(sc, LOCK_HELD);
5164         return (rc);
5165 }
5166
5167 static int
5168 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
5169 {
5170         struct port_info *pi = arg1;
5171         struct adapter *sc = pi->adapter;
5172         int qsize, rc;
5173
5174         qsize = pi->qsize_txq;
5175
5176         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5177         if (rc != 0 || req->newptr == NULL)
5178                 return (rc);
5179
5180         /* bufring size must be powerof2 */
5181         if (qsize < 128 || !powerof2(qsize))
5182                 return (EINVAL);
5183
5184         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5185             "t4txqs");
5186         if (rc)
5187                 return (rc);
5188
5189         if (pi->flags & PORT_INIT_DONE)
5190                 rc = EBUSY; /* cannot be changed once the queues are created */
5191         else
5192                 pi->qsize_txq = qsize;
5193
5194         end_synchronized_op(sc, LOCK_HELD);
5195         return (rc);
5196 }
5197
5198 static int
5199 sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
5200 {
5201         struct port_info *pi = arg1;
5202         struct adapter *sc = pi->adapter;
5203         struct link_config *lc = &pi->link_cfg;
5204         int rc;
5205
5206         if (req->newptr == NULL) {
5207                 struct sbuf *sb;
5208                 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX";
5209
5210                 rc = sysctl_wire_old_buffer(req, 0);
5211                 if (rc != 0)
5212                         return(rc);
5213
5214                 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5215                 if (sb == NULL)
5216                         return (ENOMEM);
5217
5218                 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits);
5219                 rc = sbuf_finish(sb);
5220                 sbuf_delete(sb);
5221         } else {
5222                 char s[2];
5223                 int n;
5224
5225                 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX));
5226                 s[1] = 0;
5227
5228                 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
5229                 if (rc != 0)
5230                         return(rc);
5231
5232                 if (s[1] != 0)
5233                         return (EINVAL);
5234                 if (s[0] < '0' || s[0] > '9')
5235                         return (EINVAL);        /* not a number */
5236                 n = s[0] - '0';
5237                 if (n & ~(PAUSE_TX | PAUSE_RX))
5238                         return (EINVAL);        /* some other bit is set too */
5239
5240                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE");
5241                 if (rc)
5242                         return (rc);
5243                 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) {
5244                         int link_ok = lc->link_ok;
5245
5246                         lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX);
5247                         lc->requested_fc |= n;
5248                         rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc);
5249                         lc->link_ok = link_ok;  /* restore */
5250                 }
5251                 end_synchronized_op(sc, 0);
5252         }
5253
5254         return (rc);
5255 }
5256
5257 static int
5258 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
5259 {
5260         struct adapter *sc = arg1;
5261         int reg = arg2;
5262         uint64_t val;
5263
5264         val = t4_read_reg64(sc, reg);
5265
5266         return (sysctl_handle_64(oidp, &val, 0, req));
5267 }
5268
5269 static int
5270 sysctl_temperature(SYSCTL_HANDLER_ARGS)
5271 {
5272         struct adapter *sc = arg1;
5273         int rc, t;
5274         uint32_t param, val;
5275
5276         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
5277         if (rc)
5278                 return (rc);
5279         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5280             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5281             V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
5282         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
5283         end_synchronized_op(sc, 0);
5284         if (rc)
5285                 return (rc);
5286
5287         /* unknown is returned as 0 but we display -1 in that case */
5288         t = val == 0 ? -1 : val;
5289
5290         rc = sysctl_handle_int(oidp, &t, 0, req);
5291         return (rc);
5292 }
5293
5294 #ifdef SBUF_DRAIN
5295 static int
5296 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
5297 {
5298         struct adapter *sc = arg1;
5299         struct sbuf *sb;
5300         int rc, i;
5301         uint16_t incr[NMTUS][NCCTRL_WIN];
5302         static const char *dec_fac[] = {
5303                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
5304                 "0.9375"
5305         };
5306
5307         rc = sysctl_wire_old_buffer(req, 0);
5308         if (rc != 0)
5309                 return (rc);
5310
5311         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5312         if (sb == NULL)
5313                 return (ENOMEM);
5314
5315         t4_read_cong_tbl(sc, incr);
5316
5317         for (i = 0; i < NCCTRL_WIN; ++i) {
5318                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
5319                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
5320                     incr[5][i], incr[6][i], incr[7][i]);
5321                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
5322                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
5323                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
5324                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
5325         }
5326
5327         rc = sbuf_finish(sb);
5328         sbuf_delete(sb);
5329
5330         return (rc);
5331 }
5332
5333 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
5334         "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",   /* ibq's */
5335         "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
5336         "SGE0-RX", "SGE1-RX"    /* additional obq's (T5 onwards) */
5337 };
5338
5339 static int
5340 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
5341 {
5342         struct adapter *sc = arg1;
5343         struct sbuf *sb;
5344         int rc, i, n, qid = arg2;
5345         uint32_t *buf, *p;
5346         char *qtype;
5347         u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
5348
5349         KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
5350             ("%s: bad qid %d\n", __func__, qid));
5351
5352         if (qid < CIM_NUM_IBQ) {
5353                 /* inbound queue */
5354                 qtype = "IBQ";
5355                 n = 4 * CIM_IBQ_SIZE;
5356                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5357                 rc = t4_read_cim_ibq(sc, qid, buf, n);
5358         } else {
5359                 /* outbound queue */
5360                 qtype = "OBQ";
5361                 qid -= CIM_NUM_IBQ;
5362                 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
5363                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5364                 rc = t4_read_cim_obq(sc, qid, buf, n);
5365         }
5366
5367         if (rc < 0) {
5368                 rc = -rc;
5369                 goto done;
5370         }
5371         n = rc * sizeof(uint32_t);      /* rc has # of words actually read */
5372
5373         rc = sysctl_wire_old_buffer(req, 0);
5374         if (rc != 0)
5375                 goto done;
5376
5377         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5378         if (sb == NULL) {
5379                 rc = ENOMEM;
5380                 goto done;
5381         }
5382
5383         sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
5384         for (i = 0, p = buf; i < n; i += 16, p += 4)
5385                 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
5386                     p[2], p[3]);
5387
5388         rc = sbuf_finish(sb);
5389         sbuf_delete(sb);
5390 done:
5391         free(buf, M_CXGBE);
5392         return (rc);
5393 }
5394
5395 static int
5396 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
5397 {
5398         struct adapter *sc = arg1;
5399         u_int cfg;
5400         struct sbuf *sb;
5401         uint32_t *buf, *p;
5402         int rc;
5403
5404         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
5405         if (rc != 0)
5406                 return (rc);
5407
5408         rc = sysctl_wire_old_buffer(req, 0);
5409         if (rc != 0)
5410                 return (rc);
5411
5412         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5413         if (sb == NULL)
5414                 return (ENOMEM);
5415
5416         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
5417             M_ZERO | M_WAITOK);
5418
5419         rc = -t4_cim_read_la(sc, buf, NULL);
5420         if (rc != 0)
5421                 goto done;
5422
5423         sbuf_printf(sb, "Status   Data      PC%s",
5424             cfg & F_UPDBGLACAPTPCONLY ? "" :
5425             "     LS0Stat  LS0Addr             LS0Data");
5426
5427         KASSERT((sc->params.cim_la_size & 7) == 0,
5428             ("%s: p will walk off the end of buf", __func__));
5429
5430         for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
5431                 if (cfg & F_UPDBGLACAPTPCONLY) {
5432                         sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
5433                             p[6], p[7]);
5434                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
5435                             (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
5436                             p[4] & 0xff, p[5] >> 8);
5437                         sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
5438                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5439                             p[1] & 0xf, p[2] >> 4);
5440                 } else {
5441                         sbuf_printf(sb,
5442                             "\n  %02x   %x%07x %x%07x %08x %08x "
5443                             "%08x%08x%08x%08x",
5444                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5445                             p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
5446                             p[6], p[7]);
5447                 }
5448         }
5449
5450         rc = sbuf_finish(sb);
5451         sbuf_delete(sb);
5452 done:
5453         free(buf, M_CXGBE);
5454         return (rc);
5455 }
5456
5457 static int
5458 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
5459 {
5460         struct adapter *sc = arg1;
5461         u_int i;
5462         struct sbuf *sb;
5463         uint32_t *buf, *p;
5464         int rc;
5465
5466         rc = sysctl_wire_old_buffer(req, 0);
5467         if (rc != 0)
5468                 return (rc);
5469
5470         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5471         if (sb == NULL)
5472                 return (ENOMEM);
5473
5474         buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
5475             M_ZERO | M_WAITOK);
5476
5477         t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
5478         p = buf;
5479
5480         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5481                 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
5482                     p[1], p[0]);
5483         }
5484
5485         sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
5486         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5487                 sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
5488                     (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
5489                     (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
5490                     (p[1] >> 2) | ((p[2] & 3) << 30),
5491                     (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
5492                     p[0] & 1);
5493         }
5494
5495         rc = sbuf_finish(sb);
5496         sbuf_delete(sb);
5497         free(buf, M_CXGBE);
5498         return (rc);
5499 }
5500
5501 static int
5502 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
5503 {
5504         struct adapter *sc = arg1;
5505         u_int i;
5506         struct sbuf *sb;
5507         uint32_t *buf, *p;
5508         int rc;
5509
5510         rc = sysctl_wire_old_buffer(req, 0);
5511         if (rc != 0)
5512                 return (rc);
5513
5514         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5515         if (sb == NULL)
5516                 return (ENOMEM);
5517
5518         buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
5519             M_ZERO | M_WAITOK);
5520
5521         t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
5522         p = buf;
5523
5524         sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
5525         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5526                 sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
5527                     (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
5528                     p[4], p[3], p[2], p[1], p[0]);
5529         }
5530
5531         sbuf_printf(sb, "\n\nCntl ID               Data");
5532         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5533                 sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
5534                     (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
5535         }
5536
5537         rc = sbuf_finish(sb);
5538         sbuf_delete(sb);
5539         free(buf, M_CXGBE);
5540         return (rc);
5541 }
5542
5543 static int
5544 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
5545 {
5546         struct adapter *sc = arg1;
5547         struct sbuf *sb;
5548         int rc, i;
5549         uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5550         uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5551         uint16_t thres[CIM_NUM_IBQ];
5552         uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
5553         uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
5554         u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
5555
5556         if (is_t4(sc)) {
5557                 cim_num_obq = CIM_NUM_OBQ;
5558                 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
5559                 obq_rdaddr = A_UP_OBQ_0_REALADDR;
5560         } else {
5561                 cim_num_obq = CIM_NUM_OBQ_T5;
5562                 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
5563                 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
5564         }
5565         nq = CIM_NUM_IBQ + cim_num_obq;
5566
5567         rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
5568         if (rc == 0)
5569                 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
5570         if (rc != 0)
5571                 return (rc);
5572
5573         t4_read_cimq_cfg(sc, base, size, thres);
5574
5575         rc = sysctl_wire_old_buffer(req, 0);
5576         if (rc != 0)
5577                 return (rc);
5578
5579         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5580         if (sb == NULL)
5581                 return (ENOMEM);
5582
5583         sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
5584
5585         for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
5586                 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
5587                     qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
5588                     G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5589                     G_QUEREMFLITS(p[2]) * 16);
5590         for ( ; i < nq; i++, p += 4, wr += 2)
5591                 sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
5592                     base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
5593                     wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5594                     G_QUEREMFLITS(p[2]) * 16);
5595
5596         rc = sbuf_finish(sb);
5597         sbuf_delete(sb);
5598
5599         return (rc);
5600 }
5601
5602 static int
5603 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
5604 {
5605         struct adapter *sc = arg1;
5606         struct sbuf *sb;
5607         int rc;
5608         struct tp_cpl_stats stats;
5609
5610         rc = sysctl_wire_old_buffer(req, 0);
5611         if (rc != 0)
5612                 return (rc);
5613
5614         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5615         if (sb == NULL)
5616                 return (ENOMEM);
5617
5618         t4_tp_get_cpl_stats(sc, &stats);
5619
5620         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
5621             "channel 3\n");
5622         sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
5623                    stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
5624         sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
5625                    stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
5626
5627         rc = sbuf_finish(sb);
5628         sbuf_delete(sb);
5629
5630         return (rc);
5631 }
5632
5633 static int
5634 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
5635 {
5636         struct adapter *sc = arg1;
5637         struct sbuf *sb;
5638         int rc;
5639         struct tp_usm_stats stats;
5640
5641         rc = sysctl_wire_old_buffer(req, 0);
5642         if (rc != 0)
5643                 return(rc);
5644
5645         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5646         if (sb == NULL)
5647                 return (ENOMEM);
5648
5649         t4_get_usm_stats(sc, &stats);
5650
5651         sbuf_printf(sb, "Frames: %u\n", stats.frames);
5652         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
5653         sbuf_printf(sb, "Drops:  %u", stats.drops);
5654
5655         rc = sbuf_finish(sb);
5656         sbuf_delete(sb);
5657
5658         return (rc);
5659 }
5660
5661 const char *devlog_level_strings[] = {
5662         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
5663         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
5664         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
5665         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
5666         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
5667         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
5668 };
5669
5670 const char *devlog_facility_strings[] = {
5671         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
5672         [FW_DEVLOG_FACILITY_CF]         = "CF",
5673         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
5674         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
5675         [FW_DEVLOG_FACILITY_RES]        = "RES",
5676         [FW_DEVLOG_FACILITY_HW]         = "HW",
5677         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
5678         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
5679         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
5680         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
5681         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
5682         [FW_DEVLOG_FACILITY_VI]         = "VI",
5683         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
5684         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
5685         [FW_DEVLOG_FACILITY_TM]         = "TM",
5686         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
5687         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
5688         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
5689         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
5690         [FW_DEVLOG_FACILITY_RI]         = "RI",
5691         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
5692         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
5693         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
5694         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE"
5695 };
5696
5697 static int
5698 sysctl_devlog(SYSCTL_HANDLER_ARGS)
5699 {
5700         struct adapter *sc = arg1;
5701         struct devlog_params *dparams = &sc->params.devlog;
5702         struct fw_devlog_e *buf, *e;
5703         int i, j, rc, nentries, first = 0, m;
5704         struct sbuf *sb;
5705         uint64_t ftstamp = UINT64_MAX;
5706
5707         if (dparams->start == 0) {
5708                 dparams->memtype = FW_MEMTYPE_EDC0;
5709                 dparams->start = 0x84000;
5710                 dparams->size = 32768;
5711         }
5712
5713         nentries = dparams->size / sizeof(struct fw_devlog_e);
5714
5715         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
5716         if (buf == NULL)
5717                 return (ENOMEM);
5718
5719         m = fwmtype_to_hwmtype(dparams->memtype);
5720         rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
5721         if (rc != 0)
5722                 goto done;
5723
5724         for (i = 0; i < nentries; i++) {
5725                 e = &buf[i];
5726
5727                 if (e->timestamp == 0)
5728                         break;  /* end */
5729
5730                 e->timestamp = be64toh(e->timestamp);
5731                 e->seqno = be32toh(e->seqno);
5732                 for (j = 0; j < 8; j++)
5733                         e->params[j] = be32toh(e->params[j]);
5734
5735                 if (e->timestamp < ftstamp) {
5736                         ftstamp = e->timestamp;
5737                         first = i;
5738                 }
5739         }
5740
5741         if (buf[first].timestamp == 0)
5742                 goto done;      /* nothing in the log */
5743
5744         rc = sysctl_wire_old_buffer(req, 0);
5745         if (rc != 0)
5746                 goto done;
5747
5748         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5749         if (sb == NULL) {
5750                 rc = ENOMEM;
5751                 goto done;
5752         }
5753         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
5754             "Seq#", "Tstamp", "Level", "Facility", "Message");
5755
5756         i = first;
5757         do {
5758                 e = &buf[i];
5759                 if (e->timestamp == 0)
5760                         break;  /* end */
5761
5762                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
5763                     e->seqno, e->timestamp,
5764                     (e->level < nitems(devlog_level_strings) ?
5765                         devlog_level_strings[e->level] : "UNKNOWN"),
5766                     (e->facility < nitems(devlog_facility_strings) ?
5767                         devlog_facility_strings[e->facility] : "UNKNOWN"));
5768                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
5769                     e->params[2], e->params[3], e->params[4],
5770                     e->params[5], e->params[6], e->params[7]);
5771
5772                 if (++i == nentries)
5773                         i = 0;
5774         } while (i != first);
5775
5776         rc = sbuf_finish(sb);
5777         sbuf_delete(sb);
5778 done:
5779         free(buf, M_CXGBE);
5780         return (rc);
5781 }
5782
5783 static int
5784 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
5785 {
5786         struct adapter *sc = arg1;
5787         struct sbuf *sb;
5788         int rc;
5789         struct tp_fcoe_stats stats[4];
5790
5791         rc = sysctl_wire_old_buffer(req, 0);
5792         if (rc != 0)
5793                 return (rc);
5794
5795         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5796         if (sb == NULL)
5797                 return (ENOMEM);
5798
5799         t4_get_fcoe_stats(sc, 0, &stats[0]);
5800         t4_get_fcoe_stats(sc, 1, &stats[1]);
5801         t4_get_fcoe_stats(sc, 2, &stats[2]);
5802         t4_get_fcoe_stats(sc, 3, &stats[3]);
5803
5804         sbuf_printf(sb, "                   channel 0        channel 1        "
5805             "channel 2        channel 3\n");
5806         sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
5807             stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
5808             stats[3].octetsDDP);
5809         sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
5810             stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
5811         sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
5812             stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
5813             stats[3].framesDrop);
5814
5815         rc = sbuf_finish(sb);
5816         sbuf_delete(sb);
5817
5818         return (rc);
5819 }
5820
5821 static int
5822 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
5823 {
5824         struct adapter *sc = arg1;
5825         struct sbuf *sb;
5826         int rc, i;
5827         unsigned int map, kbps, ipg, mode;
5828         unsigned int pace_tab[NTX_SCHED];
5829
5830         rc = sysctl_wire_old_buffer(req, 0);
5831         if (rc != 0)
5832                 return (rc);
5833
5834         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5835         if (sb == NULL)
5836                 return (ENOMEM);
5837
5838         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
5839         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
5840         t4_read_pace_tbl(sc, pace_tab);
5841
5842         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
5843             "Class IPG (0.1 ns)   Flow IPG (us)");
5844
5845         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
5846                 t4_get_tx_sched(sc, i, &kbps, &ipg);
5847                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
5848                     (mode & (1 << i)) ? "flow" : "class", map & 3);
5849                 if (kbps)
5850                         sbuf_printf(sb, "%9u     ", kbps);
5851                 else
5852                         sbuf_printf(sb, " disabled     ");
5853
5854                 if (ipg)
5855                         sbuf_printf(sb, "%13u        ", ipg);
5856                 else
5857                         sbuf_printf(sb, "     disabled        ");
5858
5859                 if (pace_tab[i])
5860                         sbuf_printf(sb, "%10u", pace_tab[i]);
5861                 else
5862                         sbuf_printf(sb, "  disabled");
5863         }
5864
5865         rc = sbuf_finish(sb);
5866         sbuf_delete(sb);
5867
5868         return (rc);
5869 }
5870
5871 static int
5872 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
5873 {
5874         struct adapter *sc = arg1;
5875         struct sbuf *sb;
5876         int rc, i, j;
5877         uint64_t *p0, *p1;
5878         struct lb_port_stats s[2];
5879         static const char *stat_name[] = {
5880                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
5881                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
5882                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
5883                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
5884                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
5885                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
5886                 "BG2FramesTrunc:", "BG3FramesTrunc:"
5887         };
5888
5889         rc = sysctl_wire_old_buffer(req, 0);
5890         if (rc != 0)
5891                 return (rc);
5892
5893         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5894         if (sb == NULL)
5895                 return (ENOMEM);
5896
5897         memset(s, 0, sizeof(s));
5898
5899         for (i = 0; i < 4; i += 2) {
5900                 t4_get_lb_stats(sc, i, &s[0]);
5901                 t4_get_lb_stats(sc, i + 1, &s[1]);
5902
5903                 p0 = &s[0].octets;
5904                 p1 = &s[1].octets;
5905                 sbuf_printf(sb, "%s                       Loopback %u"
5906                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
5907
5908                 for (j = 0; j < nitems(stat_name); j++)
5909                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
5910                                    *p0++, *p1++);
5911         }
5912
5913         rc = sbuf_finish(sb);
5914         sbuf_delete(sb);
5915
5916         return (rc);
5917 }
5918
5919 static int
5920 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
5921 {
5922         int rc = 0;
5923         struct port_info *pi = arg1;
5924         struct sbuf *sb;
5925         static const char *linkdnreasons[] = {
5926                 "non-specific", "remote fault", "autoneg failed", "reserved3",
5927                 "PHY overheated", "unknown", "rx los", "reserved7"
5928         };
5929
5930         rc = sysctl_wire_old_buffer(req, 0);
5931         if (rc != 0)
5932                 return(rc);
5933         sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
5934         if (sb == NULL)
5935                 return (ENOMEM);
5936
5937         if (pi->linkdnrc < 0)
5938                 sbuf_printf(sb, "n/a");
5939         else if (pi->linkdnrc < nitems(linkdnreasons))
5940                 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
5941         else
5942                 sbuf_printf(sb, "%d", pi->linkdnrc);
5943
5944         rc = sbuf_finish(sb);
5945         sbuf_delete(sb);
5946
5947         return (rc);
5948 }
5949
5950 struct mem_desc {
5951         unsigned int base;
5952         unsigned int limit;
5953         unsigned int idx;
5954 };
5955
5956 static int
5957 mem_desc_cmp(const void *a, const void *b)
5958 {
5959         return ((const struct mem_desc *)a)->base -
5960                ((const struct mem_desc *)b)->base;
5961 }
5962
5963 static void
5964 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
5965     unsigned int to)
5966 {
5967         unsigned int size;
5968
5969         size = to - from + 1;
5970         if (size == 0)
5971                 return;
5972
5973         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
5974         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
5975 }
5976
5977 static int
5978 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
5979 {
5980         struct adapter *sc = arg1;
5981         struct sbuf *sb;
5982         int rc, i, n;
5983         uint32_t lo, hi, used, alloc;
5984         static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
5985         static const char *region[] = {
5986                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
5987                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
5988                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
5989                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
5990                 "RQUDP region:", "PBL region:", "TXPBL region:",
5991                 "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
5992                 "On-chip queues:"
5993         };
5994         struct mem_desc avail[4];
5995         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
5996         struct mem_desc *md = mem;
5997
5998         rc = sysctl_wire_old_buffer(req, 0);
5999         if (rc != 0)
6000                 return (rc);
6001
6002         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6003         if (sb == NULL)
6004                 return (ENOMEM);
6005
6006         for (i = 0; i < nitems(mem); i++) {
6007                 mem[i].limit = 0;
6008                 mem[i].idx = i;
6009         }
6010
6011         /* Find and sort the populated memory ranges */
6012         i = 0;
6013         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
6014         if (lo & F_EDRAM0_ENABLE) {
6015                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
6016                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
6017                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
6018                 avail[i].idx = 0;
6019                 i++;
6020         }
6021         if (lo & F_EDRAM1_ENABLE) {
6022                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
6023                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
6024                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
6025                 avail[i].idx = 1;
6026                 i++;
6027         }
6028         if (lo & F_EXT_MEM_ENABLE) {
6029                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
6030                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
6031                 avail[i].limit = avail[i].base +
6032                     (G_EXT_MEM_SIZE(hi) << 20);
6033                 avail[i].idx = is_t4(sc) ? 2 : 3;       /* Call it MC for T4 */
6034                 i++;
6035         }
6036         if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
6037                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
6038                 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
6039                 avail[i].limit = avail[i].base +
6040                     (G_EXT_MEM1_SIZE(hi) << 20);
6041                 avail[i].idx = 4;
6042                 i++;
6043         }
6044         if (!i)                                    /* no memory available */
6045                 return 0;
6046         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
6047
6048         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
6049         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
6050         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
6051         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
6052         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
6053         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
6054         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
6055         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
6056         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
6057
6058         /* the next few have explicit upper bounds */
6059         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
6060         md->limit = md->base - 1 +
6061                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
6062                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
6063         md++;
6064
6065         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
6066         md->limit = md->base - 1 +
6067                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
6068                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
6069         md++;
6070
6071         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6072                 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
6073                 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
6074                 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
6075         } else {
6076                 md->base = 0;
6077                 md->idx = nitems(region);  /* hide it */
6078         }
6079         md++;
6080
6081 #define ulp_region(reg) \
6082         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
6083         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
6084
6085         ulp_region(RX_ISCSI);
6086         ulp_region(RX_TDDP);
6087         ulp_region(TX_TPT);
6088         ulp_region(RX_STAG);
6089         ulp_region(RX_RQ);
6090         ulp_region(RX_RQUDP);
6091         ulp_region(RX_PBL);
6092         ulp_region(TX_PBL);
6093 #undef ulp_region
6094
6095         md->base = 0;
6096         md->idx = nitems(region);
6097         if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
6098                 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
6099                 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
6100                     A_SGE_DBVFIFO_SIZE))) << 2) - 1;
6101         }
6102         md++;
6103
6104         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
6105         md->limit = md->base + sc->tids.ntids - 1;
6106         md++;
6107         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
6108         md->limit = md->base + sc->tids.ntids - 1;
6109         md++;
6110
6111         md->base = sc->vres.ocq.start;
6112         if (sc->vres.ocq.size)
6113                 md->limit = md->base + sc->vres.ocq.size - 1;
6114         else
6115                 md->idx = nitems(region);  /* hide it */
6116         md++;
6117
6118         /* add any address-space holes, there can be up to 3 */
6119         for (n = 0; n < i - 1; n++)
6120                 if (avail[n].limit < avail[n + 1].base)
6121                         (md++)->base = avail[n].limit;
6122         if (avail[n].limit)
6123                 (md++)->base = avail[n].limit;
6124
6125         n = md - mem;
6126         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
6127
6128         for (lo = 0; lo < i; lo++)
6129                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
6130                                 avail[lo].limit - 1);
6131
6132         sbuf_printf(sb, "\n");
6133         for (i = 0; i < n; i++) {
6134                 if (mem[i].idx >= nitems(region))
6135                         continue;                        /* skip holes */
6136                 if (!mem[i].limit)
6137                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
6138                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
6139                                 mem[i].limit);
6140         }
6141
6142         sbuf_printf(sb, "\n");
6143         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
6144         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
6145         mem_region_show(sb, "uP RAM:", lo, hi);
6146
6147         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
6148         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
6149         mem_region_show(sb, "uP Extmem2:", lo, hi);
6150
6151         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
6152         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
6153                    G_PMRXMAXPAGE(lo),
6154                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
6155                    (lo & F_PMRXNUMCHN) ? 2 : 1);
6156
6157         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
6158         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
6159         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
6160                    G_PMTXMAXPAGE(lo),
6161                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
6162                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
6163         sbuf_printf(sb, "%u p-structs\n",
6164                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
6165
6166         for (i = 0; i < 4; i++) {
6167                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
6168                 if (is_t4(sc)) {
6169                         used = G_USED(lo);
6170                         alloc = G_ALLOC(lo);
6171                 } else {
6172                         used = G_T5_USED(lo);
6173                         alloc = G_T5_ALLOC(lo);
6174                 }
6175                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
6176                            i, used, alloc);
6177         }
6178         for (i = 0; i < 4; i++) {
6179                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
6180                 if (is_t4(sc)) {
6181                         used = G_USED(lo);
6182                         alloc = G_ALLOC(lo);
6183                 } else {
6184                         used = G_T5_USED(lo);
6185                         alloc = G_T5_ALLOC(lo);
6186                 }
6187                 sbuf_printf(sb,
6188                            "\nLoopback %d using %u pages out of %u allocated",
6189                            i, used, alloc);
6190         }
6191
6192         rc = sbuf_finish(sb);
6193         sbuf_delete(sb);
6194
6195         return (rc);
6196 }
6197
6198 static inline void
6199 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
6200 {
6201         *mask = x | y;
6202         y = htobe64(y);
6203         memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
6204 }
6205
6206 static int
6207 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
6208 {
6209         struct adapter *sc = arg1;
6210         struct sbuf *sb;
6211         int rc, i, n;
6212
6213         rc = sysctl_wire_old_buffer(req, 0);
6214         if (rc != 0)
6215                 return (rc);
6216
6217         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6218         if (sb == NULL)
6219                 return (ENOMEM);
6220
6221         sbuf_printf(sb,
6222             "Idx  Ethernet address     Mask     Vld Ports PF"
6223             "  VF              Replication             P0 P1 P2 P3  ML");
6224         n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
6225             NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
6226         for (i = 0; i < n; i++) {
6227                 uint64_t tcamx, tcamy, mask;
6228                 uint32_t cls_lo, cls_hi;
6229                 uint8_t addr[ETHER_ADDR_LEN];
6230
6231                 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
6232                 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
6233                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
6234                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
6235
6236                 if (tcamx & tcamy)
6237                         continue;
6238
6239                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
6240                 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
6241                            "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
6242                            addr[3], addr[4], addr[5], (uintmax_t)mask,
6243                            (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
6244                            G_PORTMAP(cls_hi), G_PF(cls_lo),
6245                            (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
6246
6247                 if (cls_lo & F_REPLICATE) {
6248                         struct fw_ldst_cmd ldst_cmd;
6249
6250                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
6251                         ldst_cmd.op_to_addrspace =
6252                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
6253                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
6254                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
6255                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
6256                         ldst_cmd.u.mps.fid_ctl =
6257                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
6258                                 V_FW_LDST_CMD_CTL(i));
6259
6260                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
6261                             "t4mps");
6262                         if (rc)
6263                                 break;
6264                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
6265                             sizeof(ldst_cmd), &ldst_cmd);
6266                         end_synchronized_op(sc, 0);
6267
6268                         if (rc != 0) {
6269                                 sbuf_printf(sb,
6270                                     " ------------ error %3u ------------", rc);
6271                                 rc = 0;
6272                         } else {
6273                                 sbuf_printf(sb, " %08x %08x %08x %08x",
6274                                     be32toh(ldst_cmd.u.mps.rplc127_96),
6275                                     be32toh(ldst_cmd.u.mps.rplc95_64),
6276                                     be32toh(ldst_cmd.u.mps.rplc63_32),
6277                                     be32toh(ldst_cmd.u.mps.rplc31_0));
6278                         }
6279                 } else
6280                         sbuf_printf(sb, "%36s", "");
6281
6282                 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
6283                     G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
6284                     G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
6285         }
6286
6287         if (rc)
6288                 (void) sbuf_finish(sb);
6289         else
6290                 rc = sbuf_finish(sb);
6291         sbuf_delete(sb);
6292
6293         return (rc);
6294 }
6295
6296 static int
6297 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
6298 {
6299         struct adapter *sc = arg1;
6300         struct sbuf *sb;
6301         int rc;
6302         uint16_t mtus[NMTUS];
6303
6304         rc = sysctl_wire_old_buffer(req, 0);
6305         if (rc != 0)
6306                 return (rc);
6307
6308         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6309         if (sb == NULL)
6310                 return (ENOMEM);
6311
6312         t4_read_mtu_tbl(sc, mtus, NULL);
6313
6314         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
6315             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
6316             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
6317             mtus[14], mtus[15]);
6318
6319         rc = sbuf_finish(sb);
6320         sbuf_delete(sb);
6321
6322         return (rc);
6323 }
6324
6325 static int
6326 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
6327 {
6328         struct adapter *sc = arg1;
6329         struct sbuf *sb;
6330         int rc, i;
6331         uint32_t cnt[PM_NSTATS];
6332         uint64_t cyc[PM_NSTATS];
6333         static const char *rx_stats[] = {
6334                 "Read:", "Write bypass:", "Write mem:", "Flush:"
6335         };
6336         static const char *tx_stats[] = {
6337                 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
6338         };
6339
6340         rc = sysctl_wire_old_buffer(req, 0);
6341         if (rc != 0)
6342                 return (rc);
6343
6344         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6345         if (sb == NULL)
6346                 return (ENOMEM);
6347
6348         t4_pmtx_get_stats(sc, cnt, cyc);
6349         sbuf_printf(sb, "                Tx pcmds             Tx bytes");
6350         for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
6351                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
6352                     cyc[i]);
6353
6354         t4_pmrx_get_stats(sc, cnt, cyc);
6355         sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
6356         for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
6357                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
6358                     cyc[i]);
6359
6360         rc = sbuf_finish(sb);
6361         sbuf_delete(sb);
6362
6363         return (rc);
6364 }
6365
6366 static int
6367 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
6368 {
6369         struct adapter *sc = arg1;
6370         struct sbuf *sb;
6371         int rc;
6372         struct tp_rdma_stats stats;
6373
6374         rc = sysctl_wire_old_buffer(req, 0);
6375         if (rc != 0)
6376                 return (rc);
6377
6378         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6379         if (sb == NULL)
6380                 return (ENOMEM);
6381
6382         t4_tp_get_rdma_stats(sc, &stats);
6383         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
6384         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
6385
6386         rc = sbuf_finish(sb);
6387         sbuf_delete(sb);
6388
6389         return (rc);
6390 }
6391
6392 static int
6393 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
6394 {
6395         struct adapter *sc = arg1;
6396         struct sbuf *sb;
6397         int rc;
6398         struct tp_tcp_stats v4, v6;
6399
6400         rc = sysctl_wire_old_buffer(req, 0);
6401         if (rc != 0)
6402                 return (rc);
6403
6404         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6405         if (sb == NULL)
6406                 return (ENOMEM);
6407
6408         t4_tp_get_tcp_stats(sc, &v4, &v6);
6409         sbuf_printf(sb,
6410             "                                IP                 IPv6\n");
6411         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
6412             v4.tcpOutRsts, v6.tcpOutRsts);
6413         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
6414             v4.tcpInSegs, v6.tcpInSegs);
6415         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
6416             v4.tcpOutSegs, v6.tcpOutSegs);
6417         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
6418             v4.tcpRetransSegs, v6.tcpRetransSegs);
6419
6420         rc = sbuf_finish(sb);
6421         sbuf_delete(sb);
6422
6423         return (rc);
6424 }
6425
6426 static int
6427 sysctl_tids(SYSCTL_HANDLER_ARGS)
6428 {
6429         struct adapter *sc = arg1;
6430         struct sbuf *sb;
6431         int rc;
6432         struct tid_info *t = &sc->tids;
6433
6434         rc = sysctl_wire_old_buffer(req, 0);
6435         if (rc != 0)
6436                 return (rc);
6437
6438         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6439         if (sb == NULL)
6440                 return (ENOMEM);
6441
6442         if (t->natids) {
6443                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
6444                     t->atids_in_use);
6445         }
6446
6447         if (t->ntids) {
6448                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6449                         uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
6450
6451                         if (b) {
6452                                 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
6453                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6454                                     t->ntids - 1);
6455                         } else {
6456                                 sbuf_printf(sb, "TID range: %u-%u",
6457                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6458                                     t->ntids - 1);
6459                         }
6460                 } else
6461                         sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
6462                 sbuf_printf(sb, ", in use: %u\n",
6463                     atomic_load_acq_int(&t->tids_in_use));
6464         }
6465
6466         if (t->nstids) {
6467                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
6468                     t->stid_base + t->nstids - 1, t->stids_in_use);
6469         }
6470
6471         if (t->nftids) {
6472                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
6473                     t->ftid_base + t->nftids - 1);
6474         }
6475
6476         if (t->netids) {
6477                 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
6478                     t->etid_base + t->netids - 1);
6479         }
6480
6481         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
6482             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
6483             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
6484
6485         rc = sbuf_finish(sb);
6486         sbuf_delete(sb);
6487
6488         return (rc);
6489 }
6490
6491 static int
6492 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
6493 {
6494         struct adapter *sc = arg1;
6495         struct sbuf *sb;
6496         int rc;
6497         struct tp_err_stats stats;
6498
6499         rc = sysctl_wire_old_buffer(req, 0);
6500         if (rc != 0)
6501                 return (rc);
6502
6503         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6504         if (sb == NULL)
6505                 return (ENOMEM);
6506
6507         t4_tp_get_err_stats(sc, &stats);
6508
6509         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
6510                       "channel 3\n");
6511         sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
6512             stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
6513             stats.macInErrs[3]);
6514         sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
6515             stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
6516             stats.hdrInErrs[3]);
6517         sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
6518             stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
6519             stats.tcpInErrs[3]);
6520         sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
6521             stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
6522             stats.tcp6InErrs[3]);
6523         sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
6524             stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
6525             stats.tnlCongDrops[3]);
6526         sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
6527             stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
6528             stats.tnlTxDrops[3]);
6529         sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
6530             stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
6531             stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
6532         sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
6533             stats.ofldChanDrops[0], stats.ofldChanDrops[1],
6534             stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
6535         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
6536             stats.ofldNoNeigh, stats.ofldCongDefer);
6537
6538         rc = sbuf_finish(sb);
6539         sbuf_delete(sb);
6540
6541         return (rc);
6542 }
6543
6544 struct field_desc {
6545         const char *name;
6546         u_int start;
6547         u_int width;
6548 };
6549
6550 static void
6551 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
6552 {
6553         char buf[32];
6554         int line_size = 0;
6555
6556         while (f->name) {
6557                 uint64_t mask = (1ULL << f->width) - 1;
6558                 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
6559                     ((uintmax_t)v >> f->start) & mask);
6560
6561                 if (line_size + len >= 79) {
6562                         line_size = 8;
6563                         sbuf_printf(sb, "\n        ");
6564                 }
6565                 sbuf_printf(sb, "%s ", buf);
6566                 line_size += len + 1;
6567                 f++;
6568         }
6569         sbuf_printf(sb, "\n");
6570 }
6571
6572 static struct field_desc tp_la0[] = {
6573         { "RcfOpCodeOut", 60, 4 },
6574         { "State", 56, 4 },
6575         { "WcfState", 52, 4 },
6576         { "RcfOpcSrcOut", 50, 2 },
6577         { "CRxError", 49, 1 },
6578         { "ERxError", 48, 1 },
6579         { "SanityFailed", 47, 1 },
6580         { "SpuriousMsg", 46, 1 },
6581         { "FlushInputMsg", 45, 1 },
6582         { "FlushInputCpl", 44, 1 },
6583         { "RssUpBit", 43, 1 },
6584         { "RssFilterHit", 42, 1 },
6585         { "Tid", 32, 10 },
6586         { "InitTcb", 31, 1 },
6587         { "LineNumber", 24, 7 },
6588         { "Emsg", 23, 1 },
6589         { "EdataOut", 22, 1 },
6590         { "Cmsg", 21, 1 },
6591         { "CdataOut", 20, 1 },
6592         { "EreadPdu", 19, 1 },
6593         { "CreadPdu", 18, 1 },
6594         { "TunnelPkt", 17, 1 },
6595         { "RcfPeerFin", 16, 1 },
6596         { "RcfReasonOut", 12, 4 },
6597         { "TxCchannel", 10, 2 },
6598         { "RcfTxChannel", 8, 2 },
6599         { "RxEchannel", 6, 2 },
6600         { "RcfRxChannel", 5, 1 },
6601         { "RcfDataOutSrdy", 4, 1 },
6602         { "RxDvld", 3, 1 },
6603         { "RxOoDvld", 2, 1 },
6604         { "RxCongestion", 1, 1 },
6605         { "TxCongestion", 0, 1 },
6606         { NULL }
6607 };
6608
6609 static struct field_desc tp_la1[] = {
6610         { "CplCmdIn", 56, 8 },
6611         { "CplCmdOut", 48, 8 },
6612         { "ESynOut", 47, 1 },
6613         { "EAckOut", 46, 1 },
6614         { "EFinOut", 45, 1 },
6615         { "ERstOut", 44, 1 },
6616         { "SynIn", 43, 1 },
6617         { "AckIn", 42, 1 },
6618         { "FinIn", 41, 1 },
6619         { "RstIn", 40, 1 },
6620         { "DataIn", 39, 1 },
6621         { "DataInVld", 38, 1 },
6622         { "PadIn", 37, 1 },
6623         { "RxBufEmpty", 36, 1 },
6624         { "RxDdp", 35, 1 },
6625         { "RxFbCongestion", 34, 1 },
6626         { "TxFbCongestion", 33, 1 },
6627         { "TxPktSumSrdy", 32, 1 },
6628         { "RcfUlpType", 28, 4 },
6629         { "Eread", 27, 1 },
6630         { "Ebypass", 26, 1 },
6631         { "Esave", 25, 1 },
6632         { "Static0", 24, 1 },
6633         { "Cread", 23, 1 },
6634         { "Cbypass", 22, 1 },
6635         { "Csave", 21, 1 },
6636         { "CPktOut", 20, 1 },
6637         { "RxPagePoolFull", 18, 2 },
6638         { "RxLpbkPkt", 17, 1 },
6639         { "TxLpbkPkt", 16, 1 },
6640         { "RxVfValid", 15, 1 },
6641         { "SynLearned", 14, 1 },
6642         { "SetDelEntry", 13, 1 },
6643         { "SetInvEntry", 12, 1 },
6644         { "CpcmdDvld", 11, 1 },
6645         { "CpcmdSave", 10, 1 },
6646         { "RxPstructsFull", 8, 2 },
6647         { "EpcmdDvld", 7, 1 },
6648         { "EpcmdFlush", 6, 1 },
6649         { "EpcmdTrimPrefix", 5, 1 },
6650         { "EpcmdTrimPostfix", 4, 1 },
6651         { "ERssIp4Pkt", 3, 1 },
6652         { "ERssIp6Pkt", 2, 1 },
6653         { "ERssTcpUdpPkt", 1, 1 },
6654         { "ERssFceFipPkt", 0, 1 },
6655         { NULL }
6656 };
6657
6658 static struct field_desc tp_la2[] = {
6659         { "CplCmdIn", 56, 8 },
6660         { "MpsVfVld", 55, 1 },
6661         { "MpsPf", 52, 3 },
6662         { "MpsVf", 44, 8 },
6663         { "SynIn", 43, 1 },
6664         { "AckIn", 42, 1 },
6665         { "FinIn", 41, 1 },
6666         { "RstIn", 40, 1 },
6667         { "DataIn", 39, 1 },
6668         { "DataInVld", 38, 1 },
6669         { "PadIn", 37, 1 },
6670         { "RxBufEmpty", 36, 1 },
6671         { "RxDdp", 35, 1 },
6672         { "RxFbCongestion", 34, 1 },
6673         { "TxFbCongestion", 33, 1 },
6674         { "TxPktSumSrdy", 32, 1 },
6675         { "RcfUlpType", 28, 4 },
6676         { "Eread", 27, 1 },
6677         { "Ebypass", 26, 1 },
6678         { "Esave", 25, 1 },
6679         { "Static0", 24, 1 },
6680         { "Cread", 23, 1 },
6681         { "Cbypass", 22, 1 },
6682         { "Csave", 21, 1 },
6683         { "CPktOut", 20, 1 },
6684         { "RxPagePoolFull", 18, 2 },
6685         { "RxLpbkPkt", 17, 1 },
6686         { "TxLpbkPkt", 16, 1 },
6687         { "RxVfValid", 15, 1 },
6688         { "SynLearned", 14, 1 },
6689         { "SetDelEntry", 13, 1 },
6690         { "SetInvEntry", 12, 1 },
6691         { "CpcmdDvld", 11, 1 },
6692         { "CpcmdSave", 10, 1 },
6693         { "RxPstructsFull", 8, 2 },
6694         { "EpcmdDvld", 7, 1 },
6695         { "EpcmdFlush", 6, 1 },
6696         { "EpcmdTrimPrefix", 5, 1 },
6697         { "EpcmdTrimPostfix", 4, 1 },
6698         { "ERssIp4Pkt", 3, 1 },
6699         { "ERssIp6Pkt", 2, 1 },
6700         { "ERssTcpUdpPkt", 1, 1 },
6701         { "ERssFceFipPkt", 0, 1 },
6702         { NULL }
6703 };
6704
6705 static void
6706 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
6707 {
6708
6709         field_desc_show(sb, *p, tp_la0);
6710 }
6711
6712 static void
6713 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
6714 {
6715
6716         if (idx)
6717                 sbuf_printf(sb, "\n");
6718         field_desc_show(sb, p[0], tp_la0);
6719         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6720                 field_desc_show(sb, p[1], tp_la0);
6721 }
6722
6723 static void
6724 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
6725 {
6726
6727         if (idx)
6728                 sbuf_printf(sb, "\n");
6729         field_desc_show(sb, p[0], tp_la0);
6730         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6731                 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
6732 }
6733
6734 static int
6735 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
6736 {
6737         struct adapter *sc = arg1;
6738         struct sbuf *sb;
6739         uint64_t *buf, *p;
6740         int rc;
6741         u_int i, inc;
6742         void (*show_func)(struct sbuf *, uint64_t *, int);
6743
6744         rc = sysctl_wire_old_buffer(req, 0);
6745         if (rc != 0)
6746                 return (rc);
6747
6748         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6749         if (sb == NULL)
6750                 return (ENOMEM);
6751
6752         buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
6753
6754         t4_tp_read_la(sc, buf, NULL);
6755         p = buf;
6756
6757         switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
6758         case 2:
6759                 inc = 2;
6760                 show_func = tp_la_show2;
6761                 break;
6762         case 3:
6763                 inc = 2;
6764                 show_func = tp_la_show3;
6765                 break;
6766         default:
6767                 inc = 1;
6768                 show_func = tp_la_show;
6769         }
6770
6771         for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
6772                 (*show_func)(sb, p, i);
6773
6774         rc = sbuf_finish(sb);
6775         sbuf_delete(sb);
6776         free(buf, M_CXGBE);
6777         return (rc);
6778 }
6779
6780 static int
6781 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
6782 {
6783         struct adapter *sc = arg1;
6784         struct sbuf *sb;
6785         int rc;
6786         u64 nrate[NCHAN], orate[NCHAN];
6787
6788         rc = sysctl_wire_old_buffer(req, 0);
6789         if (rc != 0)
6790                 return (rc);
6791
6792         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6793         if (sb == NULL)
6794                 return (ENOMEM);
6795
6796         t4_get_chan_txrate(sc, nrate, orate);
6797         sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
6798                  "channel 3\n");
6799         sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
6800             nrate[0], nrate[1], nrate[2], nrate[3]);
6801         sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
6802             orate[0], orate[1], orate[2], orate[3]);
6803
6804         rc = sbuf_finish(sb);
6805         sbuf_delete(sb);
6806
6807         return (rc);
6808 }
6809
6810 static int
6811 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
6812 {
6813         struct adapter *sc = arg1;
6814         struct sbuf *sb;
6815         uint32_t *buf, *p;
6816         int rc, i;
6817
6818         rc = sysctl_wire_old_buffer(req, 0);
6819         if (rc != 0)
6820                 return (rc);
6821
6822         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6823         if (sb == NULL)
6824                 return (ENOMEM);
6825
6826         buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
6827             M_ZERO | M_WAITOK);
6828
6829         t4_ulprx_read_la(sc, buf);
6830         p = buf;
6831
6832         sbuf_printf(sb, "      Pcmd        Type   Message"
6833             "                Data");
6834         for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
6835                 sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
6836                     p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
6837         }
6838
6839         rc = sbuf_finish(sb);
6840         sbuf_delete(sb);
6841         free(buf, M_CXGBE);
6842         return (rc);
6843 }
6844
6845 static int
6846 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
6847 {
6848         struct adapter *sc = arg1;
6849         struct sbuf *sb;
6850         int rc, v;
6851
6852         rc = sysctl_wire_old_buffer(req, 0);
6853         if (rc != 0)
6854                 return (rc);
6855
6856         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6857         if (sb == NULL)
6858                 return (ENOMEM);
6859
6860         v = t4_read_reg(sc, A_SGE_STAT_CFG);
6861         if (G_STATSOURCE_T5(v) == 7) {
6862                 if (G_STATMODE(v) == 0) {
6863                         sbuf_printf(sb, "total %d, incomplete %d",
6864                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6865                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6866                 } else if (G_STATMODE(v) == 1) {
6867                         sbuf_printf(sb, "total %d, data overflow %d",
6868                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6869                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6870                 }
6871         }
6872         rc = sbuf_finish(sb);
6873         sbuf_delete(sb);
6874
6875         return (rc);
6876 }
6877 #endif
6878
6879 static inline void
6880 txq_start(struct ifnet *ifp, struct sge_txq *txq)
6881 {
6882         struct buf_ring *br;
6883         struct mbuf *m;
6884
6885         TXQ_LOCK_ASSERT_OWNED(txq);
6886
6887         br = txq->br;
6888         m = txq->m ? txq->m : drbr_dequeue(ifp, br);
6889         if (m)
6890                 t4_eth_tx(ifp, txq, m);
6891 }
6892
6893 void
6894 t4_tx_callout(void *arg)
6895 {
6896         struct sge_eq *eq = arg;
6897         struct adapter *sc;
6898
6899         if (EQ_TRYLOCK(eq) == 0)
6900                 goto reschedule;
6901
6902         if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) {
6903                 EQ_UNLOCK(eq);
6904 reschedule:
6905                 if (__predict_true(!(eq->flags && EQ_DOOMED)))
6906                         callout_schedule(&eq->tx_callout, 1);
6907                 return;
6908         }
6909
6910         EQ_LOCK_ASSERT_OWNED(eq);
6911
6912         if (__predict_true((eq->flags & EQ_DOOMED) == 0)) {
6913
6914                 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
6915                         struct sge_txq *txq = arg;
6916                         struct port_info *pi = txq->ifp->if_softc;
6917
6918                         sc = pi->adapter;
6919                 } else {
6920                         struct sge_wrq *wrq = arg;
6921
6922                         sc = wrq->adapter;
6923                 }
6924
6925                 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task);
6926         }
6927
6928         EQ_UNLOCK(eq);
6929 }
6930
6931 void
6932 t4_tx_task(void *arg, int count)
6933 {
6934         struct sge_eq *eq = arg;
6935
6936         EQ_LOCK(eq);
6937         if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
6938                 struct sge_txq *txq = arg;
6939                 txq_start(txq->ifp, txq);
6940         } else {
6941                 struct sge_wrq *wrq = arg;
6942                 t4_wrq_tx_locked(wrq->adapter, wrq, NULL);
6943         }
6944         EQ_UNLOCK(eq);
6945 }
6946
6947 static uint32_t
6948 fconf_to_mode(uint32_t fconf)
6949 {
6950         uint32_t mode;
6951
6952         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
6953             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
6954
6955         if (fconf & F_FRAGMENTATION)
6956                 mode |= T4_FILTER_IP_FRAGMENT;
6957
6958         if (fconf & F_MPSHITTYPE)
6959                 mode |= T4_FILTER_MPS_HIT_TYPE;
6960
6961         if (fconf & F_MACMATCH)
6962                 mode |= T4_FILTER_MAC_IDX;
6963
6964         if (fconf & F_ETHERTYPE)
6965                 mode |= T4_FILTER_ETH_TYPE;
6966
6967         if (fconf & F_PROTOCOL)
6968                 mode |= T4_FILTER_IP_PROTO;
6969
6970         if (fconf & F_TOS)
6971                 mode |= T4_FILTER_IP_TOS;
6972
6973         if (fconf & F_VLAN)
6974                 mode |= T4_FILTER_VLAN;
6975
6976         if (fconf & F_VNIC_ID)
6977                 mode |= T4_FILTER_VNIC;
6978
6979         if (fconf & F_PORT)
6980                 mode |= T4_FILTER_PORT;
6981
6982         if (fconf & F_FCOE)
6983                 mode |= T4_FILTER_FCoE;
6984
6985         return (mode);
6986 }
6987
6988 static uint32_t
6989 mode_to_fconf(uint32_t mode)
6990 {
6991         uint32_t fconf = 0;
6992
6993         if (mode & T4_FILTER_IP_FRAGMENT)
6994                 fconf |= F_FRAGMENTATION;
6995
6996         if (mode & T4_FILTER_MPS_HIT_TYPE)
6997                 fconf |= F_MPSHITTYPE;
6998
6999         if (mode & T4_FILTER_MAC_IDX)
7000                 fconf |= F_MACMATCH;
7001
7002         if (mode & T4_FILTER_ETH_TYPE)
7003                 fconf |= F_ETHERTYPE;
7004
7005         if (mode & T4_FILTER_IP_PROTO)
7006                 fconf |= F_PROTOCOL;
7007
7008         if (mode & T4_FILTER_IP_TOS)
7009                 fconf |= F_TOS;
7010
7011         if (mode & T4_FILTER_VLAN)
7012                 fconf |= F_VLAN;
7013
7014         if (mode & T4_FILTER_VNIC)
7015                 fconf |= F_VNIC_ID;
7016
7017         if (mode & T4_FILTER_PORT)
7018                 fconf |= F_PORT;
7019
7020         if (mode & T4_FILTER_FCoE)
7021                 fconf |= F_FCOE;
7022
7023         return (fconf);
7024 }
7025
7026 static uint32_t
7027 fspec_to_fconf(struct t4_filter_specification *fs)
7028 {
7029         uint32_t fconf = 0;
7030
7031         if (fs->val.frag || fs->mask.frag)
7032                 fconf |= F_FRAGMENTATION;
7033
7034         if (fs->val.matchtype || fs->mask.matchtype)
7035                 fconf |= F_MPSHITTYPE;
7036
7037         if (fs->val.macidx || fs->mask.macidx)
7038                 fconf |= F_MACMATCH;
7039
7040         if (fs->val.ethtype || fs->mask.ethtype)
7041                 fconf |= F_ETHERTYPE;
7042
7043         if (fs->val.proto || fs->mask.proto)
7044                 fconf |= F_PROTOCOL;
7045
7046         if (fs->val.tos || fs->mask.tos)
7047                 fconf |= F_TOS;
7048
7049         if (fs->val.vlan_vld || fs->mask.vlan_vld)
7050                 fconf |= F_VLAN;
7051
7052         if (fs->val.vnic_vld || fs->mask.vnic_vld)
7053                 fconf |= F_VNIC_ID;
7054
7055         if (fs->val.iport || fs->mask.iport)
7056                 fconf |= F_PORT;
7057
7058         if (fs->val.fcoe || fs->mask.fcoe)
7059                 fconf |= F_FCOE;
7060
7061         return (fconf);
7062 }
7063
7064 static int
7065 get_filter_mode(struct adapter *sc, uint32_t *mode)
7066 {
7067         int rc;
7068         uint32_t fconf;
7069
7070         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7071             "t4getfm");
7072         if (rc)
7073                 return (rc);
7074
7075         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
7076             A_TP_VLAN_PRI_MAP);
7077
7078         if (sc->params.tp.vlan_pri_map != fconf) {
7079                 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
7080                     device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
7081                     fconf);
7082         }
7083
7084         *mode = fconf_to_mode(fconf);
7085
7086         end_synchronized_op(sc, LOCK_HELD);
7087         return (0);
7088 }
7089
7090 static int
7091 set_filter_mode(struct adapter *sc, uint32_t mode)
7092 {
7093         uint32_t fconf;
7094         int rc;
7095
7096         fconf = mode_to_fconf(mode);
7097
7098         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7099             "t4setfm");
7100         if (rc)
7101                 return (rc);
7102
7103         if (sc->tids.ftids_in_use > 0) {
7104                 rc = EBUSY;
7105                 goto done;
7106         }
7107
7108 #ifdef TCP_OFFLOAD
7109         if (sc->offload_map) {
7110                 rc = EBUSY;
7111                 goto done;
7112         }
7113 #endif
7114
7115         rc = -t4_set_filter_mode(sc, fconf);
7116 done:
7117         end_synchronized_op(sc, LOCK_HELD);
7118         return (rc);
7119 }
7120
7121 static inline uint64_t
7122 get_filter_hits(struct adapter *sc, uint32_t fid)
7123 {
7124         uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
7125         uint64_t hits;
7126
7127         memwin_info(sc, 0, &mw_base, NULL);
7128         off = position_memwin(sc, 0,
7129             tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
7130         if (is_t4(sc)) {
7131                 hits = t4_read_reg64(sc, mw_base + off + 16);
7132                 hits = be64toh(hits);
7133         } else {
7134                 hits = t4_read_reg(sc, mw_base + off + 24);
7135                 hits = be32toh(hits);
7136         }
7137
7138         return (hits);
7139 }
7140
7141 static int
7142 get_filter(struct adapter *sc, struct t4_filter *t)
7143 {
7144         int i, rc, nfilters = sc->tids.nftids;
7145         struct filter_entry *f;
7146
7147         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7148             "t4getf");
7149         if (rc)
7150                 return (rc);
7151
7152         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
7153             t->idx >= nfilters) {
7154                 t->idx = 0xffffffff;
7155                 goto done;
7156         }
7157
7158         f = &sc->tids.ftid_tab[t->idx];
7159         for (i = t->idx; i < nfilters; i++, f++) {
7160                 if (f->valid) {
7161                         t->idx = i;
7162                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
7163                         t->smtidx = f->smtidx;
7164                         if (f->fs.hitcnts)
7165                                 t->hits = get_filter_hits(sc, t->idx);
7166                         else
7167                                 t->hits = UINT64_MAX;
7168                         t->fs = f->fs;
7169
7170                         goto done;
7171                 }
7172         }
7173
7174         t->idx = 0xffffffff;
7175 done:
7176         end_synchronized_op(sc, LOCK_HELD);
7177         return (0);
7178 }
7179
7180 static int
7181 set_filter(struct adapter *sc, struct t4_filter *t)
7182 {
7183         unsigned int nfilters, nports;
7184         struct filter_entry *f;
7185         int i, rc;
7186
7187         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
7188         if (rc)
7189                 return (rc);
7190
7191         nfilters = sc->tids.nftids;
7192         nports = sc->params.nports;
7193
7194         if (nfilters == 0) {
7195                 rc = ENOTSUP;
7196                 goto done;
7197         }
7198
7199         if (!(sc->flags & FULL_INIT_DONE)) {
7200                 rc = EAGAIN;
7201                 goto done;
7202         }
7203
7204         if (t->idx >= nfilters) {
7205                 rc = EINVAL;
7206                 goto done;
7207         }
7208
7209         /* Validate against the global filter mode */
7210         if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
7211             sc->params.tp.vlan_pri_map) {
7212                 rc = E2BIG;
7213                 goto done;
7214         }
7215
7216         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
7217                 rc = EINVAL;
7218                 goto done;
7219         }
7220
7221         if (t->fs.val.iport >= nports) {
7222                 rc = EINVAL;
7223                 goto done;
7224         }
7225
7226         /* Can't specify an iq if not steering to it */
7227         if (!t->fs.dirsteer && t->fs.iq) {
7228                 rc = EINVAL;
7229                 goto done;
7230         }
7231
7232         /* IPv6 filter idx must be 4 aligned */
7233         if (t->fs.type == 1 &&
7234             ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
7235                 rc = EINVAL;
7236                 goto done;
7237         }
7238
7239         if (sc->tids.ftid_tab == NULL) {
7240                 KASSERT(sc->tids.ftids_in_use == 0,
7241                     ("%s: no memory allocated but filters_in_use > 0",
7242                     __func__));
7243
7244                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
7245                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
7246                 if (sc->tids.ftid_tab == NULL) {
7247                         rc = ENOMEM;
7248                         goto done;
7249                 }
7250                 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
7251         }
7252
7253         for (i = 0; i < 4; i++) {
7254                 f = &sc->tids.ftid_tab[t->idx + i];
7255
7256                 if (f->pending || f->valid) {
7257                         rc = EBUSY;
7258                         goto done;
7259                 }
7260                 if (f->locked) {
7261                         rc = EPERM;
7262                         goto done;
7263                 }
7264
7265                 if (t->fs.type == 0)
7266                         break;
7267         }
7268
7269         f = &sc->tids.ftid_tab[t->idx];
7270         f->fs = t->fs;
7271
7272         rc = set_filter_wr(sc, t->idx);
7273 done:
7274         end_synchronized_op(sc, 0);
7275
7276         if (rc == 0) {
7277                 mtx_lock(&sc->tids.ftid_lock);
7278                 for (;;) {
7279                         if (f->pending == 0) {
7280                                 rc = f->valid ? 0 : EIO;
7281                                 break;
7282                         }
7283
7284                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7285                             PCATCH, "t4setfw", 0)) {
7286                                 rc = EINPROGRESS;
7287                                 break;
7288                         }
7289                 }
7290                 mtx_unlock(&sc->tids.ftid_lock);
7291         }
7292         return (rc);
7293 }
7294
7295 static int
7296 del_filter(struct adapter *sc, struct t4_filter *t)
7297 {
7298         unsigned int nfilters;
7299         struct filter_entry *f;
7300         int rc;
7301
7302         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
7303         if (rc)
7304                 return (rc);
7305
7306         nfilters = sc->tids.nftids;
7307
7308         if (nfilters == 0) {
7309                 rc = ENOTSUP;
7310                 goto done;
7311         }
7312
7313         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
7314             t->idx >= nfilters) {
7315                 rc = EINVAL;
7316                 goto done;
7317         }
7318
7319         if (!(sc->flags & FULL_INIT_DONE)) {
7320                 rc = EAGAIN;
7321                 goto done;
7322         }
7323
7324         f = &sc->tids.ftid_tab[t->idx];
7325
7326         if (f->pending) {
7327                 rc = EBUSY;
7328                 goto done;
7329         }
7330         if (f->locked) {
7331                 rc = EPERM;
7332                 goto done;
7333         }
7334
7335         if (f->valid) {
7336                 t->fs = f->fs;  /* extra info for the caller */
7337                 rc = del_filter_wr(sc, t->idx);
7338         }
7339
7340 done:
7341         end_synchronized_op(sc, 0);
7342
7343         if (rc == 0) {
7344                 mtx_lock(&sc->tids.ftid_lock);
7345                 for (;;) {
7346                         if (f->pending == 0) {
7347                                 rc = f->valid ? EIO : 0;
7348                                 break;
7349                         }
7350
7351                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7352                             PCATCH, "t4delfw", 0)) {
7353                                 rc = EINPROGRESS;
7354                                 break;
7355                         }
7356                 }
7357                 mtx_unlock(&sc->tids.ftid_lock);
7358         }
7359
7360         return (rc);
7361 }
7362
7363 static void
7364 clear_filter(struct filter_entry *f)
7365 {
7366         if (f->l2t)
7367                 t4_l2t_release(f->l2t);
7368
7369         bzero(f, sizeof (*f));
7370 }
7371
7372 static int
7373 set_filter_wr(struct adapter *sc, int fidx)
7374 {
7375         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7376         struct wrqe *wr;
7377         struct fw_filter_wr *fwr;
7378         unsigned int ftid;
7379
7380         ASSERT_SYNCHRONIZED_OP(sc);
7381
7382         if (f->fs.newdmac || f->fs.newvlan) {
7383                 /* This filter needs an L2T entry; allocate one. */
7384                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
7385                 if (f->l2t == NULL)
7386                         return (EAGAIN);
7387                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
7388                     f->fs.dmac)) {
7389                         t4_l2t_release(f->l2t);
7390                         f->l2t = NULL;
7391                         return (ENOMEM);
7392                 }
7393         }
7394
7395         ftid = sc->tids.ftid_base + fidx;
7396
7397         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7398         if (wr == NULL)
7399                 return (ENOMEM);
7400
7401         fwr = wrtod(wr);
7402         bzero(fwr, sizeof (*fwr));
7403
7404         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
7405         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
7406         fwr->tid_to_iq =
7407             htobe32(V_FW_FILTER_WR_TID(ftid) |
7408                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
7409                 V_FW_FILTER_WR_NOREPLY(0) |
7410                 V_FW_FILTER_WR_IQ(f->fs.iq));
7411         fwr->del_filter_to_l2tix =
7412             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
7413                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
7414                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
7415                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
7416                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
7417                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
7418                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
7419                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
7420                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
7421                     f->fs.newvlan == VLAN_REWRITE) |
7422                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
7423                     f->fs.newvlan == VLAN_REWRITE) |
7424                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
7425                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
7426                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
7427                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
7428         fwr->ethtype = htobe16(f->fs.val.ethtype);
7429         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
7430         fwr->frag_to_ovlan_vldm =
7431             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
7432                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
7433                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
7434                 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
7435                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
7436                 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
7437         fwr->smac_sel = 0;
7438         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
7439             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
7440         fwr->maci_to_matchtypem =
7441             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
7442                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
7443                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
7444                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
7445                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
7446                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
7447                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
7448                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
7449         fwr->ptcl = f->fs.val.proto;
7450         fwr->ptclm = f->fs.mask.proto;
7451         fwr->ttyp = f->fs.val.tos;
7452         fwr->ttypm = f->fs.mask.tos;
7453         fwr->ivlan = htobe16(f->fs.val.vlan);
7454         fwr->ivlanm = htobe16(f->fs.mask.vlan);
7455         fwr->ovlan = htobe16(f->fs.val.vnic);
7456         fwr->ovlanm = htobe16(f->fs.mask.vnic);
7457         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
7458         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
7459         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
7460         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
7461         fwr->lp = htobe16(f->fs.val.dport);
7462         fwr->lpm = htobe16(f->fs.mask.dport);
7463         fwr->fp = htobe16(f->fs.val.sport);
7464         fwr->fpm = htobe16(f->fs.mask.sport);
7465         if (f->fs.newsmac)
7466                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
7467
7468         f->pending = 1;
7469         sc->tids.ftids_in_use++;
7470
7471         t4_wrq_tx(sc, wr);
7472         return (0);
7473 }
7474
7475 static int
7476 del_filter_wr(struct adapter *sc, int fidx)
7477 {
7478         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7479         struct wrqe *wr;
7480         struct fw_filter_wr *fwr;
7481         unsigned int ftid;
7482
7483         ftid = sc->tids.ftid_base + fidx;
7484
7485         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7486         if (wr == NULL)
7487                 return (ENOMEM);
7488         fwr = wrtod(wr);
7489         bzero(fwr, sizeof (*fwr));
7490
7491         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
7492
7493         f->pending = 1;
7494         t4_wrq_tx(sc, wr);
7495         return (0);
7496 }
7497
7498 int
7499 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
7500 {
7501         struct adapter *sc = iq->adapter;
7502         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
7503         unsigned int idx = GET_TID(rpl);
7504         unsigned int rc;
7505         struct filter_entry *f;
7506
7507         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
7508             rss->opcode));
7509
7510         if (is_ftid(sc, idx)) {
7511
7512                 idx -= sc->tids.ftid_base;
7513                 f = &sc->tids.ftid_tab[idx];
7514                 rc = G_COOKIE(rpl->cookie);
7515
7516                 mtx_lock(&sc->tids.ftid_lock);
7517                 if (rc == FW_FILTER_WR_FLT_ADDED) {
7518                         KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
7519                             __func__, idx));
7520                         f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
7521                         f->pending = 0;  /* asynchronous setup completed */
7522                         f->valid = 1;
7523                 } else {
7524                         if (rc != FW_FILTER_WR_FLT_DELETED) {
7525                                 /* Add or delete failed, display an error */
7526                                 log(LOG_ERR,
7527                                     "filter %u setup failed with error %u\n",
7528                                     idx, rc);
7529                         }
7530
7531                         clear_filter(f);
7532                         sc->tids.ftids_in_use--;
7533                 }
7534                 wakeup(&sc->tids.ftid_tab);
7535                 mtx_unlock(&sc->tids.ftid_lock);
7536         }
7537
7538         return (0);
7539 }
7540
7541 static int
7542 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
7543 {
7544         int rc;
7545
7546         if (cntxt->cid > M_CTXTQID)
7547                 return (EINVAL);
7548
7549         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
7550             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
7551                 return (EINVAL);
7552
7553         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
7554         if (rc)
7555                 return (rc);
7556
7557         if (sc->flags & FW_OK) {
7558                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
7559                     &cntxt->data[0]);
7560                 if (rc == 0)
7561                         goto done;
7562         }
7563
7564         /*
7565          * Read via firmware failed or wasn't even attempted.  Read directly via
7566          * the backdoor.
7567          */
7568         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
7569 done:
7570         end_synchronized_op(sc, 0);
7571         return (rc);
7572 }
7573
7574 static int
7575 load_fw(struct adapter *sc, struct t4_data *fw)
7576 {
7577         int rc;
7578         uint8_t *fw_data;
7579
7580         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
7581         if (rc)
7582                 return (rc);
7583
7584         if (sc->flags & FULL_INIT_DONE) {
7585                 rc = EBUSY;
7586                 goto done;
7587         }
7588
7589         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
7590         if (fw_data == NULL) {
7591                 rc = ENOMEM;
7592                 goto done;
7593         }
7594
7595         rc = copyin(fw->data, fw_data, fw->len);
7596         if (rc == 0)
7597                 rc = -t4_load_fw(sc, fw_data, fw->len);
7598
7599         free(fw_data, M_CXGBE);
7600 done:
7601         end_synchronized_op(sc, 0);
7602         return (rc);
7603 }
7604
7605 static int
7606 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
7607 {
7608         uint32_t addr, off, remaining, i, n;
7609         uint32_t *buf, *b;
7610         uint32_t mw_base, mw_aperture;
7611         int rc;
7612         uint8_t *dst;
7613
7614         rc = validate_mem_range(sc, mr->addr, mr->len);
7615         if (rc != 0)
7616                 return (rc);
7617
7618         memwin_info(sc, win, &mw_base, &mw_aperture);
7619         buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
7620         addr = mr->addr;
7621         remaining = mr->len;
7622         dst = (void *)mr->data;
7623
7624         while (remaining) {
7625                 off = position_memwin(sc, win, addr);
7626
7627                 /* number of bytes that we'll copy in the inner loop */
7628                 n = min(remaining, mw_aperture - off);
7629                 for (i = 0; i < n; i += 4)
7630                         *b++ = t4_read_reg(sc, mw_base + off + i);
7631
7632                 rc = copyout(buf, dst, n);
7633                 if (rc != 0)
7634                         break;
7635
7636                 b = buf;
7637                 dst += n;
7638                 remaining -= n;
7639                 addr += n;
7640         }
7641
7642         free(buf, M_CXGBE);
7643         return (rc);
7644 }
7645
7646 static int
7647 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
7648 {
7649         int rc;
7650
7651         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
7652                 return (EINVAL);
7653
7654         if (i2cd->len > sizeof(i2cd->data))
7655                 return (EFBIG);
7656
7657         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
7658         if (rc)
7659                 return (rc);
7660         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
7661             i2cd->offset, i2cd->len, &i2cd->data[0]);
7662         end_synchronized_op(sc, 0);
7663
7664         return (rc);
7665 }
7666
7667 static int
7668 in_range(int val, int lo, int hi)
7669 {
7670
7671         return (val < 0 || (val <= hi && val >= lo));
7672 }
7673
7674 static int
7675 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
7676 {
7677         int fw_subcmd, fw_type, rc;
7678
7679         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
7680         if (rc)
7681                 return (rc);
7682
7683         if (!(sc->flags & FULL_INIT_DONE)) {
7684                 rc = EAGAIN;
7685                 goto done;
7686         }
7687
7688         /*
7689          * Translate the cxgbetool parameters into T4 firmware parameters.  (The
7690          * sub-command and type are in common locations.)
7691          */
7692         if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
7693                 fw_subcmd = FW_SCHED_SC_CONFIG;
7694         else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
7695                 fw_subcmd = FW_SCHED_SC_PARAMS;
7696         else {
7697                 rc = EINVAL;
7698                 goto done;
7699         }
7700         if (p->type == SCHED_CLASS_TYPE_PACKET)
7701                 fw_type = FW_SCHED_TYPE_PKTSCHED;
7702         else {
7703                 rc = EINVAL;
7704                 goto done;
7705         }
7706
7707         if (fw_subcmd == FW_SCHED_SC_CONFIG) {
7708                 /* Vet our parameters ..*/
7709                 if (p->u.config.minmax < 0) {
7710                         rc = EINVAL;
7711                         goto done;
7712                 }
7713
7714                 /* And pass the request to the firmware ...*/
7715                 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1);
7716                 goto done;
7717         }
7718
7719         if (fw_subcmd == FW_SCHED_SC_PARAMS) {
7720                 int fw_level;
7721                 int fw_mode;
7722                 int fw_rateunit;
7723                 int fw_ratemode;
7724
7725                 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
7726                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
7727                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
7728                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
7729                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
7730                         fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
7731                 else {
7732                         rc = EINVAL;
7733                         goto done;
7734                 }
7735
7736                 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
7737                         fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
7738                 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
7739                         fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
7740                 else {
7741                         rc = EINVAL;
7742                         goto done;
7743                 }
7744
7745                 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
7746                         fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
7747                 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
7748                         fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
7749                 else {
7750                         rc = EINVAL;
7751                         goto done;
7752                 }
7753
7754                 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
7755                         fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
7756                 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
7757                         fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
7758                 else {
7759                         rc = EINVAL;
7760                         goto done;
7761                 }
7762
7763                 /* Vet our parameters ... */
7764                 if (!in_range(p->u.params.channel, 0, 3) ||
7765                     !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
7766                     !in_range(p->u.params.minrate, 0, 10000000) ||
7767                     !in_range(p->u.params.maxrate, 0, 10000000) ||
7768                     !in_range(p->u.params.weight, 0, 100)) {
7769                         rc = ERANGE;
7770                         goto done;
7771                 }
7772
7773                 /*
7774                  * Translate any unset parameters into the firmware's
7775                  * nomenclature and/or fail the call if the parameters
7776                  * are required ...
7777                  */
7778                 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
7779                     p->u.params.channel < 0 || p->u.params.cl < 0) {
7780                         rc = EINVAL;
7781                         goto done;
7782                 }
7783                 if (p->u.params.minrate < 0)
7784                         p->u.params.minrate = 0;
7785                 if (p->u.params.maxrate < 0) {
7786                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7787                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7788                                 rc = EINVAL;
7789                                 goto done;
7790                         } else
7791                                 p->u.params.maxrate = 0;
7792                 }
7793                 if (p->u.params.weight < 0) {
7794                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
7795                                 rc = EINVAL;
7796                                 goto done;
7797                         } else
7798                                 p->u.params.weight = 0;
7799                 }
7800                 if (p->u.params.pktsize < 0) {
7801                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7802                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7803                                 rc = EINVAL;
7804                                 goto done;
7805                         } else
7806                                 p->u.params.pktsize = 0;
7807                 }
7808
7809                 /* See what the firmware thinks of the request ... */
7810                 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
7811                     fw_rateunit, fw_ratemode, p->u.params.channel,
7812                     p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
7813                     p->u.params.weight, p->u.params.pktsize, 1);
7814                 goto done;
7815         }
7816
7817         rc = EINVAL;
7818 done:
7819         end_synchronized_op(sc, 0);
7820         return (rc);
7821 }
7822
7823 static int
7824 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
7825 {
7826         struct port_info *pi = NULL;
7827         struct sge_txq *txq;
7828         uint32_t fw_mnem, fw_queue, fw_class;
7829         int i, rc;
7830
7831         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
7832         if (rc)
7833                 return (rc);
7834
7835         if (!(sc->flags & FULL_INIT_DONE)) {
7836                 rc = EAGAIN;
7837                 goto done;
7838         }
7839
7840         if (p->port >= sc->params.nports) {
7841                 rc = EINVAL;
7842                 goto done;
7843         }
7844
7845         pi = sc->port[p->port];
7846         if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
7847                 rc = EINVAL;
7848                 goto done;
7849         }
7850
7851         /*
7852          * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
7853          * Scheduling Class in this case).
7854          */
7855         fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
7856             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
7857         fw_class = p->cl < 0 ? 0xffffffff : p->cl;
7858
7859         /*
7860          * If op.queue is non-negative, then we're only changing the scheduling
7861          * on a single specified TX queue.
7862          */
7863         if (p->queue >= 0) {
7864                 txq = &sc->sge.txq[pi->first_txq + p->queue];
7865                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7866                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7867                     &fw_class);
7868                 goto done;
7869         }
7870
7871         /*
7872          * Change the scheduling on all the TX queues for the
7873          * interface.
7874          */
7875         for_each_txq(pi, i, txq) {
7876                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7877                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7878                     &fw_class);
7879                 if (rc)
7880                         goto done;
7881         }
7882
7883         rc = 0;
7884 done:
7885         end_synchronized_op(sc, 0);
7886         return (rc);
7887 }
7888
7889 int
7890 t4_os_find_pci_capability(struct adapter *sc, int cap)
7891 {
7892         int i;
7893
7894         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
7895 }
7896
7897 int
7898 t4_os_pci_save_state(struct adapter *sc)
7899 {
7900         device_t dev;
7901         struct pci_devinfo *dinfo;
7902
7903         dev = sc->dev;
7904         dinfo = device_get_ivars(dev);
7905
7906         pci_cfg_save(dev, dinfo, 0);
7907         return (0);
7908 }
7909
7910 int
7911 t4_os_pci_restore_state(struct adapter *sc)
7912 {
7913         device_t dev;
7914         struct pci_devinfo *dinfo;
7915
7916         dev = sc->dev;
7917         dinfo = device_get_ivars(dev);
7918
7919         pci_cfg_restore(dev, dinfo);
7920         return (0);
7921 }
7922
7923 void
7924 t4_os_portmod_changed(const struct adapter *sc, int idx)
7925 {
7926         struct port_info *pi = sc->port[idx];
7927         static const char *mod_str[] = {
7928                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
7929         };
7930
7931         build_medialist(pi, &pi->media);
7932 #ifdef DEV_NETMAP
7933         build_medialist(pi, &pi->nm_media);
7934 #endif
7935
7936         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
7937                 if_printf(pi->ifp, "transceiver unplugged.\n");
7938         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
7939                 if_printf(pi->ifp, "unknown transceiver inserted.\n");
7940         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
7941                 if_printf(pi->ifp, "unsupported transceiver inserted.\n");
7942         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
7943                 if_printf(pi->ifp, "%s transceiver inserted.\n",
7944                     mod_str[pi->mod_type]);
7945         } else {
7946                 if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
7947                     pi->mod_type);
7948         }
7949 }
7950
7951 void
7952 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
7953 {
7954         struct port_info *pi = sc->port[idx];
7955         struct ifnet *ifp = pi->ifp;
7956
7957         if (link_stat) {
7958                 pi->linkdnrc = -1;
7959                 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
7960                 if_link_state_change(ifp, LINK_STATE_UP);
7961         } else {
7962                 if (reason >= 0)
7963                         pi->linkdnrc = reason;
7964                 if_link_state_change(ifp, LINK_STATE_DOWN);
7965         }
7966 }
7967
7968 void
7969 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
7970 {
7971         struct adapter *sc;
7972
7973         sx_slock(&t4_list_lock);
7974         SLIST_FOREACH(sc, &t4_list, link) {
7975                 /*
7976                  * func should not make any assumptions about what state sc is
7977                  * in - the only guarantee is that sc->sc_lock is a valid lock.
7978                  */
7979                 func(sc, arg);
7980         }
7981         sx_sunlock(&t4_list_lock);
7982 }
7983
7984 static int
7985 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
7986 {
7987        return (0);
7988 }
7989
7990 static int
7991 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
7992 {
7993        return (0);
7994 }
7995
7996 static int
7997 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
7998     struct thread *td)
7999 {
8000         int rc;
8001         struct adapter *sc = dev->si_drv1;
8002
8003         rc = priv_check(td, PRIV_DRIVER);
8004         if (rc != 0)
8005                 return (rc);
8006
8007         switch (cmd) {
8008         case CHELSIO_T4_GETREG: {
8009                 struct t4_reg *edata = (struct t4_reg *)data;
8010
8011                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8012                         return (EFAULT);
8013
8014                 if (edata->size == 4)
8015                         edata->val = t4_read_reg(sc, edata->addr);
8016                 else if (edata->size == 8)
8017                         edata->val = t4_read_reg64(sc, edata->addr);
8018                 else
8019                         return (EINVAL);
8020
8021                 break;
8022         }
8023         case CHELSIO_T4_SETREG: {
8024                 struct t4_reg *edata = (struct t4_reg *)data;
8025
8026                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8027                         return (EFAULT);
8028
8029                 if (edata->size == 4) {
8030                         if (edata->val & 0xffffffff00000000)
8031                                 return (EINVAL);
8032                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
8033                 } else if (edata->size == 8)
8034                         t4_write_reg64(sc, edata->addr, edata->val);
8035                 else
8036                         return (EINVAL);
8037                 break;
8038         }
8039         case CHELSIO_T4_REGDUMP: {
8040                 struct t4_regdump *regs = (struct t4_regdump *)data;
8041                 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
8042                 uint8_t *buf;
8043
8044                 if (regs->len < reglen) {
8045                         regs->len = reglen; /* hint to the caller */
8046                         return (ENOBUFS);
8047                 }
8048
8049                 regs->len = reglen;
8050                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
8051                 t4_get_regs(sc, regs, buf);
8052                 rc = copyout(buf, regs->data, reglen);
8053                 free(buf, M_CXGBE);
8054                 break;
8055         }
8056         case CHELSIO_T4_GET_FILTER_MODE:
8057                 rc = get_filter_mode(sc, (uint32_t *)data);
8058                 break;
8059         case CHELSIO_T4_SET_FILTER_MODE:
8060                 rc = set_filter_mode(sc, *(uint32_t *)data);
8061                 break;
8062         case CHELSIO_T4_GET_FILTER:
8063                 rc = get_filter(sc, (struct t4_filter *)data);
8064                 break;
8065         case CHELSIO_T4_SET_FILTER:
8066                 rc = set_filter(sc, (struct t4_filter *)data);
8067                 break;
8068         case CHELSIO_T4_DEL_FILTER:
8069                 rc = del_filter(sc, (struct t4_filter *)data);
8070                 break;
8071         case CHELSIO_T4_GET_SGE_CONTEXT:
8072                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
8073                 break;
8074         case CHELSIO_T4_LOAD_FW:
8075                 rc = load_fw(sc, (struct t4_data *)data);
8076                 break;
8077         case CHELSIO_T4_GET_MEM:
8078                 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
8079                 break;
8080         case CHELSIO_T4_GET_I2C:
8081                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
8082                 break;
8083         case CHELSIO_T4_CLEAR_STATS: {
8084                 int i;
8085                 u_int port_id = *(uint32_t *)data;
8086                 struct port_info *pi;
8087
8088                 if (port_id >= sc->params.nports)
8089                         return (EINVAL);
8090                 pi = sc->port[port_id];
8091
8092                 /* MAC stats */
8093                 t4_clr_port_stats(sc, pi->tx_chan);
8094
8095                 if (pi->flags & PORT_INIT_DONE) {
8096                         struct sge_rxq *rxq;
8097                         struct sge_txq *txq;
8098                         struct sge_wrq *wrq;
8099
8100                         for_each_rxq(pi, i, rxq) {
8101 #if defined(INET) || defined(INET6)
8102                                 rxq->lro.lro_queued = 0;
8103                                 rxq->lro.lro_flushed = 0;
8104 #endif
8105                                 rxq->rxcsum = 0;
8106                                 rxq->vlan_extraction = 0;
8107                         }
8108
8109                         for_each_txq(pi, i, txq) {
8110                                 txq->txcsum = 0;
8111                                 txq->tso_wrs = 0;
8112                                 txq->vlan_insertion = 0;
8113                                 txq->imm_wrs = 0;
8114                                 txq->sgl_wrs = 0;
8115                                 txq->txpkt_wrs = 0;
8116                                 txq->txpkts_wrs = 0;
8117                                 txq->txpkts_pkts = 0;
8118                                 txq->br->br_drops = 0;
8119                                 txq->no_dmamap = 0;
8120                                 txq->no_desc = 0;
8121                         }
8122
8123 #ifdef TCP_OFFLOAD
8124                         /* nothing to clear for each ofld_rxq */
8125
8126                         for_each_ofld_txq(pi, i, wrq) {
8127                                 wrq->tx_wrs = 0;
8128                                 wrq->no_desc = 0;
8129                         }
8130 #endif
8131                         wrq = &sc->sge.ctrlq[pi->port_id];
8132                         wrq->tx_wrs = 0;
8133                         wrq->no_desc = 0;
8134                 }
8135                 break;
8136         }
8137         case CHELSIO_T4_SCHED_CLASS:
8138                 rc = set_sched_class(sc, (struct t4_sched_params *)data);
8139                 break;
8140         case CHELSIO_T4_SCHED_QUEUE:
8141                 rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
8142                 break;
8143         case CHELSIO_T4_GET_TRACER:
8144                 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
8145                 break;
8146         case CHELSIO_T4_SET_TRACER:
8147                 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
8148                 break;
8149         default:
8150                 rc = EINVAL;
8151         }
8152
8153         return (rc);
8154 }
8155
8156 #ifdef TCP_OFFLOAD
8157 void
8158 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask,
8159     const unsigned int *pgsz_order)
8160 {
8161         struct port_info *pi = ifp->if_softc;
8162         struct adapter *sc = pi->adapter;
8163
8164         t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
8165         t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |
8166                 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) |
8167                 V_HPZ3(pgsz_order[3]));
8168 }
8169
8170 static int
8171 toe_capability(struct port_info *pi, int enable)
8172 {
8173         int rc;
8174         struct adapter *sc = pi->adapter;
8175
8176         ASSERT_SYNCHRONIZED_OP(sc);
8177
8178         if (!is_offload(sc))
8179                 return (ENODEV);
8180
8181         if (enable) {
8182                 /*
8183                  * We need the port's queues around so that we're able to send
8184                  * and receive CPLs to/from the TOE even if the ifnet for this
8185                  * port has never been UP'd administratively.
8186                  */
8187                 if (!(pi->flags & PORT_INIT_DONE)) {
8188                         rc = cxgbe_init_synchronized(pi);
8189                         if (rc)
8190                                 return (rc);
8191                 }
8192
8193                 if (isset(&sc->offload_map, pi->port_id))
8194                         return (0);
8195
8196                 if (!(sc->flags & TOM_INIT_DONE)) {
8197                         rc = t4_activate_uld(sc, ULD_TOM);
8198                         if (rc == EAGAIN) {
8199                                 log(LOG_WARNING,
8200                                     "You must kldload t4_tom.ko before trying "
8201                                     "to enable TOE on a cxgbe interface.\n");
8202                         }
8203                         if (rc != 0)
8204                                 return (rc);
8205                         KASSERT(sc->tom_softc != NULL,
8206                             ("%s: TOM activated but softc NULL", __func__));
8207                         KASSERT(sc->flags & TOM_INIT_DONE,
8208                             ("%s: TOM activated but flag not set", __func__));
8209                 }
8210
8211                 setbit(&sc->offload_map, pi->port_id);
8212         } else {
8213                 if (!isset(&sc->offload_map, pi->port_id))
8214                         return (0);
8215
8216                 KASSERT(sc->flags & TOM_INIT_DONE,
8217                     ("%s: TOM never initialized?", __func__));
8218                 clrbit(&sc->offload_map, pi->port_id);
8219         }
8220
8221         return (0);
8222 }
8223
8224 /*
8225  * Add an upper layer driver to the global list.
8226  */
8227 int
8228 t4_register_uld(struct uld_info *ui)
8229 {
8230         int rc = 0;
8231         struct uld_info *u;
8232
8233         sx_xlock(&t4_uld_list_lock);
8234         SLIST_FOREACH(u, &t4_uld_list, link) {
8235             if (u->uld_id == ui->uld_id) {
8236                     rc = EEXIST;
8237                     goto done;
8238             }
8239         }
8240
8241         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
8242         ui->refcount = 0;
8243 done:
8244         sx_xunlock(&t4_uld_list_lock);
8245         return (rc);
8246 }
8247
8248 int
8249 t4_unregister_uld(struct uld_info *ui)
8250 {
8251         int rc = EINVAL;
8252         struct uld_info *u;
8253
8254         sx_xlock(&t4_uld_list_lock);
8255
8256         SLIST_FOREACH(u, &t4_uld_list, link) {
8257             if (u == ui) {
8258                     if (ui->refcount > 0) {
8259                             rc = EBUSY;
8260                             goto done;
8261                     }
8262
8263                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
8264                     rc = 0;
8265                     goto done;
8266             }
8267         }
8268 done:
8269         sx_xunlock(&t4_uld_list_lock);
8270         return (rc);
8271 }
8272
8273 int
8274 t4_activate_uld(struct adapter *sc, int id)
8275 {
8276         int rc = EAGAIN;
8277         struct uld_info *ui;
8278
8279         ASSERT_SYNCHRONIZED_OP(sc);
8280
8281         sx_slock(&t4_uld_list_lock);
8282
8283         SLIST_FOREACH(ui, &t4_uld_list, link) {
8284                 if (ui->uld_id == id) {
8285                         if (!(sc->flags & FULL_INIT_DONE)) {
8286                                 rc = adapter_full_init(sc);
8287                                 if (rc != 0)
8288                                         goto done;
8289                         }
8290
8291                         rc = ui->activate(sc);
8292                         if (rc == 0)
8293                                 ui->refcount++;
8294                         goto done;
8295                 }
8296         }
8297 done:
8298         sx_sunlock(&t4_uld_list_lock);
8299
8300         return (rc);
8301 }
8302
8303 int
8304 t4_deactivate_uld(struct adapter *sc, int id)
8305 {
8306         int rc = EINVAL;
8307         struct uld_info *ui;
8308
8309         ASSERT_SYNCHRONIZED_OP(sc);
8310
8311         sx_slock(&t4_uld_list_lock);
8312
8313         SLIST_FOREACH(ui, &t4_uld_list, link) {
8314                 if (ui->uld_id == id) {
8315                         rc = ui->deactivate(sc);
8316                         if (rc == 0)
8317                                 ui->refcount--;
8318                         goto done;
8319                 }
8320         }
8321 done:
8322         sx_sunlock(&t4_uld_list_lock);
8323
8324         return (rc);
8325 }
8326 #endif
8327
8328 /*
8329  * Come up with reasonable defaults for some of the tunables, provided they're
8330  * not set by the user (in which case we'll use the values as is).
8331  */
8332 static void
8333 tweak_tunables(void)
8334 {
8335         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
8336
8337         if (t4_ntxq10g < 1)
8338                 t4_ntxq10g = min(nc, NTXQ_10G);
8339
8340         if (t4_ntxq1g < 1)
8341                 t4_ntxq1g = min(nc, NTXQ_1G);
8342
8343         if (t4_nrxq10g < 1)
8344                 t4_nrxq10g = min(nc, NRXQ_10G);
8345
8346         if (t4_nrxq1g < 1)
8347                 t4_nrxq1g = min(nc, NRXQ_1G);
8348
8349 #ifdef TCP_OFFLOAD
8350         if (t4_nofldtxq10g < 1)
8351                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
8352
8353         if (t4_nofldtxq1g < 1)
8354                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
8355
8356         if (t4_nofldrxq10g < 1)
8357                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
8358
8359         if (t4_nofldrxq1g < 1)
8360                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
8361
8362         if (t4_toecaps_allowed == -1)
8363                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
8364 #else
8365         if (t4_toecaps_allowed == -1)
8366                 t4_toecaps_allowed = 0;
8367 #endif
8368
8369 #ifdef DEV_NETMAP
8370         if (t4_nnmtxq10g < 1)
8371                 t4_nnmtxq10g = min(nc, NNMTXQ_10G);
8372
8373         if (t4_nnmtxq1g < 1)
8374                 t4_nnmtxq1g = min(nc, NNMTXQ_1G);
8375
8376         if (t4_nnmrxq10g < 1)
8377                 t4_nnmrxq10g = min(nc, NNMRXQ_10G);
8378
8379         if (t4_nnmrxq1g < 1)
8380                 t4_nnmrxq1g = min(nc, NNMRXQ_1G);
8381 #endif
8382
8383         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
8384                 t4_tmr_idx_10g = TMR_IDX_10G;
8385
8386         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
8387                 t4_pktc_idx_10g = PKTC_IDX_10G;
8388
8389         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
8390                 t4_tmr_idx_1g = TMR_IDX_1G;
8391
8392         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
8393                 t4_pktc_idx_1g = PKTC_IDX_1G;
8394
8395         if (t4_qsize_txq < 128)
8396                 t4_qsize_txq = 128;
8397
8398         if (t4_qsize_rxq < 128)
8399                 t4_qsize_rxq = 128;
8400         while (t4_qsize_rxq & 7)
8401                 t4_qsize_rxq++;
8402
8403         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
8404 }
8405
8406 static struct sx mlu;   /* mod load unload */
8407 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
8408
8409 static int
8410 mod_event(module_t mod, int cmd, void *arg)
8411 {
8412         int rc = 0;
8413         static int loaded = 0;
8414
8415         switch (cmd) {
8416         case MOD_LOAD:
8417                 sx_xlock(&mlu);
8418                 if (loaded++ == 0) {
8419                         t4_sge_modload();
8420                         sx_init(&t4_list_lock, "T4/T5 adapters");
8421                         SLIST_INIT(&t4_list);
8422 #ifdef TCP_OFFLOAD
8423                         sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
8424                         SLIST_INIT(&t4_uld_list);
8425 #endif
8426                         t4_tracer_modload();
8427                         tweak_tunables();
8428                 }
8429                 sx_xunlock(&mlu);
8430                 break;
8431
8432         case MOD_UNLOAD:
8433                 sx_xlock(&mlu);
8434                 if (--loaded == 0) {
8435                         int tries;
8436
8437                         sx_slock(&t4_list_lock);
8438                         if (!SLIST_EMPTY(&t4_list)) {
8439                                 rc = EBUSY;
8440                                 sx_sunlock(&t4_list_lock);
8441                                 goto done_unload;
8442                         }
8443 #ifdef TCP_OFFLOAD
8444                         sx_slock(&t4_uld_list_lock);
8445                         if (!SLIST_EMPTY(&t4_uld_list)) {
8446                                 rc = EBUSY;
8447                                 sx_sunlock(&t4_uld_list_lock);
8448                                 sx_sunlock(&t4_list_lock);
8449                                 goto done_unload;
8450                         }
8451 #endif
8452                         tries = 0;
8453                         while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
8454                                 uprintf("%ju clusters with custom free routine "
8455                                     "still is use.\n", t4_sge_extfree_refs());
8456                                 pause("t4unload", 2 * hz);
8457                         }
8458 #ifdef TCP_OFFLOAD
8459                         sx_sunlock(&t4_uld_list_lock);
8460 #endif
8461                         sx_sunlock(&t4_list_lock);
8462
8463                         if (t4_sge_extfree_refs() == 0) {
8464                                 t4_tracer_modunload();
8465 #ifdef TCP_OFFLOAD
8466                                 sx_destroy(&t4_uld_list_lock);
8467 #endif
8468                                 sx_destroy(&t4_list_lock);
8469                                 t4_sge_modunload();
8470                                 loaded = 0;
8471                         } else {
8472                                 rc = EBUSY;
8473                                 loaded++;       /* undo earlier decrement */
8474                         }
8475                 }
8476 done_unload:
8477                 sx_xunlock(&mlu);
8478                 break;
8479         }
8480
8481         return (rc);
8482 }
8483
8484 static devclass_t t4_devclass, t5_devclass;
8485 static devclass_t cxgbe_devclass, cxl_devclass;
8486
8487 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
8488 MODULE_VERSION(t4nex, 1);
8489 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
8490
8491 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
8492 MODULE_VERSION(t5nex, 1);
8493 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
8494
8495 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
8496 MODULE_VERSION(cxgbe, 1);
8497
8498 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
8499 MODULE_VERSION(cxl, 1);