sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/conf.h>
  36 #include <sys/priv.h>
  37 #include <sys/kernel.h>
  38 #include <sys/bus.h>
  39 #include <sys/module.h>
  40 #include <sys/malloc.h>
  41 #include <sys/queue.h>
  42 #include <sys/taskqueue.h>
  43 #include <sys/pciio.h>
  44 #include <dev/pci/pcireg.h>
  45 #include <dev/pci/pcivar.h>
  46 #include <dev/pci/pci_private.h>
  47 #include <sys/firmware.h>
  48 #include <sys/sbuf.h>
  49 #include <sys/smp.h>
  50 #include <sys/socket.h>
  51 #include <sys/sockio.h>
  52 #include <sys/sysctl.h>
  53 #include <net/ethernet.h>
  54 #include <net/if.h>
  55 #include <net/if_types.h>
  56 #include <net/if_dl.h>
  57 #include <net/if_vlan_var.h>
  58 #if defined(__i386__) || defined(__amd64__)
  59 #include <vm/vm.h>
  60 #include <vm/pmap.h>
  61 #endif
  62
  63 #include "common/common.h"
  64 #include "common/t4_msg.h"
  65 #include "common/t4_regs.h"
  66 #include "common/t4_regs_values.h"
  67 #include "t4_ioctl.h"
  68 #include "t4_l2t.h"
  69
  70 /* T4 bus driver interface */
  71 static int t4_probe(device_t);
  72 static int t4_attach(device_t);
  73 static int t4_detach(device_t);
  74 static device_method_t t4_methods[] = {
  75         DEVMETHOD(device_probe,         t4_probe),
  76         DEVMETHOD(device_attach,        t4_attach),
  77         DEVMETHOD(device_detach,        t4_detach),
  78
  79         DEVMETHOD_END
  80 };
  81 static driver_t t4_driver = {
  82         "t4nex",
  83         t4_methods,
  84         sizeof(struct adapter)
  85 };
  86
  87
  88 /* T4 port (cxgbe) interface */
  89 static int cxgbe_probe(device_t);
  90 static int cxgbe_attach(device_t);
  91 static int cxgbe_detach(device_t);
  92 static device_method_t cxgbe_methods[] = {
  93         DEVMETHOD(device_probe,         cxgbe_probe),
  94         DEVMETHOD(device_attach,        cxgbe_attach),
  95         DEVMETHOD(device_detach,        cxgbe_detach),
  96         { 0, 0 }
  97 };
  98 static driver_t cxgbe_driver = {
  99         "cxgbe",
 100         cxgbe_methods,
 101         sizeof(struct port_info)
 102 };
 103
 104 static d_ioctl_t t4_ioctl;
 105 static d_open_t t4_open;
 106 static d_close_t t4_close;
 107
 108 static struct cdevsw t4_cdevsw = {
 109        .d_version = D_VERSION,
 110        .d_flags = 0,
 111        .d_open = t4_open,
 112        .d_close = t4_close,
 113        .d_ioctl = t4_ioctl,
 114        .d_name = "t4nex",
 115 };
 116
 117 /* T5 bus driver interface */
 118 static int t5_probe(device_t);
 119 static device_method_t t5_methods[] = {
 120         DEVMETHOD(device_probe,         t5_probe),
 121         DEVMETHOD(device_attach,        t4_attach),
 122         DEVMETHOD(device_detach,        t4_detach),
 123
 124         DEVMETHOD_END
 125 };
 126 static driver_t t5_driver = {
 127         "t5nex",
 128         t5_methods,
 129         sizeof(struct adapter)
 130 };
 131
 132
 133 /* T5 port (cxl) interface */
 134 static driver_t cxl_driver = {
 135         "cxl",
 136         cxgbe_methods,
 137         sizeof(struct port_info)
 138 };
 139
 140 static struct cdevsw t5_cdevsw = {
 141        .d_version = D_VERSION,
 142        .d_flags = 0,
 143        .d_open = t4_open,
 144        .d_close = t4_close,
 145        .d_ioctl = t4_ioctl,
 146        .d_name = "t5nex",
 147 };
 148
 149 /* ifnet + media interface */
 150 static void cxgbe_init(void *);
 151 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 152 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 153 static void cxgbe_qflush(struct ifnet *);
 154 static int cxgbe_media_change(struct ifnet *);
 155 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 156
 157 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
 158
 159 /*
 160  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 161  * then ADAPTER_LOCK, then t4_uld_list_lock.
 162  */
 163 static struct sx t4_list_lock;
 164 SLIST_HEAD(, adapter) t4_list;
 165 #ifdef TCP_OFFLOAD
 166 static struct sx t4_uld_list_lock;
 167 SLIST_HEAD(, uld_info) t4_uld_list;
 168 #endif
 169
 170 /*
 171  * Tunables.  See tweak_tunables() too.
 172  *
 173  * Each tunable is set to a default value here if it's known at compile-time.
 174  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
 175  * provide a reasonable default when the driver is loaded.
 176  *
 177  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
 178  * T5 are under hw.cxl.
 179  */
 180
 181 /*
 182  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 183  */
 184 #define NTXQ_10G 16
 185 static int t4_ntxq10g = -1;
 186 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 187
 188 #define NRXQ_10G 8
 189 static int t4_nrxq10g = -1;
 190 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 191
 192 #define NTXQ_1G 4
 193 static int t4_ntxq1g = -1;
 194 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 195
 196 #define NRXQ_1G 2
 197 static int t4_nrxq1g = -1;
 198 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 199
 200 static int t4_rsrv_noflowq = 0;
 201 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
 202
 203 #ifdef TCP_OFFLOAD
 204 #define NOFLDTXQ_10G 8
 205 static int t4_nofldtxq10g = -1;
 206 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 207
 208 #define NOFLDRXQ_10G 2
 209 static int t4_nofldrxq10g = -1;
 210 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 211
 212 #define NOFLDTXQ_1G 2
 213 static int t4_nofldtxq1g = -1;
 214 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 215
 216 #define NOFLDRXQ_1G 1
 217 static int t4_nofldrxq1g = -1;
 218 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 219 #endif
 220
 221 #ifdef DEV_NETMAP
 222 #define NNMTXQ_10G 2
 223 static int t4_nnmtxq10g = -1;
 224 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g);
 225
 226 #define NNMRXQ_10G 2
 227 static int t4_nnmrxq10g = -1;
 228 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g);
 229
 230 #define NNMTXQ_1G 1
 231 static int t4_nnmtxq1g = -1;
 232 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g);
 233
 234 #define NNMRXQ_1G 1
 235 static int t4_nnmrxq1g = -1;
 236 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g);
 237 #endif
 238
 239 /*
 240  * Holdoff parameters for 10G and 1G ports.
 241  */
 242 #define TMR_IDX_10G 1
 243 static int t4_tmr_idx_10g = TMR_IDX_10G;
 244 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 245
 246 #define PKTC_IDX_10G (-1)
 247 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 248 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 249
 250 #define TMR_IDX_1G 1
 251 static int t4_tmr_idx_1g = TMR_IDX_1G;
 252 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 253
 254 #define PKTC_IDX_1G (-1)
 255 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 256 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 257
 258 /*
 259  * Size (# of entries) of each tx and rx queue.
 260  */
 261 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 262 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 263
 264 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 265 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 266
 267 /*
 268  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 269  */
 270 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 271 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 272
 273 /*
 274  * Configuration file.
 275  */
 276 #define DEFAULT_CF      "default"
 277 #define FLASH_CF        "flash"
 278 #define UWIRE_CF        "uwire"
 279 #define FPGA_CF         "fpga"
 280 static char t4_cfg_file[32] = DEFAULT_CF;
 281 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 282
 283 /*
 284  * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively).
 285  * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
 286  * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
 287  *            mark or when signalled to do so, 0 to never emit PAUSE.
 288  */
 289 static int t4_pause_settings = PAUSE_TX | PAUSE_RX;
 290 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings);
 291
 292 /*
 293  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
 294  * encouraged respectively).
 295  */
 296 static unsigned int t4_fw_install = 1;
 297 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 298
 299 /*
 300  * ASIC features that will be used.  Disable the ones you don't want so that the
 301  * chip resources aren't wasted on features that will not be used.
 302  */
 303 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 304 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 305
 306 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 307 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 308
 309 static int t4_toecaps_allowed = -1;
 310 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 311
 312 static int t4_rdmacaps_allowed = 0;
 313 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 314
 315 static int t4_iscsicaps_allowed = 0;
 316 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 317
 318 static int t4_fcoecaps_allowed = 0;
 319 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 320
 321 static int t5_write_combine = 0;
 322 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 323
 324 struct intrs_and_queues {
 325         uint16_t intr_type;     /* INTx, MSI, or MSI-X */
 326         uint16_t nirq;          /* Total # of vectors */
 327         uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
 328         uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */
 329         uint16_t ntxq10g;       /* # of NIC txq's for each 10G port */
 330         uint16_t nrxq10g;       /* # of NIC rxq's for each 10G port */
 331         uint16_t ntxq1g;        /* # of NIC txq's for each 1G port */
 332         uint16_t nrxq1g;        /* # of NIC rxq's for each 1G port */
 333         uint16_t rsrv_noflowq;  /* Flag whether to reserve queue 0 */
 334 #ifdef TCP_OFFLOAD
 335         uint16_t nofldtxq10g;   /* # of TOE txq's for each 10G port */
 336         uint16_t nofldrxq10g;   /* # of TOE rxq's for each 10G port */
 337         uint16_t nofldtxq1g;    /* # of TOE txq's for each 1G port */
 338         uint16_t nofldrxq1g;    /* # of TOE rxq's for each 1G port */
 339 #endif
 340 #ifdef DEV_NETMAP
 341         uint16_t nnmtxq10g;     /* # of netmap txq's for each 10G port */
 342         uint16_t nnmrxq10g;     /* # of netmap rxq's for each 10G port */
 343         uint16_t nnmtxq1g;      /* # of netmap txq's for each 1G port */
 344         uint16_t nnmrxq1g;      /* # of netmap rxq's for each 1G port */
 345 #endif
 346 };
 347
 348 struct filter_entry {
 349         uint32_t valid:1;       /* filter allocated and valid */
 350         uint32_t locked:1;      /* filter is administratively locked */
 351         uint32_t pending:1;     /* filter action is pending firmware reply */
 352         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 353         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 354
 355         struct t4_filter_specification fs;
 356 };
 357
 358 static int map_bars_0_and_4(struct adapter *);
 359 static int map_bar_2(struct adapter *);
 360 static void setup_memwin(struct adapter *);
 361 static int validate_mem_range(struct adapter *, uint32_t, int);
 362 static int fwmtype_to_hwmtype(int);
 363 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
 364     uint32_t *);
 365 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
 366 static uint32_t position_memwin(struct adapter *, int, uint32_t);
 367 static int cfg_itype_and_nqueues(struct adapter *, int, int,
 368     struct intrs_and_queues *);
 369 static int prep_firmware(struct adapter *);
 370 static int partition_resources(struct adapter *, const struct firmware *,
 371     const char *);
 372 static int get_params__pre_init(struct adapter *);
 373 static int get_params__post_init(struct adapter *);
 374 static int set_params__post_init(struct adapter *);
 375 static void t4_set_desc(struct adapter *);
 376 static void build_medialist(struct port_info *, struct ifmedia *);
 377 static int cxgbe_init_synchronized(struct port_info *);
 378 static int cxgbe_uninit_synchronized(struct port_info *);
 379 static int setup_intr_handlers(struct adapter *);
 380 static void quiesce_eq(struct adapter *, struct sge_eq *);
 381 static void quiesce_iq(struct adapter *, struct sge_iq *);
 382 static void quiesce_fl(struct adapter *, struct sge_fl *);
 383 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 384     driver_intr_t *, void *, char *);
 385 static int t4_free_irq(struct adapter *, struct irq *);
 386 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
 387     unsigned int);
 388 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 389 static void cxgbe_tick(void *);
 390 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 391 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
 392     struct mbuf *);
 393 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 394 static int fw_msg_not_handled(struct adapter *, const __be64 *);
 395 static int t4_sysctls(struct adapter *);
 396 static int cxgbe_sysctls(struct port_info *);
 397 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 398 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 399 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
 400 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 401 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 402 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 403 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 404 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 405 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
 406 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 407 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 408 #ifdef SBUF_DRAIN
 409 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 410 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 411 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 412 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
 413 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
 414 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 415 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 416 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 417 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 418 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 419 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 420 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 421 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
 422 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 423 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
 424 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 425 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 426 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 427 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 428 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 429 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 430 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
 431 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 432 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 433 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 434 #endif
 435 static inline void txq_start(struct ifnet *, struct sge_txq *);
 436 static uint32_t fconf_to_mode(uint32_t);
 437 static uint32_t mode_to_fconf(uint32_t);
 438 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 439 static int get_filter_mode(struct adapter *, uint32_t *);
 440 static int set_filter_mode(struct adapter *, uint32_t);
 441 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 442 static int get_filter(struct adapter *, struct t4_filter *);
 443 static int set_filter(struct adapter *, struct t4_filter *);
 444 static int del_filter(struct adapter *, struct t4_filter *);
 445 static void clear_filter(struct filter_entry *);
 446 static int set_filter_wr(struct adapter *, int);
 447 static int del_filter_wr(struct adapter *, int);
 448 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 449 static int load_fw(struct adapter *, struct t4_data *);
 450 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 451 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 452 static int set_sched_class(struct adapter *, struct t4_sched_params *);
 453 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
 454 #ifdef TCP_OFFLOAD
 455 static int toe_capability(struct port_info *, int);
 456 #endif
 457 static int mod_event(module_t, int, void *);
 458
 459 struct {
 460         uint16_t device;
 461         char *desc;
 462 } t4_pciids[] = {
 463         {0xa000, "Chelsio Terminator 4 FPGA"},
 464         {0x4400, "Chelsio T440-dbg"},
 465         {0x4401, "Chelsio T420-CR"},
 466         {0x4402, "Chelsio T422-CR"},
 467         {0x4403, "Chelsio T440-CR"},
 468         {0x4404, "Chelsio T420-BCH"},
 469         {0x4405, "Chelsio T440-BCH"},
 470         {0x4406, "Chelsio T440-CH"},
 471         {0x4407, "Chelsio T420-SO"},
 472         {0x4408, "Chelsio T420-CX"},
 473         {0x4409, "Chelsio T420-BT"},
 474         {0x440a, "Chelsio T404-BT"},
 475         {0x440e, "Chelsio T440-LP-CR"},
 476 }, t5_pciids[] = {
 477         {0xb000, "Chelsio Terminator 5 FPGA"},
 478         {0x5400, "Chelsio T580-dbg"},
 479         {0x5401,  "Chelsio T520-CR"},           /* 2 x 10G */
 480         {0x5402,  "Chelsio T522-CR"},           /* 2 x 10G, 2 X 1G */
 481         {0x5403,  "Chelsio T540-CR"},           /* 4 x 10G */
 482         {0x5407,  "Chelsio T520-SO"},           /* 2 x 10G, nomem */
 483         {0x5409,  "Chelsio T520-BT"},           /* 2 x 10GBaseT */
 484         {0x540a,  "Chelsio T504-BT"},           /* 4 x 1G */
 485         {0x540d,  "Chelsio T580-CR"},           /* 2 x 40G */
 486         {0x540e,  "Chelsio T540-LP-CR"},        /* 4 x 10G */
 487         {0x5410,  "Chelsio T580-LP-CR"},        /* 2 x 40G */
 488         {0x5411,  "Chelsio T520-LL-CR"},        /* 2 x 10G */
 489         {0x5412,  "Chelsio T560-CR"},           /* 1 x 40G, 2 x 10G */
 490         {0x5414,  "Chelsio T580-LP-SO-CR"},     /* 2 x 40G, nomem */
 491 #ifdef notyet
 492         {0x5404,  "Chelsio T520-BCH"},
 493         {0x5405,  "Chelsio T540-BCH"},
 494         {0x5406,  "Chelsio T540-CH"},
 495         {0x5408,  "Chelsio T520-CX"},
 496         {0x540b,  "Chelsio B520-SR"},
 497         {0x540c,  "Chelsio B504-BT"},
 498         {0x540f,  "Chelsio Amsterdam"},
 499         {0x5413,  "Chelsio T580-CHR"},
 500 #endif
 501 };
 502
 503 #ifdef TCP_OFFLOAD
 504 /*
 505  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 506  * exactly the same for both rxq and ofld_rxq.
 507  */
 508 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 509 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 510 #endif
 511
 512 /* No easy way to include t4_msg.h before adapter.h so we check this way */
 513 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
 514 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
 515
 516 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
 517
 518 static int
 519 t4_probe(device_t dev)
 520 {
 521         int i;
 522         uint16_t v = pci_get_vendor(dev);
 523         uint16_t d = pci_get_device(dev);
 524         uint8_t f = pci_get_function(dev);
 525
 526         if (v != PCI_VENDOR_ID_CHELSIO)
 527                 return (ENXIO);
 528
 529         /* Attach only to PF0 of the FPGA */
 530         if (d == 0xa000 && f != 0)
 531                 return (ENXIO);
 532
 533         for (i = 0; i < nitems(t4_pciids); i++) {
 534                 if (d == t4_pciids[i].device) {
 535                         device_set_desc(dev, t4_pciids[i].desc);
 536                         return (BUS_PROBE_DEFAULT);
 537                 }
 538         }
 539
 540         return (ENXIO);
 541 }
 542
 543 static int
 544 t5_probe(device_t dev)
 545 {
 546         int i;
 547         uint16_t v = pci_get_vendor(dev);
 548         uint16_t d = pci_get_device(dev);
 549         uint8_t f = pci_get_function(dev);
 550
 551         if (v != PCI_VENDOR_ID_CHELSIO)
 552                 return (ENXIO);
 553
 554         /* Attach only to PF0 of the FPGA */
 555         if (d == 0xb000 && f != 0)
 556                 return (ENXIO);
 557
 558         for (i = 0; i < nitems(t5_pciids); i++) {
 559                 if (d == t5_pciids[i].device) {
 560                         device_set_desc(dev, t5_pciids[i].desc);
 561                         return (BUS_PROBE_DEFAULT);
 562                 }
 563         }
 564
 565         return (ENXIO);
 566 }
 567
 568 static int
 569 t4_attach(device_t dev)
 570 {
 571         struct adapter *sc;
 572         int rc = 0, i, n10g, n1g, rqidx, tqidx;
 573         struct intrs_and_queues iaq;
 574         struct sge *s;
 575 #ifdef TCP_OFFLOAD
 576         int ofld_rqidx, ofld_tqidx;
 577 #endif
 578 #ifdef DEV_NETMAP
 579         int nm_rqidx, nm_tqidx;
 580 #endif
 581
 582         sc = device_get_softc(dev);
 583         sc->dev = dev;
 584
 585         pci_enable_busmaster(dev);
 586         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 587                 uint32_t v;
 588
 589                 pci_set_max_read_req(dev, 4096);
 590                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 591                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 592                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 593         }
 594
 595         sc->traceq = -1;
 596         mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
 597         snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
 598             device_get_nameunit(dev));
 599
 600         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 601             device_get_nameunit(dev));
 602         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 603         sx_xlock(&t4_list_lock);
 604         SLIST_INSERT_HEAD(&t4_list, sc, link);
 605         sx_xunlock(&t4_list_lock);
 606
 607         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 608         TAILQ_INIT(&sc->sfl);
 609         callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
 610
 611         rc = map_bars_0_and_4(sc);
 612         if (rc != 0)
 613                 goto done; /* error message displayed already */
 614
 615         /*
 616          * This is the real PF# to which we're attaching.  Works from within PCI
 617          * passthrough environments too, where pci_get_function() could return a
 618          * different PF# depending on the passthrough configuration.  We need to
 619          * use the real PF# in all our communication with the firmware.
 620          */
 621         sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 622         sc->mbox = sc->pf;
 623
 624         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 625         sc->an_handler = an_not_handled;
 626         for (i = 0; i < nitems(sc->cpl_handler); i++)
 627                 sc->cpl_handler[i] = cpl_not_handled;
 628         for (i = 0; i < nitems(sc->fw_msg_handler); i++)
 629                 sc->fw_msg_handler[i] = fw_msg_not_handled;
 630         t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 631         t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
 632         t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt);
 633         t4_init_sge_cpl_handlers(sc);
 634
 635         /* Prepare the adapter for operation */
 636         rc = -t4_prep_adapter(sc);
 637         if (rc != 0) {
 638                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 639                 goto done;
 640         }
 641
 642         /*
 643          * Do this really early, with the memory windows set up even before the
 644          * character device.  The userland tool's register i/o and mem read
 645          * will work even in "recovery mode".
 646          */
 647         setup_memwin(sc);
 648         sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
 649             device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
 650             device_get_nameunit(dev));
 651         if (sc->cdev == NULL)
 652                 device_printf(dev, "failed to create nexus char device.\n");
 653         else
 654                 sc->cdev->si_drv1 = sc;
 655
 656         /* Go no further if recovery mode has been requested. */
 657         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 658                 device_printf(dev, "recovery mode.\n");
 659                 goto done;
 660         }
 661
 662         /* Prepare the firmware for operation */
 663         rc = prep_firmware(sc);
 664         if (rc != 0)
 665                 goto done; /* error message displayed already */
 666
 667         rc = get_params__post_init(sc);
 668         if (rc != 0)
 669                 goto done; /* error message displayed already */
 670
 671         rc = set_params__post_init(sc);
 672         if (rc != 0)
 673                 goto done; /* error message displayed already */
 674
 675         rc = map_bar_2(sc);
 676         if (rc != 0)
 677                 goto done; /* error message displayed already */
 678
 679         rc = t4_create_dma_tag(sc);
 680         if (rc != 0)
 681                 goto done; /* error message displayed already */
 682
 683         /*
 684          * First pass over all the ports - allocate VIs and initialize some
 685          * basic parameters like mac address, port type, etc.  We also figure
 686          * out whether a port is 10G or 1G and use that information when
 687          * calculating how many interrupts to attempt to allocate.
 688          */
 689         n10g = n1g = 0;
 690         for_each_port(sc, i) {
 691                 struct port_info *pi;
 692
 693                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 694                 sc->port[i] = pi;
 695
 696                 /* These must be set before t4_port_init */
 697                 pi->adapter = sc;
 698                 pi->port_id = i;
 699
 700                 /* Allocate the vi and initialize parameters like mac addr */
 701                 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 702                 if (rc != 0) {
 703                         device_printf(dev, "unable to initialize port %d: %d\n",
 704                             i, rc);
 705                         free(pi, M_CXGBE);
 706                         sc->port[i] = NULL;
 707                         goto done;
 708                 }
 709
 710                 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX);
 711                 pi->link_cfg.requested_fc |= t4_pause_settings;
 712                 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX);
 713                 pi->link_cfg.fc |= t4_pause_settings;
 714
 715                 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
 716                 if (rc != 0) {
 717                         device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
 718                         free(pi, M_CXGBE);
 719                         sc->port[i] = NULL;
 720                         goto done;
 721                 }
 722
 723                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 724                     device_get_nameunit(dev), i);
 725                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 726                 sc->chan_map[pi->tx_chan] = i;
 727
 728                 if (is_10G_port(pi) || is_40G_port(pi)) {
 729                         n10g++;
 730                         pi->tmr_idx = t4_tmr_idx_10g;
 731                         pi->pktc_idx = t4_pktc_idx_10g;
 732                 } else {
 733                         n1g++;
 734                         pi->tmr_idx = t4_tmr_idx_1g;
 735                         pi->pktc_idx = t4_pktc_idx_1g;
 736                 }
 737
 738                 pi->xact_addr_filt = -1;
 739                 pi->linkdnrc = -1;
 740
 741                 pi->qsize_rxq = t4_qsize_rxq;
 742                 pi->qsize_txq = t4_qsize_txq;
 743
 744                 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
 745                 if (pi->dev == NULL) {
 746                         device_printf(dev,
 747                             "failed to add device for port %d.\n", i);
 748                         rc = ENXIO;
 749                         goto done;
 750                 }
 751                 device_set_softc(pi->dev, pi);
 752         }
 753
 754         /*
 755          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 756          */
 757         rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
 758         if (rc != 0)
 759                 goto done; /* error message displayed already */
 760
 761         sc->intr_type = iaq.intr_type;
 762         sc->intr_count = iaq.nirq;
 763
 764         s = &sc->sge;
 765         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 766         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 767         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
 768         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 769         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
 770 #ifdef TCP_OFFLOAD
 771         if (is_offload(sc)) {
 772                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 773                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 774                 s->neq += s->nofldtxq + s->nofldrxq;
 775                 s->niq += s->nofldrxq;
 776
 777                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 778                     M_CXGBE, M_ZERO | M_WAITOK);
 779                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 780                     M_CXGBE, M_ZERO | M_WAITOK);
 781         }
 782 #endif
 783 #ifdef DEV_NETMAP
 784         s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g;
 785         s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g;
 786         s->neq += s->nnmtxq + s->nnmrxq;
 787         s->niq += s->nnmrxq;
 788
 789         s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
 790             M_CXGBE, M_ZERO | M_WAITOK);
 791         s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
 792             M_CXGBE, M_ZERO | M_WAITOK);
 793 #endif
 794
 795         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 796             M_ZERO | M_WAITOK);
 797         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 798             M_ZERO | M_WAITOK);
 799         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 800             M_ZERO | M_WAITOK);
 801         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 802             M_ZERO | M_WAITOK);
 803         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 804             M_ZERO | M_WAITOK);
 805
 806         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 807             M_ZERO | M_WAITOK);
 808
 809         t4_init_l2t(sc, M_WAITOK);
 810
 811         /*
 812          * Second pass over the ports.  This time we know the number of rx and
 813          * tx queues that each port should get.
 814          */
 815         rqidx = tqidx = 0;
 816 #ifdef TCP_OFFLOAD
 817         ofld_rqidx = ofld_tqidx = 0;
 818 #endif
 819 #ifdef DEV_NETMAP
 820         nm_rqidx = nm_tqidx = 0;
 821 #endif
 822         for_each_port(sc, i) {
 823                 struct port_info *pi = sc->port[i];
 824
 825                 if (pi == NULL)
 826                         continue;
 827
 828                 pi->first_rxq = rqidx;
 829                 pi->first_txq = tqidx;
 830                 if (is_10G_port(pi) || is_40G_port(pi)) {
 831                         pi->flags |= iaq.intr_flags_10g;
 832                         pi->nrxq = iaq.nrxq10g;
 833                         pi->ntxq = iaq.ntxq10g;
 834                 } else {
 835                         pi->flags |= iaq.intr_flags_1g;
 836                         pi->nrxq = iaq.nrxq1g;
 837                         pi->ntxq = iaq.ntxq1g;
 838                 }
 839
 840                 if (pi->ntxq > 1)
 841                         pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
 842                 else
 843                         pi->rsrv_noflowq = 0;
 844
 845                 rqidx += pi->nrxq;
 846                 tqidx += pi->ntxq;
 847 #ifdef TCP_OFFLOAD
 848                 if (is_offload(sc)) {
 849                         pi->first_ofld_rxq = ofld_rqidx;
 850                         pi->first_ofld_txq = ofld_tqidx;
 851                         if (is_10G_port(pi) || is_40G_port(pi)) {
 852                                 pi->nofldrxq = iaq.nofldrxq10g;
 853                                 pi->nofldtxq = iaq.nofldtxq10g;
 854                         } else {
 855                                 pi->nofldrxq = iaq.nofldrxq1g;
 856                                 pi->nofldtxq = iaq.nofldtxq1g;
 857                         }
 858                         ofld_rqidx += pi->nofldrxq;
 859                         ofld_tqidx += pi->nofldtxq;
 860                 }
 861 #endif
 862 #ifdef DEV_NETMAP
 863                 pi->first_nm_rxq = nm_rqidx;
 864                 pi->first_nm_txq = nm_tqidx;
 865                 if (is_10G_port(pi) || is_40G_port(pi)) {
 866                         pi->nnmrxq = iaq.nnmrxq10g;
 867                         pi->nnmtxq = iaq.nnmtxq10g;
 868                 } else {
 869                         pi->nnmrxq = iaq.nnmrxq1g;
 870                         pi->nnmtxq = iaq.nnmtxq1g;
 871                 }
 872                 nm_rqidx += pi->nnmrxq;
 873                 nm_tqidx += pi->nnmtxq;
 874 #endif
 875         }
 876
 877         rc = setup_intr_handlers(sc);
 878         if (rc != 0) {
 879                 device_printf(dev,
 880                     "failed to setup interrupt handlers: %d\n", rc);
 881                 goto done;
 882         }
 883
 884         rc = bus_generic_attach(dev);
 885         if (rc != 0) {
 886                 device_printf(dev,
 887                     "failed to attach all child ports: %d\n", rc);
 888                 goto done;
 889         }
 890
 891         device_printf(dev,
 892             "PCIe x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
 893             sc->params.pci.width, sc->params.nports, sc->intr_count,
 894             sc->intr_type == INTR_MSIX ? "MSI-X" :
 895             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
 896             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
 897
 898         t4_set_desc(sc);
 899
 900 done:
 901         if (rc != 0 && sc->cdev) {
 902                 /* cdev was created and so cxgbetool works; recover that way. */
 903                 device_printf(dev,
 904                     "error during attach, adapter is now in recovery mode.\n");
 905                 rc = 0;
 906         }
 907
 908         if (rc != 0)
 909                 t4_detach(dev);
 910         else
 911                 t4_sysctls(sc);
 912
 913         return (rc);
 914 }
 915
 916 /*
 917  * Idempotent
 918  */
 919 static int
 920 t4_detach(device_t dev)
 921 {
 922         struct adapter *sc;
 923         struct port_info *pi;
 924         int i, rc;
 925
 926         sc = device_get_softc(dev);
 927
 928         if (sc->flags & FULL_INIT_DONE)
 929                 t4_intr_disable(sc);
 930
 931         if (sc->cdev) {
 932                 destroy_dev(sc->cdev);
 933                 sc->cdev = NULL;
 934         }
 935
 936         rc = bus_generic_detach(dev);
 937         if (rc) {
 938                 device_printf(dev,
 939                     "failed to detach child devices: %d\n", rc);
 940                 return (rc);
 941         }
 942
 943         for (i = 0; i < sc->intr_count; i++)
 944                 t4_free_irq(sc, &sc->irq[i]);
 945
 946         for (i = 0; i < MAX_NPORTS; i++) {
 947                 pi = sc->port[i];
 948                 if (pi) {
 949                         t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid);
 950                         if (pi->dev)
 951                                 device_delete_child(dev, pi->dev);
 952
 953                         mtx_destroy(&pi->pi_lock);
 954                         free(pi, M_CXGBE);
 955                 }
 956         }
 957
 958         if (sc->flags & FULL_INIT_DONE)
 959                 adapter_full_uninit(sc);
 960
 961         if (sc->flags & FW_OK)
 962                 t4_fw_bye(sc, sc->mbox);
 963
 964         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
 965                 pci_release_msi(dev);
 966
 967         if (sc->regs_res)
 968                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
 969                     sc->regs_res);
 970
 971         if (sc->udbs_res)
 972                 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
 973                     sc->udbs_res);
 974
 975         if (sc->msix_res)
 976                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
 977                     sc->msix_res);
 978
 979         if (sc->l2t)
 980                 t4_free_l2t(sc->l2t);
 981
 982 #ifdef TCP_OFFLOAD
 983         free(sc->sge.ofld_rxq, M_CXGBE);
 984         free(sc->sge.ofld_txq, M_CXGBE);
 985 #endif
 986 #ifdef DEV_NETMAP
 987         free(sc->sge.nm_rxq, M_CXGBE);
 988         free(sc->sge.nm_txq, M_CXGBE);
 989 #endif
 990         free(sc->irq, M_CXGBE);
 991         free(sc->sge.rxq, M_CXGBE);
 992         free(sc->sge.txq, M_CXGBE);
 993         free(sc->sge.ctrlq, M_CXGBE);
 994         free(sc->sge.iqmap, M_CXGBE);
 995         free(sc->sge.eqmap, M_CXGBE);
 996         free(sc->tids.ftid_tab, M_CXGBE);
 997         t4_destroy_dma_tag(sc);
 998         if (mtx_initialized(&sc->sc_lock)) {
 999                 sx_xlock(&t4_list_lock);
1000                 SLIST_REMOVE(&t4_list, sc, adapter, link);
1001                 sx_xunlock(&t4_list_lock);
1002                 mtx_destroy(&sc->sc_lock);
1003         }
1004
1005         if (mtx_initialized(&sc->tids.ftid_lock))
1006                 mtx_destroy(&sc->tids.ftid_lock);
1007         if (mtx_initialized(&sc->sfl_lock))
1008                 mtx_destroy(&sc->sfl_lock);
1009         if (mtx_initialized(&sc->ifp_lock))
1010                 mtx_destroy(&sc->ifp_lock);
1011
1012         bzero(sc, sizeof(*sc));
1013
1014         return (0);
1015 }
1016
1017 static int
1018 cxgbe_probe(device_t dev)
1019 {
1020         char buf[128];
1021         struct port_info *pi = device_get_softc(dev);
1022
1023         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
1024         device_set_desc_copy(dev, buf);
1025
1026         return (BUS_PROBE_DEFAULT);
1027 }
1028
1029 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
1030     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1031     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
1032 #define T4_CAP_ENABLE (T4_CAP)
1033
1034 static int
1035 cxgbe_attach(device_t dev)
1036 {
1037         struct port_info *pi = device_get_softc(dev);
1038         struct ifnet *ifp;
1039         char *s;
1040         int n, o;
1041
1042         /* Allocate an ifnet and set it up */
1043         ifp = if_alloc(IFT_ETHER);
1044         if (ifp == NULL) {
1045                 device_printf(dev, "Cannot allocate ifnet\n");
1046                 return (ENOMEM);
1047         }
1048         pi->ifp = ifp;
1049         ifp->if_softc = pi;
1050
1051         callout_init(&pi->tick, CALLOUT_MPSAFE);
1052
1053         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1054         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1055
1056         ifp->if_init = cxgbe_init;
1057         ifp->if_ioctl = cxgbe_ioctl;
1058         ifp->if_transmit = cxgbe_transmit;
1059         ifp->if_qflush = cxgbe_qflush;
1060
1061         ifp->if_capabilities = T4_CAP;
1062 #ifdef TCP_OFFLOAD
1063         if (is_offload(pi->adapter))
1064                 ifp->if_capabilities |= IFCAP_TOE;
1065 #endif
1066         ifp->if_capenable = T4_CAP_ENABLE;
1067         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1068             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1069
1070         /* Initialize ifmedia for this port */
1071         ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1072             cxgbe_media_status);
1073         build_medialist(pi, &pi->media);
1074
1075         pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1076             EVENTHANDLER_PRI_ANY);
1077
1078         ether_ifattach(ifp, pi->hw_addr);
1079
1080         n = 128;
1081         s = malloc(n, M_CXGBE, M_WAITOK);
1082         o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq);
1083         MPASS(n > o);
1084 #ifdef TCP_OFFLOAD
1085         if (is_offload(pi->adapter)) {
1086                 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)",
1087                     pi->nofldtxq, pi->nofldrxq);
1088                 MPASS(n > o);
1089         }
1090 #endif
1091 #ifdef DEV_NETMAP
1092         o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq,
1093             pi->nnmrxq);
1094         MPASS(n > o);
1095 #endif
1096         device_printf(dev, "%s\n", s);
1097         free(s, M_CXGBE);
1098
1099 #ifdef DEV_NETMAP
1100         /* nm_media handled here to keep implementation private to this file */
1101         ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change,
1102             cxgbe_media_status);
1103         build_medialist(pi, &pi->nm_media);
1104         create_netmap_ifnet(pi);        /* logs errors it something fails */
1105 #endif
1106         cxgbe_sysctls(pi);
1107
1108         return (0);
1109 }
1110
1111 static int
1112 cxgbe_detach(device_t dev)
1113 {
1114         struct port_info *pi = device_get_softc(dev);
1115         struct adapter *sc = pi->adapter;
1116         struct ifnet *ifp = pi->ifp;
1117
1118         /* Tell if_ioctl and if_init that the port is going away */
1119         ADAPTER_LOCK(sc);
1120         SET_DOOMED(pi);
1121         wakeup(&sc->flags);
1122         while (IS_BUSY(sc))
1123                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
1124         SET_BUSY(sc);
1125 #ifdef INVARIANTS
1126         sc->last_op = "t4detach";
1127         sc->last_op_thr = curthread;
1128 #endif
1129         ADAPTER_UNLOCK(sc);
1130
1131         if (pi->flags & HAS_TRACEQ) {
1132                 sc->traceq = -1;        /* cloner should not create ifnet */
1133                 t4_tracer_port_detach(sc);
1134         }
1135
1136         if (pi->vlan_c)
1137                 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
1138
1139         PORT_LOCK(pi);
1140         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1141         callout_stop(&pi->tick);
1142         PORT_UNLOCK(pi);
1143         callout_drain(&pi->tick);
1144
1145         /* Let detach proceed even if these fail. */
1146         cxgbe_uninit_synchronized(pi);
1147         port_full_uninit(pi);
1148
1149         ifmedia_removeall(&pi->media);
1150         ether_ifdetach(pi->ifp);
1151         if_free(pi->ifp);
1152
1153 #ifdef DEV_NETMAP
1154         /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */
1155         destroy_netmap_ifnet(pi);
1156 #endif
1157
1158         ADAPTER_LOCK(sc);
1159         CLR_BUSY(sc);
1160         wakeup(&sc->flags);
1161         ADAPTER_UNLOCK(sc);
1162
1163         return (0);
1164 }
1165
1166 static void
1167 cxgbe_init(void *arg)
1168 {
1169         struct port_info *pi = arg;
1170         struct adapter *sc = pi->adapter;
1171
1172         if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
1173                 return;
1174         cxgbe_init_synchronized(pi);
1175         end_synchronized_op(sc, 0);
1176 }
1177
1178 static int
1179 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1180 {
1181         int rc = 0, mtu, flags, can_sleep;
1182         struct port_info *pi = ifp->if_softc;
1183         struct adapter *sc = pi->adapter;
1184         struct ifreq *ifr = (struct ifreq *)data;
1185         uint32_t mask;
1186
1187         switch (cmd) {
1188         case SIOCSIFMTU:
1189                 mtu = ifr->ifr_mtu;
1190                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1191                         return (EINVAL);
1192
1193                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
1194                 if (rc)
1195                         return (rc);
1196                 ifp->if_mtu = mtu;
1197                 if (pi->flags & PORT_INIT_DONE) {
1198                         t4_update_fl_bufsize(ifp);
1199                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1200                                 rc = update_mac_settings(ifp, XGMAC_MTU);
1201                 }
1202                 end_synchronized_op(sc, 0);
1203                 break;
1204
1205         case SIOCSIFFLAGS:
1206                 can_sleep = 0;
1207 redo_sifflags:
1208                 rc = begin_synchronized_op(sc, pi,
1209                     can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg");
1210                 if (rc)
1211                         return (rc);
1212
1213                 if (ifp->if_flags & IFF_UP) {
1214                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1215                                 flags = pi->if_flags;
1216                                 if ((ifp->if_flags ^ flags) &
1217                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1218                                         if (can_sleep == 1) {
1219                                                 end_synchronized_op(sc, 0);
1220                                                 can_sleep = 0;
1221                                                 goto redo_sifflags;
1222                                         }
1223                                         rc = update_mac_settings(ifp,
1224                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1225                                 }
1226                         } else {
1227                                 if (can_sleep == 0) {
1228                                         end_synchronized_op(sc, LOCK_HELD);
1229                                         can_sleep = 1;
1230                                         goto redo_sifflags;
1231                                 }
1232                                 rc = cxgbe_init_synchronized(pi);
1233                         }
1234                         pi->if_flags = ifp->if_flags;
1235                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1236                         if (can_sleep == 0) {
1237                                 end_synchronized_op(sc, LOCK_HELD);
1238                                 can_sleep = 1;
1239                                 goto redo_sifflags;
1240                         }
1241                         rc = cxgbe_uninit_synchronized(pi);
1242                 }
1243                 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD);
1244                 break;
1245
1246         case SIOCADDMULTI:
1247         case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1248                 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
1249                 if (rc)
1250                         return (rc);
1251                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1252                         rc = update_mac_settings(ifp, XGMAC_MCADDRS);
1253                 end_synchronized_op(sc, LOCK_HELD);
1254                 break;
1255
1256         case SIOCSIFCAP:
1257                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
1258                 if (rc)
1259                         return (rc);
1260
1261                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1262                 if (mask & IFCAP_TXCSUM) {
1263                         ifp->if_capenable ^= IFCAP_TXCSUM;
1264                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1265
1266                         if (IFCAP_TSO4 & ifp->if_capenable &&
1267                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1268                                 ifp->if_capenable &= ~IFCAP_TSO4;
1269                                 if_printf(ifp,
1270                                     "tso4 disabled due to -txcsum.\n");
1271                         }
1272                 }
1273                 if (mask & IFCAP_TXCSUM_IPV6) {
1274                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1275                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1276
1277                         if (IFCAP_TSO6 & ifp->if_capenable &&
1278                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1279                                 ifp->if_capenable &= ~IFCAP_TSO6;
1280                                 if_printf(ifp,
1281                                     "tso6 disabled due to -txcsum6.\n");
1282                         }
1283                 }
1284                 if (mask & IFCAP_RXCSUM)
1285                         ifp->if_capenable ^= IFCAP_RXCSUM;
1286                 if (mask & IFCAP_RXCSUM_IPV6)
1287                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1288
1289                 /*
1290                  * Note that we leave CSUM_TSO alone (it is always set).  The
1291                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1292                  * sending a TSO request our way, so it's sufficient to toggle
1293                  * IFCAP_TSOx only.
1294                  */
1295                 if (mask & IFCAP_TSO4) {
1296                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1297                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1298                                 if_printf(ifp, "enable txcsum first.\n");
1299                                 rc = EAGAIN;
1300                                 goto fail;
1301                         }
1302                         ifp->if_capenable ^= IFCAP_TSO4;
1303                 }
1304                 if (mask & IFCAP_TSO6) {
1305                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1306                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1307                                 if_printf(ifp, "enable txcsum6 first.\n");
1308                                 rc = EAGAIN;
1309                                 goto fail;
1310                         }
1311                         ifp->if_capenable ^= IFCAP_TSO6;
1312                 }
1313                 if (mask & IFCAP_LRO) {
1314 #if defined(INET) || defined(INET6)
1315                         int i;
1316                         struct sge_rxq *rxq;
1317
1318                         ifp->if_capenable ^= IFCAP_LRO;
1319                         for_each_rxq(pi, i, rxq) {
1320                                 if (ifp->if_capenable & IFCAP_LRO)
1321                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1322                                 else
1323                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1324                         }
1325 #endif
1326                 }
1327 #ifdef TCP_OFFLOAD
1328                 if (mask & IFCAP_TOE) {
1329                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1330
1331                         rc = toe_capability(pi, enable);
1332                         if (rc != 0)
1333                                 goto fail;
1334
1335                         ifp->if_capenable ^= mask;
1336                 }
1337 #endif
1338                 if (mask & IFCAP_VLAN_HWTAGGING) {
1339                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1340                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1341                                 rc = update_mac_settings(ifp, XGMAC_VLANEX);
1342                 }
1343                 if (mask & IFCAP_VLAN_MTU) {
1344                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1345
1346                         /* Need to find out how to disable auto-mtu-inflation */
1347                 }
1348                 if (mask & IFCAP_VLAN_HWTSO)
1349                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1350                 if (mask & IFCAP_VLAN_HWCSUM)
1351                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1352
1353 #ifdef VLAN_CAPABILITIES
1354                 VLAN_CAPABILITIES(ifp);
1355 #endif
1356 fail:
1357                 end_synchronized_op(sc, 0);
1358                 break;
1359
1360         case SIOCSIFMEDIA:
1361         case SIOCGIFMEDIA:
1362                 ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1363                 break;
1364
1365         default:
1366                 rc = ether_ioctl(ifp, cmd, data);
1367         }
1368
1369         return (rc);
1370 }
1371
1372 static int
1373 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1374 {
1375         struct port_info *pi = ifp->if_softc;
1376         struct adapter *sc = pi->adapter;
1377         struct sge_txq *txq = &sc->sge.txq[pi->first_txq];
1378         struct buf_ring *br;
1379         int rc;
1380
1381         M_ASSERTPKTHDR(m);
1382
1383         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1384                 m_freem(m);
1385                 return (ENETDOWN);
1386         }
1387
1388         if (m->m_flags & M_FLOWID)
1389                 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq))
1390                     + pi->rsrv_noflowq);
1391         br = txq->br;
1392
1393         if (TXQ_TRYLOCK(txq) == 0) {
1394                 struct sge_eq *eq = &txq->eq;
1395
1396                 /*
1397                  * It is possible that t4_eth_tx finishes up and releases the
1398                  * lock between the TRYLOCK above and the drbr_enqueue here.  We
1399                  * need to make sure that this mbuf doesn't just sit there in
1400                  * the drbr.
1401                  */
1402
1403                 rc = drbr_enqueue(ifp, br, m);
1404                 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 &&
1405                     !(eq->flags & EQ_DOOMED))
1406                         callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq);
1407                 return (rc);
1408         }
1409
1410         /*
1411          * txq->m is the mbuf that is held up due to a temporary shortage of
1412          * resources and it should be put on the wire first.  Then what's in
1413          * drbr and finally the mbuf that was just passed in to us.
1414          *
1415          * Return code should indicate the fate of the mbuf that was passed in
1416          * this time.
1417          */
1418
1419         TXQ_LOCK_ASSERT_OWNED(txq);
1420         if (drbr_needs_enqueue(ifp, br) || txq->m) {
1421
1422                 /* Queued for transmission. */
1423
1424                 rc = drbr_enqueue(ifp, br, m);
1425                 m = txq->m ? txq->m : drbr_dequeue(ifp, br);
1426                 (void) t4_eth_tx(ifp, txq, m);
1427                 TXQ_UNLOCK(txq);
1428                 return (rc);
1429         }
1430
1431         /* Direct transmission. */
1432         rc = t4_eth_tx(ifp, txq, m);
1433         if (rc != 0 && txq->m)
1434                 rc = 0; /* held, will be transmitted soon (hopefully) */
1435
1436         TXQ_UNLOCK(txq);
1437         return (rc);
1438 }
1439
1440 static void
1441 cxgbe_qflush(struct ifnet *ifp)
1442 {
1443         struct port_info *pi = ifp->if_softc;
1444         struct sge_txq *txq;
1445         int i;
1446         struct mbuf *m;
1447
1448         /* queues do not exist if !PORT_INIT_DONE. */
1449         if (pi->flags & PORT_INIT_DONE) {
1450                 for_each_txq(pi, i, txq) {
1451                         TXQ_LOCK(txq);
1452                         m_freem(txq->m);
1453                         txq->m = NULL;
1454                         while ((m = buf_ring_dequeue_sc(txq->br)) != NULL)
1455                                 m_freem(m);
1456                         TXQ_UNLOCK(txq);
1457                 }
1458         }
1459         if_qflush(ifp);
1460 }
1461
1462 static int
1463 cxgbe_media_change(struct ifnet *ifp)
1464 {
1465         struct port_info *pi = ifp->if_softc;
1466
1467         device_printf(pi->dev, "%s unimplemented.\n", __func__);
1468
1469         return (EOPNOTSUPP);
1470 }
1471
1472 static void
1473 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1474 {
1475         struct port_info *pi = ifp->if_softc;
1476         struct ifmedia *media = NULL;
1477         struct ifmedia_entry *cur;
1478         int speed = pi->link_cfg.speed;
1479         int data = (pi->port_type << 8) | pi->mod_type;
1480
1481         if (ifp == pi->ifp)
1482                 media = &pi->media;
1483 #ifdef DEV_NETMAP
1484         else if (ifp == pi->nm_ifp)
1485                 media = &pi->nm_media;
1486 #endif
1487         MPASS(media != NULL);
1488
1489         cur = media->ifm_cur;
1490         if (cur->ifm_data != data) {
1491                 build_medialist(pi, media);
1492                 cur = media->ifm_cur;
1493         }
1494
1495         ifmr->ifm_status = IFM_AVALID;
1496         if (!pi->link_cfg.link_ok)
1497                 return;
1498
1499         ifmr->ifm_status |= IFM_ACTIVE;
1500
1501         /* active and current will differ iff current media is autoselect. */
1502         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1503                 return;
1504
1505         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1506         if (speed == SPEED_10000)
1507                 ifmr->ifm_active |= IFM_10G_T;
1508         else if (speed == SPEED_1000)
1509                 ifmr->ifm_active |= IFM_1000_T;
1510         else if (speed == SPEED_100)
1511                 ifmr->ifm_active |= IFM_100_TX;
1512         else if (speed == SPEED_10)
1513                 ifmr->ifm_active |= IFM_10_T;
1514         else
1515                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1516                             speed));
1517 }
1518
1519 void
1520 t4_fatal_err(struct adapter *sc)
1521 {
1522         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1523         t4_intr_disable(sc);
1524         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1525             device_get_nameunit(sc->dev));
1526 }
1527
1528 static int
1529 map_bars_0_and_4(struct adapter *sc)
1530 {
1531         sc->regs_rid = PCIR_BAR(0);
1532         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1533             &sc->regs_rid, RF_ACTIVE);
1534         if (sc->regs_res == NULL) {
1535                 device_printf(sc->dev, "cannot map registers.\n");
1536                 return (ENXIO);
1537         }
1538         sc->bt = rman_get_bustag(sc->regs_res);
1539         sc->bh = rman_get_bushandle(sc->regs_res);
1540         sc->mmio_len = rman_get_size(sc->regs_res);
1541         setbit(&sc->doorbells, DOORBELL_KDB);
1542
1543         sc->msix_rid = PCIR_BAR(4);
1544         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1545             &sc->msix_rid, RF_ACTIVE);
1546         if (sc->msix_res == NULL) {
1547                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1548                 return (ENXIO);
1549         }
1550
1551         return (0);
1552 }
1553
1554 static int
1555 map_bar_2(struct adapter *sc)
1556 {
1557
1558         /*
1559          * T4: only iWARP driver uses the userspace doorbells.  There is no need
1560          * to map it if RDMA is disabled.
1561          */
1562         if (is_t4(sc) && sc->rdmacaps == 0)
1563                 return (0);
1564
1565         sc->udbs_rid = PCIR_BAR(2);
1566         sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1567             &sc->udbs_rid, RF_ACTIVE);
1568         if (sc->udbs_res == NULL) {
1569                 device_printf(sc->dev, "cannot map doorbell BAR.\n");
1570                 return (ENXIO);
1571         }
1572         sc->udbs_base = rman_get_virtual(sc->udbs_res);
1573
1574         if (is_t5(sc)) {
1575                 setbit(&sc->doorbells, DOORBELL_UDB);
1576 #if defined(__i386__) || defined(__amd64__)
1577                 if (t5_write_combine) {
1578                         int rc;
1579
1580                         /*
1581                          * Enable write combining on BAR2.  This is the
1582                          * userspace doorbell BAR and is split into 128B
1583                          * (UDBS_SEG_SIZE) doorbell regions, each associated
1584                          * with an egress queue.  The first 64B has the doorbell
1585                          * and the second 64B can be used to submit a tx work
1586                          * request with an implicit doorbell.
1587                          */
1588
1589                         rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
1590                             rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
1591                         if (rc == 0) {
1592                                 clrbit(&sc->doorbells, DOORBELL_UDB);
1593                                 setbit(&sc->doorbells, DOORBELL_WCWR);
1594                                 setbit(&sc->doorbells, DOORBELL_UDBWC);
1595                         } else {
1596                                 device_printf(sc->dev,
1597                                     "couldn't enable write combining: %d\n",
1598                                     rc);
1599                         }
1600
1601                         t4_write_reg(sc, A_SGE_STAT_CFG,
1602                             V_STATSOURCE_T5(7) | V_STATMODE(0));
1603                 }
1604 #endif
1605         }
1606
1607         return (0);
1608 }
1609
1610 static const struct memwin t4_memwin[] = {
1611         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1612         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1613         { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
1614 };
1615
1616 static const struct memwin t5_memwin[] = {
1617         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1618         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1619         { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1620 };
1621
1622 static void
1623 setup_memwin(struct adapter *sc)
1624 {
1625         const struct memwin *mw;
1626         int i, n;
1627         uint32_t bar0;
1628
1629         if (is_t4(sc)) {
1630                 /*
1631                  * Read low 32b of bar0 indirectly via the hardware backdoor
1632                  * mechanism.  Works from within PCI passthrough environments
1633                  * too, where rman_get_start() can return a different value.  We
1634                  * need to program the T4 memory window decoders with the actual
1635                  * addresses that will be coming across the PCIe link.
1636                  */
1637                 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1638                 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1639
1640                 mw = &t4_memwin[0];
1641                 n = nitems(t4_memwin);
1642         } else {
1643                 /* T5 uses the relative offset inside the PCIe BAR */
1644                 bar0 = 0;
1645
1646                 mw = &t5_memwin[0];
1647                 n = nitems(t5_memwin);
1648         }
1649
1650         for (i = 0; i < n; i++, mw++) {
1651                 t4_write_reg(sc,
1652                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
1653                     (mw->base + bar0) | V_BIR(0) |
1654                     V_WINDOW(ilog2(mw->aperture) - 10));
1655         }
1656
1657         /* flush */
1658         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1659 }
1660
1661 /*
1662  * Verify that the memory range specified by the addr/len pair is valid and lies
1663  * entirely within a single region (EDCx or MCx).
1664  */
1665 static int
1666 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
1667 {
1668         uint32_t em, addr_len, maddr, mlen;
1669
1670         /* Memory can only be accessed in naturally aligned 4 byte units */
1671         if (addr & 3 || len & 3 || len == 0)
1672                 return (EINVAL);
1673
1674         /* Enabled memories */
1675         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1676         if (em & F_EDRAM0_ENABLE) {
1677                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1678                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1679                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1680                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1681                     addr + len <= maddr + mlen)
1682                         return (0);
1683         }
1684         if (em & F_EDRAM1_ENABLE) {
1685                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1686                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1687                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1688                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1689                     addr + len <= maddr + mlen)
1690                         return (0);
1691         }
1692         if (em & F_EXT_MEM_ENABLE) {
1693                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1694                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1695                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1696                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1697                     addr + len <= maddr + mlen)
1698                         return (0);
1699         }
1700         if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
1701                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1702                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1703                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1704                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1705                     addr + len <= maddr + mlen)
1706                         return (0);
1707         }
1708
1709         return (EFAULT);
1710 }
1711
1712 static int
1713 fwmtype_to_hwmtype(int mtype)
1714 {
1715
1716         switch (mtype) {
1717         case FW_MEMTYPE_EDC0:
1718                 return (MEM_EDC0);
1719         case FW_MEMTYPE_EDC1:
1720                 return (MEM_EDC1);
1721         case FW_MEMTYPE_EXTMEM:
1722                 return (MEM_MC0);
1723         case FW_MEMTYPE_EXTMEM1:
1724                 return (MEM_MC1);
1725         default:
1726                 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
1727         }
1728 }
1729
1730 /*
1731  * Verify that the memory range specified by the memtype/offset/len pair is
1732  * valid and lies entirely within the memtype specified.  The global address of
1733  * the start of the range is returned in addr.
1734  */
1735 static int
1736 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1737     uint32_t *addr)
1738 {
1739         uint32_t em, addr_len, maddr, mlen;
1740
1741         /* Memory can only be accessed in naturally aligned 4 byte units */
1742         if (off & 3 || len & 3 || len == 0)
1743                 return (EINVAL);
1744
1745         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1746         switch (fwmtype_to_hwmtype(mtype)) {
1747         case MEM_EDC0:
1748                 if (!(em & F_EDRAM0_ENABLE))
1749                         return (EINVAL);
1750                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1751                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1752                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1753                 break;
1754         case MEM_EDC1:
1755                 if (!(em & F_EDRAM1_ENABLE))
1756                         return (EINVAL);
1757                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1758                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1759                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1760                 break;
1761         case MEM_MC:
1762                 if (!(em & F_EXT_MEM_ENABLE))
1763                         return (EINVAL);
1764                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1765                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1766                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1767                 break;
1768         case MEM_MC1:
1769                 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
1770                         return (EINVAL);
1771                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1772                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1773                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1774                 break;
1775         default:
1776                 return (EINVAL);
1777         }
1778
1779         if (mlen > 0 && off < mlen && off + len <= mlen) {
1780                 *addr = maddr + off;    /* global address */
1781                 return (0);
1782         }
1783
1784         return (EFAULT);
1785 }
1786
1787 static void
1788 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1789 {
1790         const struct memwin *mw;
1791
1792         if (is_t4(sc)) {
1793                 KASSERT(win >= 0 && win < nitems(t4_memwin),
1794                     ("%s: incorrect memwin# (%d)", __func__, win));
1795                 mw = &t4_memwin[win];
1796         } else {
1797                 KASSERT(win >= 0 && win < nitems(t5_memwin),
1798                     ("%s: incorrect memwin# (%d)", __func__, win));
1799                 mw = &t5_memwin[win];
1800         }
1801
1802         if (base != NULL)
1803                 *base = mw->base;
1804         if (aperture != NULL)
1805                 *aperture = mw->aperture;
1806 }
1807
1808 /*
1809  * Positions the memory window such that it can be used to access the specified
1810  * address in the chip's address space.  The return value is the offset of addr
1811  * from the start of the window.
1812  */
1813 static uint32_t
1814 position_memwin(struct adapter *sc, int n, uint32_t addr)
1815 {
1816         uint32_t start, pf;
1817         uint32_t reg;
1818
1819         KASSERT(n >= 0 && n <= 3,
1820             ("%s: invalid window %d.", __func__, n));
1821         KASSERT((addr & 3) == 0,
1822             ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
1823
1824         if (is_t4(sc)) {
1825                 pf = 0;
1826                 start = addr & ~0xf;    /* start must be 16B aligned */
1827         } else {
1828                 pf = V_PFNUM(sc->pf);
1829                 start = addr & ~0x7f;   /* start must be 128B aligned */
1830         }
1831         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1832
1833         t4_write_reg(sc, reg, start | pf);
1834         t4_read_reg(sc, reg);
1835
1836         return (addr - start);
1837 }
1838
1839 static int
1840 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1841     struct intrs_and_queues *iaq)
1842 {
1843         int rc, itype, navail, nrxq10g, nrxq1g, n;
1844         int nofldrxq10g = 0, nofldrxq1g = 0;
1845         int nnmrxq10g = 0, nnmrxq1g = 0;
1846
1847         bzero(iaq, sizeof(*iaq));
1848
1849         iaq->ntxq10g = t4_ntxq10g;
1850         iaq->ntxq1g = t4_ntxq1g;
1851         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1852         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1853         iaq->rsrv_noflowq = t4_rsrv_noflowq;
1854 #ifdef TCP_OFFLOAD
1855         if (is_offload(sc)) {
1856                 iaq->nofldtxq10g = t4_nofldtxq10g;
1857                 iaq->nofldtxq1g = t4_nofldtxq1g;
1858                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1859                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1860         }
1861 #endif
1862 #ifdef DEV_NETMAP
1863         iaq->nnmtxq10g = t4_nnmtxq10g;
1864         iaq->nnmtxq1g = t4_nnmtxq1g;
1865         iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g;
1866         iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g;
1867 #endif
1868
1869         for (itype = INTR_MSIX; itype; itype >>= 1) {
1870
1871                 if ((itype & t4_intr_types) == 0)
1872                         continue;       /* not allowed */
1873
1874                 if (itype == INTR_MSIX)
1875                         navail = pci_msix_count(sc->dev);
1876                 else if (itype == INTR_MSI)
1877                         navail = pci_msi_count(sc->dev);
1878                 else
1879                         navail = 1;
1880 restart:
1881                 if (navail == 0)
1882                         continue;
1883
1884                 iaq->intr_type = itype;
1885                 iaq->intr_flags_10g = 0;
1886                 iaq->intr_flags_1g = 0;
1887
1888                 /*
1889                  * Best option: an interrupt vector for errors, one for the
1890                  * firmware event queue, and one for every rxq (NIC, TOE, and
1891                  * netmap).
1892                  */
1893                 iaq->nirq = T4_EXTRA_INTR;
1894                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g);
1895                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g);
1896                 if (iaq->nirq <= navail &&
1897                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
1898                         iaq->intr_flags_10g = INTR_ALL;
1899                         iaq->intr_flags_1g = INTR_ALL;
1900                         goto allocate;
1901                 }
1902
1903                 /*
1904                  * Second best option: a vector for errors, one for the firmware
1905                  * event queue, and vectors for either all the NIC rx queues or
1906                  * all the TOE rx queues.  The queues that don't get vectors
1907                  * will forward their interrupts to those that do.
1908                  *
1909                  * Note: netmap rx queues cannot be created early and so they
1910                  * can't be setup to receive forwarded interrupts for others.
1911                  */
1912                 iaq->nirq = T4_EXTRA_INTR;
1913                 if (nrxq10g >= nofldrxq10g) {
1914                         iaq->intr_flags_10g = INTR_RXQ;
1915                         iaq->nirq += n10g * nrxq10g;
1916 #ifdef DEV_NETMAP
1917                         iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g);
1918 #endif
1919                 } else {
1920                         iaq->intr_flags_10g = INTR_OFLD_RXQ;
1921                         iaq->nirq += n10g * nofldrxq10g;
1922 #ifdef DEV_NETMAP
1923                         iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g);
1924 #endif
1925                 }
1926                 if (nrxq1g >= nofldrxq1g) {
1927                         iaq->intr_flags_1g = INTR_RXQ;
1928                         iaq->nirq += n1g * nrxq1g;
1929 #ifdef DEV_NETMAP
1930                         iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g);
1931 #endif
1932                 } else {
1933                         iaq->intr_flags_1g = INTR_OFLD_RXQ;
1934                         iaq->nirq += n1g * nofldrxq1g;
1935 #ifdef DEV_NETMAP
1936                         iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g);
1937 #endif
1938                 }
1939                 if (iaq->nirq <= navail &&
1940                     (itype != INTR_MSI || powerof2(iaq->nirq)))
1941                         goto allocate;
1942
1943                 /*
1944                  * Next best option: an interrupt vector for errors, one for the
1945                  * firmware event queue, and at least one per port.  At this
1946                  * point we know we'll have to downsize nrxq and/or nofldrxq
1947                  * and/or nnmrxq to fit what's available to us.
1948                  */
1949                 iaq->nirq = T4_EXTRA_INTR;
1950                 iaq->nirq += n10g + n1g;
1951                 if (iaq->nirq <= navail) {
1952                         int leftover = navail - iaq->nirq;
1953
1954                         if (n10g > 0) {
1955                                 int target = max(nrxq10g, nofldrxq10g);
1956
1957                                 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
1958                                     INTR_RXQ : INTR_OFLD_RXQ;
1959
1960                                 n = 1;
1961                                 while (n < target && leftover >= n10g) {
1962                                         leftover -= n10g;
1963                                         iaq->nirq += n10g;
1964                                         n++;
1965                                 }
1966                                 iaq->nrxq10g = min(n, nrxq10g);
1967 #ifdef TCP_OFFLOAD
1968                                 iaq->nofldrxq10g = min(n, nofldrxq10g);
1969 #endif
1970 #ifdef DEV_NETMAP
1971                                 iaq->nnmrxq10g = min(n, nnmrxq10g);
1972 #endif
1973                         }
1974
1975                         if (n1g > 0) {
1976                                 int target = max(nrxq1g, nofldrxq1g);
1977
1978                                 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
1979                                     INTR_RXQ : INTR_OFLD_RXQ;
1980
1981                                 n = 1;
1982                                 while (n < target && leftover >= n1g) {
1983                                         leftover -= n1g;
1984                                         iaq->nirq += n1g;
1985                                         n++;
1986                                 }
1987                                 iaq->nrxq1g = min(n, nrxq1g);
1988 #ifdef TCP_OFFLOAD
1989                                 iaq->nofldrxq1g = min(n, nofldrxq1g);
1990 #endif
1991 #ifdef DEV_NETMAP
1992                                 iaq->nnmrxq1g = min(n, nnmrxq1g);
1993 #endif
1994                         }
1995
1996                         if (itype != INTR_MSI || powerof2(iaq->nirq))
1997                                 goto allocate;
1998                 }
1999
2000                 /*
2001                  * Least desirable option: one interrupt vector for everything.
2002                  */
2003                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
2004                 iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
2005 #ifdef TCP_OFFLOAD
2006                 if (is_offload(sc))
2007                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2008 #endif
2009 #ifdef DEV_NETMAP
2010                 iaq->nnmrxq10g = iaq->nnmrxq1g = 1;
2011 #endif
2012
2013 allocate:
2014                 navail = iaq->nirq;
2015                 rc = 0;
2016                 if (itype == INTR_MSIX)
2017                         rc = pci_alloc_msix(sc->dev, &navail);
2018                 else if (itype == INTR_MSI)
2019                         rc = pci_alloc_msi(sc->dev, &navail);
2020
2021                 if (rc == 0) {
2022                         if (navail == iaq->nirq)
2023                                 return (0);
2024
2025                         /*
2026                          * Didn't get the number requested.  Use whatever number
2027                          * the kernel is willing to allocate (it's in navail).
2028                          */
2029                         device_printf(sc->dev, "fewer vectors than requested, "
2030                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
2031                             itype, iaq->nirq, navail);
2032                         pci_release_msi(sc->dev);
2033                         goto restart;
2034                 }
2035
2036                 device_printf(sc->dev,
2037                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
2038                     itype, rc, iaq->nirq, navail);
2039         }
2040
2041         device_printf(sc->dev,
2042             "failed to find a usable interrupt type.  "
2043             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
2044             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
2045
2046         return (ENXIO);
2047 }
2048
2049 #define FW_VERSION(chip) ( \
2050     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
2051     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
2052     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
2053     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
2054 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
2055
2056 struct fw_info {
2057         uint8_t chip;
2058         char *kld_name;
2059         char *fw_mod_name;
2060         struct fw_hdr fw_hdr;   /* XXX: waste of space, need a sparse struct */
2061 } fw_info[] = {
2062         {
2063                 .chip = CHELSIO_T4,
2064                 .kld_name = "t4fw_cfg",
2065                 .fw_mod_name = "t4fw",
2066                 .fw_hdr = {
2067                         .chip = FW_HDR_CHIP_T4,
2068                         .fw_ver = htobe32_const(FW_VERSION(T4)),
2069                         .intfver_nic = FW_INTFVER(T4, NIC),
2070                         .intfver_vnic = FW_INTFVER(T4, VNIC),
2071                         .intfver_ofld = FW_INTFVER(T4, OFLD),
2072                         .intfver_ri = FW_INTFVER(T4, RI),
2073                         .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
2074                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
2075                         .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
2076                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
2077                 },
2078         }, {
2079                 .chip = CHELSIO_T5,
2080                 .kld_name = "t5fw_cfg",
2081                 .fw_mod_name = "t5fw",
2082                 .fw_hdr = {
2083                         .chip = FW_HDR_CHIP_T5,
2084                         .fw_ver = htobe32_const(FW_VERSION(T5)),
2085                         .intfver_nic = FW_INTFVER(T5, NIC),
2086                         .intfver_vnic = FW_INTFVER(T5, VNIC),
2087                         .intfver_ofld = FW_INTFVER(T5, OFLD),
2088                         .intfver_ri = FW_INTFVER(T5, RI),
2089                         .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
2090                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
2091                         .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
2092                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
2093                 },
2094         }
2095 };
2096
2097 static struct fw_info *
2098 find_fw_info(int chip)
2099 {
2100         int i;
2101
2102         for (i = 0; i < nitems(fw_info); i++) {
2103                 if (fw_info[i].chip == chip)
2104                         return (&fw_info[i]);
2105         }
2106         return (NULL);
2107 }
2108
2109 /*
2110  * Is the given firmware API compatible with the one the driver was compiled
2111  * with?
2112  */
2113 static int
2114 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
2115 {
2116
2117         /* short circuit if it's the exact same firmware version */
2118         if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
2119                 return (1);
2120
2121         /*
2122          * XXX: Is this too conservative?  Perhaps I should limit this to the
2123          * features that are supported in the driver.
2124          */
2125 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
2126         if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
2127             SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
2128             SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
2129                 return (1);
2130 #undef SAME_INTF
2131
2132         return (0);
2133 }
2134
2135 /*
2136  * The firmware in the KLD is usable, but should it be installed?  This routine
2137  * explains itself in detail if it indicates the KLD firmware should be
2138  * installed.
2139  */
2140 static int
2141 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
2142 {
2143         const char *reason;
2144
2145         if (!card_fw_usable) {
2146                 reason = "incompatible or unusable";
2147                 goto install;
2148         }
2149
2150         if (k > c) {
2151                 reason = "older than the version bundled with this driver";
2152                 goto install;
2153         }
2154
2155         if (t4_fw_install == 2 && k != c) {
2156                 reason = "different than the version bundled with this driver";
2157                 goto install;
2158         }
2159
2160         return (0);
2161
2162 install:
2163         if (t4_fw_install == 0) {
2164                 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2165                     "but the driver is prohibited from installing a different "
2166                     "firmware on the card.\n",
2167                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2168                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2169
2170                 return (0);
2171         }
2172
2173         device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2174             "installing firmware %u.%u.%u.%u on card.\n",
2175             G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2176             G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2177             G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2178             G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2179
2180         return (1);
2181 }
2182 /*
2183  * Establish contact with the firmware and determine if we are the master driver
2184  * or not, and whether we are responsible for chip initialization.
2185  */
2186 static int
2187 prep_firmware(struct adapter *sc)
2188 {
2189         const struct firmware *fw = NULL, *default_cfg;
2190         int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2191         enum dev_state state;
2192         struct fw_info *fw_info;
2193         struct fw_hdr *card_fw;         /* fw on the card */
2194         const struct fw_hdr *kld_fw;    /* fw in the KLD */
2195         const struct fw_hdr *drv_fw;    /* fw header the driver was compiled
2196                                            against */
2197
2198         /* Contact firmware. */
2199         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2200         if (rc < 0 || state == DEV_STATE_ERR) {
2201                 rc = -rc;
2202                 device_printf(sc->dev,
2203                     "failed to connect to the firmware: %d, %d.\n", rc, state);
2204                 return (rc);
2205         }
2206         pf = rc;
2207         if (pf == sc->mbox)
2208                 sc->flags |= MASTER_PF;
2209         else if (state == DEV_STATE_UNINIT) {
2210                 /*
2211                  * We didn't get to be the master so we definitely won't be
2212                  * configuring the chip.  It's a bug if someone else hasn't
2213                  * configured it already.
2214                  */
2215                 device_printf(sc->dev, "couldn't be master(%d), "
2216                     "device not already initialized either(%d).\n", rc, state);
2217                 return (EDOOFUS);
2218         }
2219
2220         /* This is the firmware whose headers the driver was compiled against */
2221         fw_info = find_fw_info(chip_id(sc));
2222         if (fw_info == NULL) {
2223                 device_printf(sc->dev,
2224                     "unable to look up firmware information for chip %d.\n",
2225                     chip_id(sc));
2226                 return (EINVAL);
2227         }
2228         drv_fw = &fw_info->fw_hdr;
2229
2230         /*
2231          * The firmware KLD contains many modules.  The KLD name is also the
2232          * name of the module that contains the default config file.
2233          */
2234         default_cfg = firmware_get(fw_info->kld_name);
2235
2236         /* Read the header of the firmware on the card */
2237         card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2238         rc = -t4_read_flash(sc, FLASH_FW_START,
2239             sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2240         if (rc == 0)
2241                 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2242         else {
2243                 device_printf(sc->dev,
2244                     "Unable to read card's firmware header: %d\n", rc);
2245                 card_fw_usable = 0;
2246         }
2247
2248         /* This is the firmware in the KLD */
2249         fw = firmware_get(fw_info->fw_mod_name);
2250         if (fw != NULL) {
2251                 kld_fw = (const void *)fw->data;
2252                 kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2253         } else {
2254                 kld_fw = NULL;
2255                 kld_fw_usable = 0;
2256         }
2257
2258         if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2259             (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2260                 /*
2261                  * Common case: the firmware on the card is an exact match and
2262                  * the KLD is an exact match too, or the KLD is
2263                  * absent/incompatible.  Note that t4_fw_install = 2 is ignored
2264                  * here -- use cxgbetool loadfw if you want to reinstall the
2265                  * same firmware as the one on the card.
2266                  */
2267         } else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2268             should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2269             be32toh(card_fw->fw_ver))) {
2270
2271                 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
2272                 if (rc != 0) {
2273                         device_printf(sc->dev,
2274                             "failed to install firmware: %d\n", rc);
2275                         goto done;
2276                 }
2277
2278                 /* Installed successfully, update the cached header too. */
2279                 memcpy(card_fw, kld_fw, sizeof(*card_fw));
2280                 card_fw_usable = 1;
2281                 need_fw_reset = 0;      /* already reset as part of load_fw */
2282         }
2283
2284         if (!card_fw_usable) {
2285                 uint32_t d, c, k;
2286
2287                 d = ntohl(drv_fw->fw_ver);
2288                 c = ntohl(card_fw->fw_ver);
2289                 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
2290
2291                 device_printf(sc->dev, "Cannot find a usable firmware: "
2292                     "fw_install %d, chip state %d, "
2293                     "driver compiled with %d.%d.%d.%d, "
2294                     "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
2295                     t4_fw_install, state,
2296                     G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
2297                     G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
2298                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2299                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
2300                     G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2301                     G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2302                 rc = EINVAL;
2303                 goto done;
2304         }
2305
2306         /* We're using whatever's on the card and it's known to be good. */
2307         sc->params.fw_vers = ntohl(card_fw->fw_ver);
2308         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
2309             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2310             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2311             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2312             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2313         t4_get_tp_version(sc, &sc->params.tp_vers);
2314
2315         /* Reset device */
2316         if (need_fw_reset &&
2317             (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
2318                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
2319                 if (rc != ETIMEDOUT && rc != EIO)
2320                         t4_fw_bye(sc, sc->mbox);
2321                 goto done;
2322         }
2323         sc->flags |= FW_OK;
2324
2325         rc = get_params__pre_init(sc);
2326         if (rc != 0)
2327                 goto done; /* error message displayed already */
2328
2329         /* Partition adapter resources as specified in the config file. */
2330         if (state == DEV_STATE_UNINIT) {
2331
2332                 KASSERT(sc->flags & MASTER_PF,
2333                     ("%s: trying to change chip settings when not master.",
2334                     __func__));
2335
2336                 rc = partition_resources(sc, default_cfg, fw_info->kld_name);
2337                 if (rc != 0)
2338                         goto done;      /* error message displayed already */
2339
2340                 t4_tweak_chip_settings(sc);
2341
2342                 /* get basic stuff going */
2343                 rc = -t4_fw_initialize(sc, sc->mbox);
2344                 if (rc != 0) {
2345                         device_printf(sc->dev, "fw init failed: %d.\n", rc);
2346                         goto done;
2347                 }
2348         } else {
2349                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
2350                 sc->cfcsum = 0;
2351         }
2352
2353 done:
2354         free(card_fw, M_CXGBE);
2355         if (fw != NULL)
2356                 firmware_put(fw, FIRMWARE_UNLOAD);
2357         if (default_cfg != NULL)
2358                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
2359
2360         return (rc);
2361 }
2362
2363 #define FW_PARAM_DEV(param) \
2364         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2365          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2366 #define FW_PARAM_PFVF(param) \
2367         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2368          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2369
2370 /*
2371  * Partition chip resources for use between various PFs, VFs, etc.
2372  */
2373 static int
2374 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
2375     const char *name_prefix)
2376 {
2377         const struct firmware *cfg = NULL;
2378         int rc = 0;
2379         struct fw_caps_config_cmd caps;
2380         uint32_t mtype, moff, finicsum, cfcsum;
2381
2382         /*
2383          * Figure out what configuration file to use.  Pick the default config
2384          * file for the card if the user hasn't specified one explicitly.
2385          */
2386         snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
2387         if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
2388                 /* Card specific overrides go here. */
2389                 if (pci_get_device(sc->dev) == 0x440a)
2390                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
2391                 if (is_fpga(sc))
2392                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
2393         }
2394
2395         /*
2396          * We need to load another module if the profile is anything except
2397          * "default" or "flash".
2398          */
2399         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
2400             strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2401                 char s[32];
2402
2403                 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
2404                 cfg = firmware_get(s);
2405                 if (cfg == NULL) {
2406                         if (default_cfg != NULL) {
2407                                 device_printf(sc->dev,
2408                                     "unable to load module \"%s\" for "
2409                                     "configuration profile \"%s\", will use "
2410                                     "the default config file instead.\n",
2411                                     s, sc->cfg_file);
2412                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2413                                     "%s", DEFAULT_CF);
2414                         } else {
2415                                 device_printf(sc->dev,
2416                                     "unable to load module \"%s\" for "
2417                                     "configuration profile \"%s\", will use "
2418                                     "the config file on the card's flash "
2419                                     "instead.\n", s, sc->cfg_file);
2420                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2421                                     "%s", FLASH_CF);
2422                         }
2423                 }
2424         }
2425
2426         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
2427             default_cfg == NULL) {
2428                 device_printf(sc->dev,
2429                     "default config file not available, will use the config "
2430                     "file on the card's flash instead.\n");
2431                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
2432         }
2433
2434         if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2435                 u_int cflen, i, n;
2436                 const uint32_t *cfdata;
2437                 uint32_t param, val, addr, off, mw_base, mw_aperture;
2438
2439                 KASSERT(cfg != NULL || default_cfg != NULL,
2440                     ("%s: no config to upload", __func__));
2441
2442                 /*
2443                  * Ask the firmware where it wants us to upload the config file.
2444                  */
2445                 param = FW_PARAM_DEV(CF);
2446                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2447                 if (rc != 0) {
2448                         /* No support for config file?  Shouldn't happen. */
2449                         device_printf(sc->dev,
2450                             "failed to query config file location: %d.\n", rc);
2451                         goto done;
2452                 }
2453                 mtype = G_FW_PARAMS_PARAM_Y(val);
2454                 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
2455
2456                 /*
2457                  * XXX: sheer laziness.  We deliberately added 4 bytes of
2458                  * useless stuffing/comments at the end of the config file so
2459                  * it's ok to simply throw away the last remaining bytes when
2460                  * the config file is not an exact multiple of 4.  This also
2461                  * helps with the validate_mt_off_len check.
2462                  */
2463                 if (cfg != NULL) {
2464                         cflen = cfg->datasize & ~3;
2465                         cfdata = cfg->data;
2466                 } else {
2467                         cflen = default_cfg->datasize & ~3;
2468                         cfdata = default_cfg->data;
2469                 }
2470
2471                 if (cflen > FLASH_CFG_MAX_SIZE) {
2472                         device_printf(sc->dev,
2473                             "config file too long (%d, max allowed is %d).  "
2474                             "Will try to use the config on the card, if any.\n",
2475                             cflen, FLASH_CFG_MAX_SIZE);
2476                         goto use_config_on_flash;
2477                 }
2478
2479                 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
2480                 if (rc != 0) {
2481                         device_printf(sc->dev,
2482                             "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
2483                             "Will try to use the config on the card, if any.\n",
2484                             __func__, mtype, moff, cflen, rc);
2485                         goto use_config_on_flash;
2486                 }
2487
2488                 memwin_info(sc, 2, &mw_base, &mw_aperture);
2489                 while (cflen) {
2490                         off = position_memwin(sc, 2, addr);
2491                         n = min(cflen, mw_aperture - off);
2492                         for (i = 0; i < n; i += 4)
2493                                 t4_write_reg(sc, mw_base + off + i, *cfdata++);
2494                         cflen -= n;
2495                         addr += n;
2496                 }
2497         } else {
2498 use_config_on_flash:
2499                 mtype = FW_MEMTYPE_FLASH;
2500                 moff = t4_flash_cfg_addr(sc);
2501         }
2502
2503         bzero(&caps, sizeof(caps));
2504         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2505             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2506         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
2507             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
2508             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
2509         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2510         if (rc != 0) {
2511                 device_printf(sc->dev,
2512                     "failed to pre-process config file: %d "
2513                     "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
2514                 goto done;
2515         }
2516
2517         finicsum = be32toh(caps.finicsum);
2518         cfcsum = be32toh(caps.cfcsum);
2519         if (finicsum != cfcsum) {
2520                 device_printf(sc->dev,
2521                     "WARNING: config file checksum mismatch: %08x %08x\n",
2522                     finicsum, cfcsum);
2523         }
2524         sc->cfcsum = cfcsum;
2525
2526 #define LIMIT_CAPS(x) do { \
2527         caps.x &= htobe16(t4_##x##_allowed); \
2528 } while (0)
2529
2530         /*
2531          * Let the firmware know what features will (not) be used so it can tune
2532          * things accordingly.
2533          */
2534         LIMIT_CAPS(linkcaps);
2535         LIMIT_CAPS(niccaps);
2536         LIMIT_CAPS(toecaps);
2537         LIMIT_CAPS(rdmacaps);
2538         LIMIT_CAPS(iscsicaps);
2539         LIMIT_CAPS(fcoecaps);
2540 #undef LIMIT_CAPS
2541
2542         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2543             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
2544         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2545         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
2546         if (rc != 0) {
2547                 device_printf(sc->dev,
2548                     "failed to process config file: %d.\n", rc);
2549         }
2550 done:
2551         if (cfg != NULL)
2552                 firmware_put(cfg, FIRMWARE_UNLOAD);
2553         return (rc);
2554 }
2555
2556 /*
2557  * Retrieve parameters that are needed (or nice to have) very early.
2558  */
2559 static int
2560 get_params__pre_init(struct adapter *sc)
2561 {
2562         int rc;
2563         uint32_t param[2], val[2];
2564         struct fw_devlog_cmd cmd;
2565         struct devlog_params *dlog = &sc->params.devlog;
2566
2567         param[0] = FW_PARAM_DEV(PORTVEC);
2568         param[1] = FW_PARAM_DEV(CCLK);
2569         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2570         if (rc != 0) {
2571                 device_printf(sc->dev,
2572                     "failed to query parameters (pre_init): %d.\n", rc);
2573                 return (rc);
2574         }
2575
2576         sc->params.portvec = val[0];
2577         sc->params.nports = bitcount32(val[0]);
2578         sc->params.vpd.cclk = val[1];
2579
2580         /* Read device log parameters. */
2581         bzero(&cmd, sizeof(cmd));
2582         cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
2583             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2584         cmd.retval_len16 = htobe32(FW_LEN16(cmd));
2585         rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
2586         if (rc != 0) {
2587                 device_printf(sc->dev,
2588                     "failed to get devlog parameters: %d.\n", rc);
2589                 bzero(dlog, sizeof (*dlog));
2590                 rc = 0; /* devlog isn't critical for device operation */
2591         } else {
2592                 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
2593                 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
2594                 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
2595                 dlog->size = be32toh(cmd.memsize_devlog);
2596         }
2597
2598         return (rc);
2599 }
2600
2601 /*
2602  * Retrieve various parameters that are of interest to the driver.  The device
2603  * has been initialized by the firmware at this point.
2604  */
2605 static int
2606 get_params__post_init(struct adapter *sc)
2607 {
2608         int rc;
2609         uint32_t param[7], val[7];
2610         struct fw_caps_config_cmd caps;
2611
2612         param[0] = FW_PARAM_PFVF(IQFLINT_START);
2613         param[1] = FW_PARAM_PFVF(EQ_START);
2614         param[2] = FW_PARAM_PFVF(FILTER_START);
2615         param[3] = FW_PARAM_PFVF(FILTER_END);
2616         param[4] = FW_PARAM_PFVF(L2T_START);
2617         param[5] = FW_PARAM_PFVF(L2T_END);
2618         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2619         if (rc != 0) {
2620                 device_printf(sc->dev,
2621                     "failed to query parameters (post_init): %d.\n", rc);
2622                 return (rc);
2623         }
2624
2625         sc->sge.iq_start = val[0];
2626         sc->sge.eq_start = val[1];
2627         sc->tids.ftid_base = val[2];
2628         sc->tids.nftids = val[3] - val[2] + 1;
2629         sc->params.ftid_min = val[2];
2630         sc->params.ftid_max = val[3];
2631         sc->vres.l2t.start = val[4];
2632         sc->vres.l2t.size = val[5] - val[4] + 1;
2633         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
2634             ("%s: L2 table size (%u) larger than expected (%u)",
2635             __func__, sc->vres.l2t.size, L2T_SIZE));
2636
2637         /* get capabilites */
2638         bzero(&caps, sizeof(caps));
2639         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2640             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2641         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2642         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2643         if (rc != 0) {
2644                 device_printf(sc->dev,
2645                     "failed to get card capabilities: %d.\n", rc);
2646                 return (rc);
2647         }
2648
2649 #define READ_CAPS(x) do { \
2650         sc->x = htobe16(caps.x); \
2651 } while (0)
2652         READ_CAPS(linkcaps);
2653         READ_CAPS(niccaps);
2654         READ_CAPS(toecaps);
2655         READ_CAPS(rdmacaps);
2656         READ_CAPS(iscsicaps);
2657         READ_CAPS(fcoecaps);
2658
2659         if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
2660                 param[0] = FW_PARAM_PFVF(ETHOFLD_START);
2661                 param[1] = FW_PARAM_PFVF(ETHOFLD_END);
2662                 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2663                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
2664                 if (rc != 0) {
2665                         device_printf(sc->dev,
2666                             "failed to query NIC parameters: %d.\n", rc);
2667                         return (rc);
2668                 }
2669                 sc->tids.etid_base = val[0];
2670                 sc->params.etid_min = val[0];
2671                 sc->tids.netids = val[1] - val[0] + 1;
2672                 sc->params.netids = sc->tids.netids;
2673                 sc->params.eo_wr_cred = val[2];
2674                 sc->params.ethoffload = 1;
2675         }
2676
2677         if (sc->toecaps) {
2678                 /* query offload-related parameters */
2679                 param[0] = FW_PARAM_DEV(NTID);
2680                 param[1] = FW_PARAM_PFVF(SERVER_START);
2681                 param[2] = FW_PARAM_PFVF(SERVER_END);
2682                 param[3] = FW_PARAM_PFVF(TDDP_START);
2683                 param[4] = FW_PARAM_PFVF(TDDP_END);
2684                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2685                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2686                 if (rc != 0) {
2687                         device_printf(sc->dev,
2688                             "failed to query TOE parameters: %d.\n", rc);
2689                         return (rc);
2690                 }
2691                 sc->tids.ntids = val[0];
2692                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
2693                 sc->tids.stid_base = val[1];
2694                 sc->tids.nstids = val[2] - val[1] + 1;
2695                 sc->vres.ddp.start = val[3];
2696                 sc->vres.ddp.size = val[4] - val[3] + 1;
2697                 sc->params.ofldq_wr_cred = val[5];
2698                 sc->params.offload = 1;
2699         }
2700         if (sc->rdmacaps) {
2701                 param[0] = FW_PARAM_PFVF(STAG_START);
2702                 param[1] = FW_PARAM_PFVF(STAG_END);
2703                 param[2] = FW_PARAM_PFVF(RQ_START);
2704                 param[3] = FW_PARAM_PFVF(RQ_END);
2705                 param[4] = FW_PARAM_PFVF(PBL_START);
2706                 param[5] = FW_PARAM_PFVF(PBL_END);
2707                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2708                 if (rc != 0) {
2709                         device_printf(sc->dev,
2710                             "failed to query RDMA parameters(1): %d.\n", rc);
2711                         return (rc);
2712                 }
2713                 sc->vres.stag.start = val[0];
2714                 sc->vres.stag.size = val[1] - val[0] + 1;
2715                 sc->vres.rq.start = val[2];
2716                 sc->vres.rq.size = val[3] - val[2] + 1;
2717                 sc->vres.pbl.start = val[4];
2718                 sc->vres.pbl.size = val[5] - val[4] + 1;
2719
2720                 param[0] = FW_PARAM_PFVF(SQRQ_START);
2721                 param[1] = FW_PARAM_PFVF(SQRQ_END);
2722                 param[2] = FW_PARAM_PFVF(CQ_START);
2723                 param[3] = FW_PARAM_PFVF(CQ_END);
2724                 param[4] = FW_PARAM_PFVF(OCQ_START);
2725                 param[5] = FW_PARAM_PFVF(OCQ_END);
2726                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2727                 if (rc != 0) {
2728                         device_printf(sc->dev,
2729                             "failed to query RDMA parameters(2): %d.\n", rc);
2730                         return (rc);
2731                 }
2732                 sc->vres.qp.start = val[0];
2733                 sc->vres.qp.size = val[1] - val[0] + 1;
2734                 sc->vres.cq.start = val[2];
2735                 sc->vres.cq.size = val[3] - val[2] + 1;
2736                 sc->vres.ocq.start = val[4];
2737                 sc->vres.ocq.size = val[5] - val[4] + 1;
2738         }
2739         if (sc->iscsicaps) {
2740                 param[0] = FW_PARAM_PFVF(ISCSI_START);
2741                 param[1] = FW_PARAM_PFVF(ISCSI_END);
2742                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2743                 if (rc != 0) {
2744                         device_printf(sc->dev,
2745                             "failed to query iSCSI parameters: %d.\n", rc);
2746                         return (rc);
2747                 }
2748                 sc->vres.iscsi.start = val[0];
2749                 sc->vres.iscsi.size = val[1] - val[0] + 1;
2750         }
2751
2752         /*
2753          * We've got the params we wanted to query via the firmware.  Now grab
2754          * some others directly from the chip.
2755          */
2756         rc = t4_read_chip_settings(sc);
2757
2758         return (rc);
2759 }
2760
2761 static int
2762 set_params__post_init(struct adapter *sc)
2763 {
2764         uint32_t param, val;
2765
2766         /* ask for encapsulated CPLs */
2767         param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
2768         val = 1;
2769         (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2770
2771         return (0);
2772 }
2773
2774 #undef FW_PARAM_PFVF
2775 #undef FW_PARAM_DEV
2776
2777 static void
2778 t4_set_desc(struct adapter *sc)
2779 {
2780         char buf[128];
2781         struct adapter_params *p = &sc->params;
2782
2783         snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
2784             "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
2785             chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
2786
2787         device_set_desc_copy(sc->dev, buf);
2788 }
2789
2790 static void
2791 build_medialist(struct port_info *pi, struct ifmedia *media)
2792 {
2793         int data, m;
2794
2795         PORT_LOCK(pi);
2796
2797         ifmedia_removeall(media);
2798
2799         m = IFM_ETHER | IFM_FDX;
2800         data = (pi->port_type << 8) | pi->mod_type;
2801
2802         switch(pi->port_type) {
2803         case FW_PORT_TYPE_BT_XFI:
2804                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2805                 break;
2806
2807         case FW_PORT_TYPE_BT_XAUI:
2808                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2809                 /* fall through */
2810
2811         case FW_PORT_TYPE_BT_SGMII:
2812                 ifmedia_add(media, m | IFM_1000_T, data, NULL);
2813                 ifmedia_add(media, m | IFM_100_TX, data, NULL);
2814                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL);
2815                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2816                 break;
2817
2818         case FW_PORT_TYPE_CX4:
2819                 ifmedia_add(media, m | IFM_10G_CX4, data, NULL);
2820                 ifmedia_set(media, m | IFM_10G_CX4);
2821                 break;
2822
2823         case FW_PORT_TYPE_QSFP_10G:
2824         case FW_PORT_TYPE_SFP:
2825         case FW_PORT_TYPE_FIBER_XFI:
2826         case FW_PORT_TYPE_FIBER_XAUI:
2827                 switch (pi->mod_type) {
2828
2829                 case FW_PORT_MOD_TYPE_LR:
2830                         ifmedia_add(media, m | IFM_10G_LR, data, NULL);
2831                         ifmedia_set(media, m | IFM_10G_LR);
2832                         break;
2833
2834                 case FW_PORT_MOD_TYPE_SR:
2835                         ifmedia_add(media, m | IFM_10G_SR, data, NULL);
2836                         ifmedia_set(media, m | IFM_10G_SR);
2837                         break;
2838
2839                 case FW_PORT_MOD_TYPE_LRM:
2840                         ifmedia_add(media, m | IFM_10G_LRM, data, NULL);
2841                         ifmedia_set(media, m | IFM_10G_LRM);
2842                         break;
2843
2844                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2845                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2846                         ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL);
2847                         ifmedia_set(media, m | IFM_10G_TWINAX);
2848                         break;
2849
2850                 case FW_PORT_MOD_TYPE_NONE:
2851                         m &= ~IFM_FDX;
2852                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2853                         ifmedia_set(media, m | IFM_NONE);
2854                         break;
2855
2856                 case FW_PORT_MOD_TYPE_NA:
2857                 case FW_PORT_MOD_TYPE_ER:
2858                 default:
2859                         device_printf(pi->dev,
2860                             "unknown port_type (%d), mod_type (%d)\n",
2861                             pi->port_type, pi->mod_type);
2862                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2863                         ifmedia_set(media, m | IFM_UNKNOWN);
2864                         break;
2865                 }
2866                 break;
2867
2868         case FW_PORT_TYPE_QSFP:
2869                 switch (pi->mod_type) {
2870
2871                 case FW_PORT_MOD_TYPE_LR:
2872                         ifmedia_add(media, m | IFM_40G_LR4, data, NULL);
2873                         ifmedia_set(media, m | IFM_40G_LR4);
2874                         break;
2875
2876                 case FW_PORT_MOD_TYPE_SR:
2877                         ifmedia_add(media, m | IFM_40G_SR4, data, NULL);
2878                         ifmedia_set(media, m | IFM_40G_SR4);
2879                         break;
2880
2881                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2882                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2883                         ifmedia_add(media, m | IFM_40G_CR4, data, NULL);
2884                         ifmedia_set(media, m | IFM_40G_CR4);
2885                         break;
2886
2887                 case FW_PORT_MOD_TYPE_NONE:
2888                         m &= ~IFM_FDX;
2889                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2890                         ifmedia_set(media, m | IFM_NONE);
2891                         break;
2892
2893                 default:
2894                         device_printf(pi->dev,
2895                             "unknown port_type (%d), mod_type (%d)\n",
2896                             pi->port_type, pi->mod_type);
2897                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2898                         ifmedia_set(media, m | IFM_UNKNOWN);
2899                         break;
2900                 }
2901                 break;
2902
2903         default:
2904                 device_printf(pi->dev,
2905                     "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
2906                     pi->mod_type);
2907                 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2908                 ifmedia_set(media, m | IFM_UNKNOWN);
2909                 break;
2910         }
2911
2912         PORT_UNLOCK(pi);
2913 }
2914
2915 #define FW_MAC_EXACT_CHUNK      7
2916
2917 /*
2918  * Program the port's XGMAC based on parameters in ifnet.  The caller also
2919  * indicates which parameters should be programmed (the rest are left alone).
2920  */
2921 int
2922 update_mac_settings(struct ifnet *ifp, int flags)
2923 {
2924         int rc = 0;
2925         struct port_info *pi = ifp->if_softc;
2926         struct adapter *sc = pi->adapter;
2927         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
2928         uint16_t viid = 0xffff;
2929         int16_t *xact_addr_filt = NULL;
2930
2931         ASSERT_SYNCHRONIZED_OP(sc);
2932         KASSERT(flags, ("%s: not told what to update.", __func__));
2933
2934         if (ifp == pi->ifp) {
2935                 viid = pi->viid;
2936                 xact_addr_filt = &pi->xact_addr_filt;
2937         }
2938 #ifdef DEV_NETMAP
2939         else if (ifp == pi->nm_ifp) {
2940                 viid = pi->nm_viid;
2941                 xact_addr_filt = &pi->nm_xact_addr_filt;
2942         }
2943 #endif
2944         if (flags & XGMAC_MTU)
2945                 mtu = ifp->if_mtu;
2946
2947         if (flags & XGMAC_PROMISC)
2948                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
2949
2950         if (flags & XGMAC_ALLMULTI)
2951                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
2952
2953         if (flags & XGMAC_VLANEX)
2954                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
2955
2956         if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
2957                 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti,
2958                     1, vlanex, false);
2959                 if (rc) {
2960                         if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
2961                             rc);
2962                         return (rc);
2963                 }
2964         }
2965
2966         if (flags & XGMAC_UCADDR) {
2967                 uint8_t ucaddr[ETHER_ADDR_LEN];
2968
2969                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
2970                 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr,
2971                     true, true);
2972                 if (rc < 0) {
2973                         rc = -rc;
2974                         if_printf(ifp, "change_mac failed: %d\n", rc);
2975                         return (rc);
2976                 } else {
2977                         *xact_addr_filt = rc;
2978                         rc = 0;
2979                 }
2980         }
2981
2982         if (flags & XGMAC_MCADDRS) {
2983                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
2984                 int del = 1;
2985                 uint64_t hash = 0;
2986                 struct ifmultiaddr *ifma;
2987                 int i = 0, j;
2988
2989                 if_maddr_rlock(ifp);
2990                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2991                         if (ifma->ifma_addr->sa_family != AF_LINK)
2992                                 continue;
2993                         mcaddr[i++] =
2994                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
2995
2996                         if (i == FW_MAC_EXACT_CHUNK) {
2997                                 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del,
2998                                     i, mcaddr, NULL, &hash, 0);
2999                                 if (rc < 0) {
3000                                         rc = -rc;
3001                                         for (j = 0; j < i; j++) {
3002                                                 if_printf(ifp,
3003                                                     "failed to add mc address"
3004                                                     " %02x:%02x:%02x:"
3005                                                     "%02x:%02x:%02x rc=%d\n",
3006                                                     mcaddr[j][0], mcaddr[j][1],
3007                                                     mcaddr[j][2], mcaddr[j][3],
3008                                                     mcaddr[j][4], mcaddr[j][5],
3009                                                     rc);
3010                                         }
3011                                         goto mcfail;
3012                                 }
3013                                 del = 0;
3014                                 i = 0;
3015                         }
3016                 }
3017                 if (i > 0) {
3018                         rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i,
3019                             mcaddr, NULL, &hash, 0);
3020                         if (rc < 0) {
3021                                 rc = -rc;
3022                                 for (j = 0; j < i; j++) {
3023                                         if_printf(ifp,
3024                                             "failed to add mc address"
3025                                             " %02x:%02x:%02x:"
3026                                             "%02x:%02x:%02x rc=%d\n",
3027                                             mcaddr[j][0], mcaddr[j][1],
3028                                             mcaddr[j][2], mcaddr[j][3],
3029                                             mcaddr[j][4], mcaddr[j][5],
3030                                             rc);
3031                                 }
3032                                 goto mcfail;
3033                         }
3034                 }
3035
3036                 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0);
3037                 if (rc != 0)
3038                         if_printf(ifp, "failed to set mc address hash: %d", rc);
3039 mcfail:
3040                 if_maddr_runlock(ifp);
3041         }
3042
3043         return (rc);
3044 }
3045
3046 int
3047 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
3048     char *wmesg)
3049 {
3050         int rc, pri;
3051
3052 #ifdef WITNESS
3053         /* the caller thinks it's ok to sleep, but is it really? */
3054         if (flags & SLEEP_OK)
3055                 pause("t4slptst", 1);
3056 #endif
3057
3058         if (INTR_OK)
3059                 pri = PCATCH;
3060         else
3061                 pri = 0;
3062
3063         ADAPTER_LOCK(sc);
3064         for (;;) {
3065
3066                 if (pi && IS_DOOMED(pi)) {
3067                         rc = ENXIO;
3068                         goto done;
3069                 }
3070
3071                 if (!IS_BUSY(sc)) {
3072                         rc = 0;
3073                         break;
3074                 }
3075
3076                 if (!(flags & SLEEP_OK)) {
3077                         rc = EBUSY;
3078                         goto done;
3079                 }
3080
3081                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
3082                         rc = EINTR;
3083                         goto done;
3084                 }
3085         }
3086
3087         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
3088         SET_BUSY(sc);
3089 #ifdef INVARIANTS
3090         sc->last_op = wmesg;
3091         sc->last_op_thr = curthread;
3092 #endif
3093
3094 done:
3095         if (!(flags & HOLD_LOCK) || rc)
3096                 ADAPTER_UNLOCK(sc);
3097
3098         return (rc);
3099 }
3100
3101 void
3102 end_synchronized_op(struct adapter *sc, int flags)
3103 {
3104
3105         if (flags & LOCK_HELD)
3106                 ADAPTER_LOCK_ASSERT_OWNED(sc);
3107         else
3108                 ADAPTER_LOCK(sc);
3109
3110         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
3111         CLR_BUSY(sc);
3112         wakeup(&sc->flags);
3113         ADAPTER_UNLOCK(sc);
3114 }
3115
3116 static int
3117 cxgbe_init_synchronized(struct port_info *pi)
3118 {
3119         struct adapter *sc = pi->adapter;
3120         struct ifnet *ifp = pi->ifp;
3121         int rc = 0;
3122
3123         ASSERT_SYNCHRONIZED_OP(sc);
3124
3125         if (isset(&sc->open_device_map, pi->port_id)) {
3126                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
3127                     ("mismatch between open_device_map and if_drv_flags"));
3128                 return (0);     /* already running */
3129         }
3130
3131         if (!(sc->flags & FULL_INIT_DONE) &&
3132             ((rc = adapter_full_init(sc)) != 0))
3133                 return (rc);    /* error message displayed already */
3134
3135         if (!(pi->flags & PORT_INIT_DONE) &&
3136             ((rc = port_full_init(pi)) != 0))
3137                 return (rc); /* error message displayed already */
3138
3139         rc = update_mac_settings(ifp, XGMAC_ALL);
3140         if (rc)
3141                 goto done;      /* error message displayed already */
3142
3143         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
3144         if (rc != 0) {
3145                 if_printf(ifp, "enable_vi failed: %d\n", rc);
3146                 goto done;
3147         }
3148
3149         /*
3150          * The first iq of the first port to come up is used for tracing.
3151          */
3152         if (sc->traceq < 0) {
3153                 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
3154                 t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
3155                     A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
3156                     V_QUEUENUMBER(sc->traceq));
3157                 pi->flags |= HAS_TRACEQ;
3158         }
3159
3160         /* all ok */
3161         setbit(&sc->open_device_map, pi->port_id);
3162         PORT_LOCK(pi);
3163         ifp->if_drv_flags |= IFF_DRV_RUNNING;
3164         PORT_UNLOCK(pi);
3165
3166         callout_reset(&pi->tick, hz, cxgbe_tick, pi);
3167 done:
3168         if (rc != 0)
3169                 cxgbe_uninit_synchronized(pi);
3170
3171         return (rc);
3172 }
3173
3174 /*
3175  * Idempotent.
3176  */
3177 static int
3178 cxgbe_uninit_synchronized(struct port_info *pi)
3179 {
3180         struct adapter *sc = pi->adapter;
3181         struct ifnet *ifp = pi->ifp;
3182         int rc;
3183
3184         ASSERT_SYNCHRONIZED_OP(sc);
3185
3186         /*
3187          * Disable the VI so that all its data in either direction is discarded
3188          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
3189          * tick) intact as the TP can deliver negative advice or data that it's
3190          * holding in its RAM (for an offloaded connection) even after the VI is
3191          * disabled.
3192          */
3193         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
3194         if (rc) {
3195                 if_printf(ifp, "disable_vi failed: %d\n", rc);
3196                 return (rc);
3197         }
3198
3199         clrbit(&sc->open_device_map, pi->port_id);
3200         PORT_LOCK(pi);
3201         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3202         PORT_UNLOCK(pi);
3203
3204         pi->link_cfg.link_ok = 0;
3205         pi->link_cfg.speed = 0;
3206         pi->linkdnrc = -1;
3207         t4_os_link_changed(sc, pi->port_id, 0, -1);
3208
3209         return (0);
3210 }
3211
3212 /*
3213  * It is ok for this function to fail midway and return right away.  t4_detach
3214  * will walk the entire sc->irq list and clean up whatever is valid.
3215  */
3216 static int
3217 setup_intr_handlers(struct adapter *sc)
3218 {
3219         int rc, rid, p, q;
3220         char s[8];
3221         struct irq *irq;
3222         struct port_info *pi;
3223         struct sge_rxq *rxq;
3224 #ifdef TCP_OFFLOAD
3225         struct sge_ofld_rxq *ofld_rxq;
3226 #endif
3227 #ifdef DEV_NETMAP
3228         struct sge_nm_rxq *nm_rxq;
3229 #endif
3230
3231         /*
3232          * Setup interrupts.
3233          */
3234         irq = &sc->irq[0];
3235         rid = sc->intr_type == INTR_INTX ? 0 : 1;
3236         if (sc->intr_count == 1)
3237                 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
3238
3239         /* Multiple interrupts. */
3240         KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
3241             ("%s: too few intr.", __func__));
3242
3243         /* The first one is always error intr */
3244         rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
3245         if (rc != 0)
3246                 return (rc);
3247         irq++;
3248         rid++;
3249
3250         /* The second one is always the firmware event queue */
3251         rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
3252         if (rc != 0)
3253                 return (rc);
3254         irq++;
3255         rid++;
3256
3257         for_each_port(sc, p) {
3258                 pi = sc->port[p];
3259
3260                 if (pi->flags & INTR_RXQ) {
3261                         for_each_rxq(pi, q, rxq) {
3262                                 snprintf(s, sizeof(s), "%d.%d", p, q);
3263                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
3264                                     s);
3265                                 if (rc != 0)
3266                                         return (rc);
3267                                 irq++;
3268                                 rid++;
3269                         }
3270                 }
3271 #ifdef TCP_OFFLOAD
3272                 if (pi->flags & INTR_OFLD_RXQ) {
3273                         for_each_ofld_rxq(pi, q, ofld_rxq) {
3274                                 snprintf(s, sizeof(s), "%d,%d", p, q);
3275                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr,
3276                                     ofld_rxq, s);
3277                                 if (rc != 0)
3278                                         return (rc);
3279                                 irq++;
3280                                 rid++;
3281                         }
3282                 }
3283 #endif
3284 #ifdef DEV_NETMAP
3285                 if (pi->flags & INTR_NM_RXQ) {
3286                         for_each_nm_rxq(pi, q, nm_rxq) {
3287                                 snprintf(s, sizeof(s), "%d-%d", p, q);
3288                                 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr,
3289                                     nm_rxq, s);
3290                                 if (rc != 0)
3291                                         return (rc);
3292                                 irq++;
3293                                 rid++;
3294                         }
3295                 }
3296 #endif
3297         }
3298         MPASS(irq == &sc->irq[sc->intr_count]);
3299
3300         return (0);
3301 }
3302
3303 int
3304 adapter_full_init(struct adapter *sc)
3305 {
3306         int rc, i;
3307
3308         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3309         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
3310             ("%s: FULL_INIT_DONE already", __func__));
3311
3312         /*
3313          * queues that belong to the adapter (not any particular port).
3314          */
3315         rc = t4_setup_adapter_queues(sc);
3316         if (rc != 0)
3317                 goto done;
3318
3319         for (i = 0; i < nitems(sc->tq); i++) {
3320                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
3321                     taskqueue_thread_enqueue, &sc->tq[i]);
3322                 if (sc->tq[i] == NULL) {
3323                         device_printf(sc->dev,
3324                             "failed to allocate task queue %d\n", i);
3325                         rc = ENOMEM;
3326                         goto done;
3327                 }
3328                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
3329                     device_get_nameunit(sc->dev), i);
3330         }
3331
3332         t4_intr_enable(sc);
3333         sc->flags |= FULL_INIT_DONE;
3334 done:
3335         if (rc != 0)
3336                 adapter_full_uninit(sc);
3337
3338         return (rc);
3339 }
3340
3341 int
3342 adapter_full_uninit(struct adapter *sc)
3343 {
3344         int i;
3345
3346         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3347
3348         t4_teardown_adapter_queues(sc);
3349
3350         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
3351                 taskqueue_free(sc->tq[i]);
3352                 sc->tq[i] = NULL;
3353         }
3354
3355         sc->flags &= ~FULL_INIT_DONE;
3356
3357         return (0);
3358 }
3359
3360 int
3361 port_full_init(struct port_info *pi)
3362 {
3363         struct adapter *sc = pi->adapter;
3364         struct ifnet *ifp = pi->ifp;
3365         uint16_t *rss;
3366         struct sge_rxq *rxq;
3367         int rc, i, j;
3368
3369         ASSERT_SYNCHRONIZED_OP(sc);
3370         KASSERT((pi->flags & PORT_INIT_DONE) == 0,
3371             ("%s: PORT_INIT_DONE already", __func__));
3372
3373         sysctl_ctx_init(&pi->ctx);
3374         pi->flags |= PORT_SYSCTL_CTX;
3375
3376         /*
3377          * Allocate tx/rx/fl queues for this port.
3378          */
3379         rc = t4_setup_port_queues(pi);
3380         if (rc != 0)
3381                 goto done;      /* error message displayed already */
3382
3383         /*
3384          * Setup RSS for this port.  Save a copy of the RSS table for later use.
3385          */
3386         rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
3387         for (i = 0; i < pi->rss_size;) {
3388                 for_each_rxq(pi, j, rxq) {
3389                         rss[i++] = rxq->iq.abs_id;
3390                         if (i == pi->rss_size)
3391                                 break;
3392                 }
3393         }
3394
3395         rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
3396             pi->rss_size);
3397         if (rc != 0) {
3398                 if_printf(ifp, "rss_config failed: %d\n", rc);
3399                 goto done;
3400         }
3401
3402         pi->rss = rss;
3403         pi->flags |= PORT_INIT_DONE;
3404 done:
3405         if (rc != 0)
3406                 port_full_uninit(pi);
3407
3408         return (rc);
3409 }
3410
3411 /*
3412  * Idempotent.
3413  */
3414 int
3415 port_full_uninit(struct port_info *pi)
3416 {
3417         struct adapter *sc = pi->adapter;
3418         int i;
3419         struct sge_rxq *rxq;
3420         struct sge_txq *txq;
3421 #ifdef TCP_OFFLOAD
3422         struct sge_ofld_rxq *ofld_rxq;
3423         struct sge_wrq *ofld_txq;
3424 #endif
3425
3426         if (pi->flags & PORT_INIT_DONE) {
3427
3428                 /* Need to quiesce queues.  XXX: ctrl queues? */
3429
3430                 for_each_txq(pi, i, txq) {
3431                         quiesce_eq(sc, &txq->eq);
3432                 }
3433
3434 #ifdef TCP_OFFLOAD
3435                 for_each_ofld_txq(pi, i, ofld_txq) {
3436                         quiesce_eq(sc, &ofld_txq->eq);
3437                 }
3438 #endif
3439
3440                 for_each_rxq(pi, i, rxq) {
3441                         quiesce_iq(sc, &rxq->iq);
3442                         quiesce_fl(sc, &rxq->fl);
3443                 }
3444
3445 #ifdef TCP_OFFLOAD
3446                 for_each_ofld_rxq(pi, i, ofld_rxq) {
3447                         quiesce_iq(sc, &ofld_rxq->iq);
3448                         quiesce_fl(sc, &ofld_rxq->fl);
3449                 }
3450 #endif
3451                 free(pi->rss, M_CXGBE);
3452         }
3453
3454         t4_teardown_port_queues(pi);
3455         pi->flags &= ~PORT_INIT_DONE;
3456
3457         return (0);
3458 }
3459
3460 static void
3461 quiesce_eq(struct adapter *sc, struct sge_eq *eq)
3462 {
3463         EQ_LOCK(eq);
3464         eq->flags |= EQ_DOOMED;
3465
3466         /*
3467          * Wait for the response to a credit flush if one's
3468          * pending.
3469          */
3470         while (eq->flags & EQ_CRFLUSHED)
3471                 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0);
3472         EQ_UNLOCK(eq);
3473
3474         callout_drain(&eq->tx_callout); /* XXX: iffy */
3475         pause("callout", 10);           /* Still iffy */
3476
3477         taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task);
3478 }
3479
3480 static void
3481 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
3482 {
3483         (void) sc;      /* unused */
3484
3485         /* Synchronize with the interrupt handler */
3486         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
3487                 pause("iqfree", 1);
3488 }
3489
3490 static void
3491 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
3492 {
3493         mtx_lock(&sc->sfl_lock);
3494         FL_LOCK(fl);
3495         fl->flags |= FL_DOOMED;
3496         FL_UNLOCK(fl);
3497         mtx_unlock(&sc->sfl_lock);
3498
3499         callout_drain(&sc->sfl_callout);
3500         KASSERT((fl->flags & FL_STARVING) == 0,
3501             ("%s: still starving", __func__));
3502 }
3503
3504 static int
3505 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
3506     driver_intr_t *handler, void *arg, char *name)
3507 {
3508         int rc;
3509
3510         irq->rid = rid;
3511         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
3512             RF_SHAREABLE | RF_ACTIVE);
3513         if (irq->res == NULL) {
3514                 device_printf(sc->dev,
3515                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
3516                 return (ENOMEM);
3517         }
3518
3519         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
3520             NULL, handler, arg, &irq->tag);
3521         if (rc != 0) {
3522                 device_printf(sc->dev,
3523                     "failed to setup interrupt for rid %d, name %s: %d\n",
3524                     rid, name, rc);
3525         } else if (name)
3526                 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
3527
3528         return (rc);
3529 }
3530
3531 static int
3532 t4_free_irq(struct adapter *sc, struct irq *irq)
3533 {
3534         if (irq->tag)
3535                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
3536         if (irq->res)
3537                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
3538
3539         bzero(irq, sizeof(*irq));
3540
3541         return (0);
3542 }
3543
3544 static void
3545 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
3546     unsigned int end)
3547 {
3548         uint32_t *p = (uint32_t *)(buf + start);
3549
3550         for ( ; start <= end; start += sizeof(uint32_t))
3551                 *p++ = t4_read_reg(sc, start);
3552 }
3553
3554 static void
3555 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
3556 {
3557         int i, n;
3558         const unsigned int *reg_ranges;
3559         static const unsigned int t4_reg_ranges[] = {
3560                 0x1008, 0x1108,
3561                 0x1180, 0x11b4,
3562                 0x11fc, 0x123c,
3563                 0x1300, 0x173c,
3564                 0x1800, 0x18fc,
3565                 0x3000, 0x30d8,
3566                 0x30e0, 0x5924,
3567                 0x5960, 0x59d4,
3568                 0x5a00, 0x5af8,
3569                 0x6000, 0x6098,
3570                 0x6100, 0x6150,
3571                 0x6200, 0x6208,
3572                 0x6240, 0x6248,
3573                 0x6280, 0x6338,
3574                 0x6370, 0x638c,
3575                 0x6400, 0x643c,
3576                 0x6500, 0x6524,
3577                 0x6a00, 0x6a38,
3578                 0x6a60, 0x6a78,
3579                 0x6b00, 0x6b84,
3580                 0x6bf0, 0x6c84,
3581                 0x6cf0, 0x6d84,
3582                 0x6df0, 0x6e84,
3583                 0x6ef0, 0x6f84,
3584                 0x6ff0, 0x7084,
3585                 0x70f0, 0x7184,
3586                 0x71f0, 0x7284,
3587                 0x72f0, 0x7384,
3588                 0x73f0, 0x7450,
3589                 0x7500, 0x7530,
3590                 0x7600, 0x761c,
3591                 0x7680, 0x76cc,
3592                 0x7700, 0x7798,
3593                 0x77c0, 0x77fc,
3594                 0x7900, 0x79fc,
3595                 0x7b00, 0x7c38,
3596                 0x7d00, 0x7efc,
3597                 0x8dc0, 0x8e1c,
3598                 0x8e30, 0x8e78,
3599                 0x8ea0, 0x8f6c,
3600                 0x8fc0, 0x9074,
3601                 0x90fc, 0x90fc,
3602                 0x9400, 0x9458,
3603                 0x9600, 0x96bc,
3604                 0x9800, 0x9808,
3605                 0x9820, 0x983c,
3606                 0x9850, 0x9864,
3607                 0x9c00, 0x9c6c,
3608                 0x9c80, 0x9cec,
3609                 0x9d00, 0x9d6c,
3610                 0x9d80, 0x9dec,
3611                 0x9e00, 0x9e6c,
3612                 0x9e80, 0x9eec,
3613                 0x9f00, 0x9f6c,
3614                 0x9f80, 0x9fec,
3615                 0xd004, 0xd03c,
3616                 0xdfc0, 0xdfe0,
3617                 0xe000, 0xea7c,
3618                 0xf000, 0x11110,
3619                 0x11118, 0x11190,
3620                 0x19040, 0x1906c,
3621                 0x19078, 0x19080,
3622                 0x1908c, 0x19124,
3623                 0x19150, 0x191b0,
3624                 0x191d0, 0x191e8,
3625                 0x19238, 0x1924c,
3626                 0x193f8, 0x19474,
3627                 0x19490, 0x194f8,
3628                 0x19800, 0x19f30,
3629                 0x1a000, 0x1a06c,
3630                 0x1a0b0, 0x1a120,
3631                 0x1a128, 0x1a138,
3632                 0x1a190, 0x1a1c4,
3633                 0x1a1fc, 0x1a1fc,
3634                 0x1e040, 0x1e04c,
3635                 0x1e284, 0x1e28c,
3636                 0x1e2c0, 0x1e2c0,
3637                 0x1e2e0, 0x1e2e0,
3638                 0x1e300, 0x1e384,
3639                 0x1e3c0, 0x1e3c8,
3640                 0x1e440, 0x1e44c,
3641                 0x1e684, 0x1e68c,
3642                 0x1e6c0, 0x1e6c0,
3643                 0x1e6e0, 0x1e6e0,
3644                 0x1e700, 0x1e784,
3645                 0x1e7c0, 0x1e7c8,
3646                 0x1e840, 0x1e84c,
3647                 0x1ea84, 0x1ea8c,
3648                 0x1eac0, 0x1eac0,
3649                 0x1eae0, 0x1eae0,
3650                 0x1eb00, 0x1eb84,
3651                 0x1ebc0, 0x1ebc8,
3652                 0x1ec40, 0x1ec4c,
3653                 0x1ee84, 0x1ee8c,
3654                 0x1eec0, 0x1eec0,
3655                 0x1eee0, 0x1eee0,
3656                 0x1ef00, 0x1ef84,
3657                 0x1efc0, 0x1efc8,
3658                 0x1f040, 0x1f04c,
3659                 0x1f284, 0x1f28c,
3660                 0x1f2c0, 0x1f2c0,
3661                 0x1f2e0, 0x1f2e0,
3662                 0x1f300, 0x1f384,
3663                 0x1f3c0, 0x1f3c8,
3664                 0x1f440, 0x1f44c,
3665                 0x1f684, 0x1f68c,
3666                 0x1f6c0, 0x1f6c0,
3667                 0x1f6e0, 0x1f6e0,
3668                 0x1f700, 0x1f784,
3669                 0x1f7c0, 0x1f7c8,
3670                 0x1f840, 0x1f84c,
3671                 0x1fa84, 0x1fa8c,
3672                 0x1fac0, 0x1fac0,
3673                 0x1fae0, 0x1fae0,
3674                 0x1fb00, 0x1fb84,
3675                 0x1fbc0, 0x1fbc8,
3676                 0x1fc40, 0x1fc4c,
3677                 0x1fe84, 0x1fe8c,
3678                 0x1fec0, 0x1fec0,
3679                 0x1fee0, 0x1fee0,
3680                 0x1ff00, 0x1ff84,
3681                 0x1ffc0, 0x1ffc8,
3682                 0x20000, 0x2002c,
3683                 0x20100, 0x2013c,
3684                 0x20190, 0x201c8,
3685                 0x20200, 0x20318,
3686                 0x20400, 0x20528,
3687                 0x20540, 0x20614,
3688                 0x21000, 0x21040,
3689                 0x2104c, 0x21060,
3690                 0x210c0, 0x210ec,
3691                 0x21200, 0x21268,
3692                 0x21270, 0x21284,
3693                 0x212fc, 0x21388,
3694                 0x21400, 0x21404,
3695                 0x21500, 0x21518,
3696                 0x2152c, 0x2153c,
3697                 0x21550, 0x21554,
3698                 0x21600, 0x21600,
3699                 0x21608, 0x21628,
3700                 0x21630, 0x2163c,
3701                 0x21700, 0x2171c,
3702                 0x21780, 0x2178c,
3703                 0x21800, 0x21c38,
3704                 0x21c80, 0x21d7c,
3705                 0x21e00, 0x21e04,
3706                 0x22000, 0x2202c,
3707                 0x22100, 0x2213c,
3708                 0x22190, 0x221c8,
3709                 0x22200, 0x22318,
3710                 0x22400, 0x22528,
3711                 0x22540, 0x22614,
3712                 0x23000, 0x23040,
3713                 0x2304c, 0x23060,
3714                 0x230c0, 0x230ec,
3715                 0x23200, 0x23268,
3716                 0x23270, 0x23284,
3717                 0x232fc, 0x23388,
3718                 0x23400, 0x23404,
3719                 0x23500, 0x23518,
3720                 0x2352c, 0x2353c,
3721                 0x23550, 0x23554,
3722                 0x23600, 0x23600,
3723                 0x23608, 0x23628,
3724                 0x23630, 0x2363c,
3725                 0x23700, 0x2371c,
3726                 0x23780, 0x2378c,
3727                 0x23800, 0x23c38,
3728                 0x23c80, 0x23d7c,
3729                 0x23e00, 0x23e04,
3730                 0x24000, 0x2402c,
3731                 0x24100, 0x2413c,
3732                 0x24190, 0x241c8,
3733                 0x24200, 0x24318,
3734                 0x24400, 0x24528,
3735                 0x24540, 0x24614,
3736                 0x25000, 0x25040,
3737                 0x2504c, 0x25060,
3738                 0x250c0, 0x250ec,
3739                 0x25200, 0x25268,
3740                 0x25270, 0x25284,
3741                 0x252fc, 0x25388,
3742                 0x25400, 0x25404,
3743                 0x25500, 0x25518,
3744                 0x2552c, 0x2553c,
3745                 0x25550, 0x25554,
3746                 0x25600, 0x25600,
3747                 0x25608, 0x25628,
3748                 0x25630, 0x2563c,
3749                 0x25700, 0x2571c,
3750                 0x25780, 0x2578c,
3751                 0x25800, 0x25c38,
3752                 0x25c80, 0x25d7c,
3753                 0x25e00, 0x25e04,
3754                 0x26000, 0x2602c,
3755                 0x26100, 0x2613c,
3756                 0x26190, 0x261c8,
3757                 0x26200, 0x26318,
3758                 0x26400, 0x26528,
3759                 0x26540, 0x26614,
3760                 0x27000, 0x27040,
3761                 0x2704c, 0x27060,
3762                 0x270c0, 0x270ec,
3763                 0x27200, 0x27268,
3764                 0x27270, 0x27284,
3765                 0x272fc, 0x27388,
3766                 0x27400, 0x27404,
3767                 0x27500, 0x27518,
3768                 0x2752c, 0x2753c,
3769                 0x27550, 0x27554,
3770                 0x27600, 0x27600,
3771                 0x27608, 0x27628,
3772                 0x27630, 0x2763c,
3773                 0x27700, 0x2771c,
3774                 0x27780, 0x2778c,
3775                 0x27800, 0x27c38,
3776                 0x27c80, 0x27d7c,
3777                 0x27e00, 0x27e04
3778         };
3779         static const unsigned int t5_reg_ranges[] = {
3780                 0x1008, 0x1148,
3781                 0x1180, 0x11b4,
3782                 0x11fc, 0x123c,
3783                 0x1280, 0x173c,
3784                 0x1800, 0x18fc,
3785                 0x3000, 0x3028,
3786                 0x3060, 0x30d8,
3787                 0x30e0, 0x30fc,
3788                 0x3140, 0x357c,
3789                 0x35a8, 0x35cc,
3790                 0x35ec, 0x35ec,
3791                 0x3600, 0x5624,
3792                 0x56cc, 0x575c,
3793                 0x580c, 0x5814,
3794                 0x5890, 0x58bc,
3795                 0x5940, 0x59dc,
3796                 0x59fc, 0x5a18,
3797                 0x5a60, 0x5a9c,
3798                 0x5b94, 0x5bfc,
3799                 0x6000, 0x6040,
3800                 0x6058, 0x614c,
3801                 0x7700, 0x7798,
3802                 0x77c0, 0x78fc,
3803                 0x7b00, 0x7c54,
3804                 0x7d00, 0x7efc,
3805                 0x8dc0, 0x8de0,
3806                 0x8df8, 0x8e84,
3807                 0x8ea0, 0x8f84,
3808                 0x8fc0, 0x90f8,
3809                 0x9400, 0x9470,
3810                 0x9600, 0x96f4,
3811                 0x9800, 0x9808,
3812                 0x9820, 0x983c,
3813                 0x9850, 0x9864,
3814                 0x9c00, 0x9c6c,
3815                 0x9c80, 0x9cec,
3816                 0x9d00, 0x9d6c,
3817                 0x9d80, 0x9dec,
3818                 0x9e00, 0x9e6c,
3819                 0x9e80, 0x9eec,
3820                 0x9f00, 0x9f6c,
3821                 0x9f80, 0xa020,
3822                 0xd004, 0xd03c,
3823                 0xdfc0, 0xdfe0,
3824                 0xe000, 0x11088,
3825                 0x1109c, 0x11110,
3826                 0x11118, 0x1117c,
3827                 0x11190, 0x11204,
3828                 0x19040, 0x1906c,
3829                 0x19078, 0x19080,
3830                 0x1908c, 0x19124,
3831                 0x19150, 0x191b0,
3832                 0x191d0, 0x191e8,
3833                 0x19238, 0x19290,
3834                 0x193f8, 0x19474,
3835                 0x19490, 0x194cc,
3836                 0x194f0, 0x194f8,
3837                 0x19c00, 0x19c60,
3838                 0x19c94, 0x19e10,
3839                 0x19e50, 0x19f34,
3840                 0x19f40, 0x19f50,
3841                 0x19f90, 0x19fe4,
3842                 0x1a000, 0x1a06c,
3843                 0x1a0b0, 0x1a120,
3844                 0x1a128, 0x1a138,
3845                 0x1a190, 0x1a1c4,
3846                 0x1a1fc, 0x1a1fc,
3847                 0x1e008, 0x1e00c,
3848                 0x1e040, 0x1e04c,
3849                 0x1e284, 0x1e290,
3850                 0x1e2c0, 0x1e2c0,
3851                 0x1e2e0, 0x1e2e0,
3852                 0x1e300, 0x1e384,
3853                 0x1e3c0, 0x1e3c8,
3854                 0x1e408, 0x1e40c,
3855                 0x1e440, 0x1e44c,
3856                 0x1e684, 0x1e690,
3857                 0x1e6c0, 0x1e6c0,
3858                 0x1e6e0, 0x1e6e0,
3859                 0x1e700, 0x1e784,
3860                 0x1e7c0, 0x1e7c8,
3861                 0x1e808, 0x1e80c,
3862                 0x1e840, 0x1e84c,
3863                 0x1ea84, 0x1ea90,
3864                 0x1eac0, 0x1eac0,
3865                 0x1eae0, 0x1eae0,
3866                 0x1eb00, 0x1eb84,
3867                 0x1ebc0, 0x1ebc8,
3868                 0x1ec08, 0x1ec0c,
3869                 0x1ec40, 0x1ec4c,
3870                 0x1ee84, 0x1ee90,
3871                 0x1eec0, 0x1eec0,
3872                 0x1eee0, 0x1eee0,
3873                 0x1ef00, 0x1ef84,
3874                 0x1efc0, 0x1efc8,
3875                 0x1f008, 0x1f00c,
3876                 0x1f040, 0x1f04c,
3877                 0x1f284, 0x1f290,
3878                 0x1f2c0, 0x1f2c0,
3879                 0x1f2e0, 0x1f2e0,
3880                 0x1f300, 0x1f384,
3881                 0x1f3c0, 0x1f3c8,
3882                 0x1f408, 0x1f40c,
3883                 0x1f440, 0x1f44c,
3884                 0x1f684, 0x1f690,
3885                 0x1f6c0, 0x1f6c0,
3886                 0x1f6e0, 0x1f6e0,
3887                 0x1f700, 0x1f784,
3888                 0x1f7c0, 0x1f7c8,
3889                 0x1f808, 0x1f80c,
3890                 0x1f840, 0x1f84c,
3891                 0x1fa84, 0x1fa90,
3892                 0x1fac0, 0x1fac0,
3893                 0x1fae0, 0x1fae0,
3894                 0x1fb00, 0x1fb84,
3895                 0x1fbc0, 0x1fbc8,
3896                 0x1fc08, 0x1fc0c,
3897                 0x1fc40, 0x1fc4c,
3898                 0x1fe84, 0x1fe90,
3899                 0x1fec0, 0x1fec0,
3900                 0x1fee0, 0x1fee0,
3901                 0x1ff00, 0x1ff84,
3902                 0x1ffc0, 0x1ffc8,
3903                 0x30000, 0x30030,
3904                 0x30100, 0x30144,
3905                 0x30190, 0x301d0,
3906                 0x30200, 0x30318,
3907                 0x30400, 0x3052c,
3908                 0x30540, 0x3061c,
3909                 0x30800, 0x30834,
3910                 0x308c0, 0x30908,
3911                 0x30910, 0x309ac,
3912                 0x30a00, 0x30a2c,
3913                 0x30a44, 0x30a50,
3914                 0x30a74, 0x30c24,
3915                 0x30d00, 0x30d00,
3916                 0x30d08, 0x30d14,
3917                 0x30d1c, 0x30d20,
3918                 0x30d3c, 0x30d50,
3919                 0x31200, 0x3120c,
3920                 0x31220, 0x31220,
3921                 0x31240, 0x31240,
3922                 0x31600, 0x3160c,
3923                 0x31a00, 0x31a1c,
3924                 0x31e00, 0x31e20,
3925                 0x31e38, 0x31e3c,
3926                 0x31e80, 0x31e80,
3927                 0x31e88, 0x31ea8,
3928                 0x31eb0, 0x31eb4,
3929                 0x31ec8, 0x31ed4,
3930                 0x31fb8, 0x32004,
3931                 0x32200, 0x32200,
3932                 0x32208, 0x32240,
3933                 0x32248, 0x32280,
3934                 0x32288, 0x322c0,
3935                 0x322c8, 0x322fc,
3936                 0x32600, 0x32630,
3937                 0x32a00, 0x32abc,
3938                 0x32b00, 0x32b70,
3939                 0x33000, 0x33048,
3940                 0x33060, 0x3309c,
3941                 0x330f0, 0x33148,
3942                 0x33160, 0x3319c,
3943                 0x331f0, 0x332e4,
3944                 0x332f8, 0x333e4,
3945                 0x333f8, 0x33448,
3946                 0x33460, 0x3349c,
3947                 0x334f0, 0x33548,
3948                 0x33560, 0x3359c,
3949                 0x335f0, 0x336e4,
3950                 0x336f8, 0x337e4,
3951                 0x337f8, 0x337fc,
3952                 0x33814, 0x33814,
3953                 0x3382c, 0x3382c,
3954                 0x33880, 0x3388c,
3955                 0x338e8, 0x338ec,
3956                 0x33900, 0x33948,
3957                 0x33960, 0x3399c,
3958                 0x339f0, 0x33ae4,
3959                 0x33af8, 0x33b10,
3960                 0x33b28, 0x33b28,
3961                 0x33b3c, 0x33b50,
3962                 0x33bf0, 0x33c10,
3963                 0x33c28, 0x33c28,
3964                 0x33c3c, 0x33c50,
3965                 0x33cf0, 0x33cfc,
3966                 0x34000, 0x34030,
3967                 0x34100, 0x34144,
3968                 0x34190, 0x341d0,
3969                 0x34200, 0x34318,
3970                 0x34400, 0x3452c,
3971                 0x34540, 0x3461c,
3972                 0x34800, 0x34834,
3973                 0x348c0, 0x34908,
3974                 0x34910, 0x349ac,
3975                 0x34a00, 0x34a2c,
3976                 0x34a44, 0x34a50,
3977                 0x34a74, 0x34c24,
3978                 0x34d00, 0x34d00,
3979                 0x34d08, 0x34d14,
3980                 0x34d1c, 0x34d20,
3981                 0x34d3c, 0x34d50,
3982                 0x35200, 0x3520c,
3983                 0x35220, 0x35220,
3984                 0x35240, 0x35240,
3985                 0x35600, 0x3560c,
3986                 0x35a00, 0x35a1c,
3987                 0x35e00, 0x35e20,
3988                 0x35e38, 0x35e3c,
3989                 0x35e80, 0x35e80,
3990                 0x35e88, 0x35ea8,
3991                 0x35eb0, 0x35eb4,
3992                 0x35ec8, 0x35ed4,
3993                 0x35fb8, 0x36004,
3994                 0x36200, 0x36200,
3995                 0x36208, 0x36240,
3996                 0x36248, 0x36280,
3997                 0x36288, 0x362c0,
3998                 0x362c8, 0x362fc,
3999                 0x36600, 0x36630,
4000                 0x36a00, 0x36abc,
4001                 0x36b00, 0x36b70,
4002                 0x37000, 0x37048,
4003                 0x37060, 0x3709c,
4004                 0x370f0, 0x37148,
4005                 0x37160, 0x3719c,
4006                 0x371f0, 0x372e4,
4007                 0x372f8, 0x373e4,
4008                 0x373f8, 0x37448,
4009                 0x37460, 0x3749c,
4010                 0x374f0, 0x37548,
4011                 0x37560, 0x3759c,
4012                 0x375f0, 0x376e4,
4013                 0x376f8, 0x377e4,
4014                 0x377f8, 0x377fc,
4015                 0x37814, 0x37814,
4016                 0x3782c, 0x3782c,
4017                 0x37880, 0x3788c,
4018                 0x378e8, 0x378ec,
4019                 0x37900, 0x37948,
4020                 0x37960, 0x3799c,
4021                 0x379f0, 0x37ae4,
4022                 0x37af8, 0x37b10,
4023                 0x37b28, 0x37b28,
4024                 0x37b3c, 0x37b50,
4025                 0x37bf0, 0x37c10,
4026                 0x37c28, 0x37c28,
4027                 0x37c3c, 0x37c50,
4028                 0x37cf0, 0x37cfc,
4029                 0x38000, 0x38030,
4030                 0x38100, 0x38144,
4031                 0x38190, 0x381d0,
4032                 0x38200, 0x38318,
4033                 0x38400, 0x3852c,
4034                 0x38540, 0x3861c,
4035                 0x38800, 0x38834,
4036                 0x388c0, 0x38908,
4037                 0x38910, 0x389ac,
4038                 0x38a00, 0x38a2c,
4039                 0x38a44, 0x38a50,
4040                 0x38a74, 0x38c24,
4041                 0x38d00, 0x38d00,
4042                 0x38d08, 0x38d14,
4043                 0x38d1c, 0x38d20,
4044                 0x38d3c, 0x38d50,
4045                 0x39200, 0x3920c,
4046                 0x39220, 0x39220,
4047                 0x39240, 0x39240,
4048                 0x39600, 0x3960c,
4049                 0x39a00, 0x39a1c,
4050                 0x39e00, 0x39e20,
4051                 0x39e38, 0x39e3c,
4052                 0x39e80, 0x39e80,
4053                 0x39e88, 0x39ea8,
4054                 0x39eb0, 0x39eb4,
4055                 0x39ec8, 0x39ed4,
4056                 0x39fb8, 0x3a004,
4057                 0x3a200, 0x3a200,
4058                 0x3a208, 0x3a240,
4059                 0x3a248, 0x3a280,
4060                 0x3a288, 0x3a2c0,
4061                 0x3a2c8, 0x3a2fc,
4062                 0x3a600, 0x3a630,
4063                 0x3aa00, 0x3aabc,
4064                 0x3ab00, 0x3ab70,
4065                 0x3b000, 0x3b048,
4066                 0x3b060, 0x3b09c,
4067                 0x3b0f0, 0x3b148,
4068                 0x3b160, 0x3b19c,
4069                 0x3b1f0, 0x3b2e4,
4070                 0x3b2f8, 0x3b3e4,
4071                 0x3b3f8, 0x3b448,
4072                 0x3b460, 0x3b49c,
4073                 0x3b4f0, 0x3b548,
4074                 0x3b560, 0x3b59c,
4075                 0x3b5f0, 0x3b6e4,
4076                 0x3b6f8, 0x3b7e4,
4077                 0x3b7f8, 0x3b7fc,
4078                 0x3b814, 0x3b814,
4079                 0x3b82c, 0x3b82c,
4080                 0x3b880, 0x3b88c,
4081                 0x3b8e8, 0x3b8ec,
4082                 0x3b900, 0x3b948,
4083                 0x3b960, 0x3b99c,
4084                 0x3b9f0, 0x3bae4,
4085                 0x3baf8, 0x3bb10,
4086                 0x3bb28, 0x3bb28,
4087                 0x3bb3c, 0x3bb50,
4088                 0x3bbf0, 0x3bc10,
4089                 0x3bc28, 0x3bc28,
4090                 0x3bc3c, 0x3bc50,
4091                 0x3bcf0, 0x3bcfc,
4092                 0x3c000, 0x3c030,
4093                 0x3c100, 0x3c144,
4094                 0x3c190, 0x3c1d0,
4095                 0x3c200, 0x3c318,
4096                 0x3c400, 0x3c52c,
4097                 0x3c540, 0x3c61c,
4098                 0x3c800, 0x3c834,
4099                 0x3c8c0, 0x3c908,
4100                 0x3c910, 0x3c9ac,
4101                 0x3ca00, 0x3ca2c,
4102                 0x3ca44, 0x3ca50,
4103                 0x3ca74, 0x3cc24,
4104                 0x3cd00, 0x3cd00,
4105                 0x3cd08, 0x3cd14,
4106                 0x3cd1c, 0x3cd20,
4107                 0x3cd3c, 0x3cd50,
4108                 0x3d200, 0x3d20c,
4109                 0x3d220, 0x3d220,
4110                 0x3d240, 0x3d240,
4111                 0x3d600, 0x3d60c,
4112                 0x3da00, 0x3da1c,
4113                 0x3de00, 0x3de20,
4114                 0x3de38, 0x3de3c,
4115                 0x3de80, 0x3de80,
4116                 0x3de88, 0x3dea8,
4117                 0x3deb0, 0x3deb4,
4118                 0x3dec8, 0x3ded4,
4119                 0x3dfb8, 0x3e004,
4120                 0x3e200, 0x3e200,
4121                 0x3e208, 0x3e240,
4122                 0x3e248, 0x3e280,
4123                 0x3e288, 0x3e2c0,
4124                 0x3e2c8, 0x3e2fc,
4125                 0x3e600, 0x3e630,
4126                 0x3ea00, 0x3eabc,
4127                 0x3eb00, 0x3eb70,
4128                 0x3f000, 0x3f048,
4129                 0x3f060, 0x3f09c,
4130                 0x3f0f0, 0x3f148,
4131                 0x3f160, 0x3f19c,
4132                 0x3f1f0, 0x3f2e4,
4133                 0x3f2f8, 0x3f3e4,
4134                 0x3f3f8, 0x3f448,
4135                 0x3f460, 0x3f49c,
4136                 0x3f4f0, 0x3f548,
4137                 0x3f560, 0x3f59c,
4138                 0x3f5f0, 0x3f6e4,
4139                 0x3f6f8, 0x3f7e4,
4140                 0x3f7f8, 0x3f7fc,
4141                 0x3f814, 0x3f814,
4142                 0x3f82c, 0x3f82c,
4143                 0x3f880, 0x3f88c,
4144                 0x3f8e8, 0x3f8ec,
4145                 0x3f900, 0x3f948,
4146                 0x3f960, 0x3f99c,
4147                 0x3f9f0, 0x3fae4,
4148                 0x3faf8, 0x3fb10,
4149                 0x3fb28, 0x3fb28,
4150                 0x3fb3c, 0x3fb50,
4151                 0x3fbf0, 0x3fc10,
4152                 0x3fc28, 0x3fc28,
4153                 0x3fc3c, 0x3fc50,
4154                 0x3fcf0, 0x3fcfc,
4155                 0x40000, 0x4000c,
4156                 0x40040, 0x40068,
4157                 0x4007c, 0x40144,
4158                 0x40180, 0x4018c,
4159                 0x40200, 0x40298,
4160                 0x402ac, 0x4033c,
4161                 0x403f8, 0x403fc,
4162                 0x41304, 0x413c4,
4163                 0x41400, 0x4141c,
4164                 0x41480, 0x414d0,
4165                 0x44000, 0x44078,
4166                 0x440c0, 0x44278,
4167                 0x442c0, 0x44478,
4168                 0x444c0, 0x44678,
4169                 0x446c0, 0x44878,
4170                 0x448c0, 0x449fc,
4171                 0x45000, 0x45068,
4172                 0x45080, 0x45084,
4173                 0x450a0, 0x450b0,
4174                 0x45200, 0x45268,
4175                 0x45280, 0x45284,
4176                 0x452a0, 0x452b0,
4177                 0x460c0, 0x460e4,
4178                 0x47000, 0x4708c,
4179                 0x47200, 0x47250,
4180                 0x47400, 0x47420,
4181                 0x47600, 0x47618,
4182                 0x47800, 0x47814,
4183                 0x48000, 0x4800c,
4184                 0x48040, 0x48068,
4185                 0x4807c, 0x48144,
4186                 0x48180, 0x4818c,
4187                 0x48200, 0x48298,
4188                 0x482ac, 0x4833c,
4189                 0x483f8, 0x483fc,
4190                 0x49304, 0x493c4,
4191                 0x49400, 0x4941c,
4192                 0x49480, 0x494d0,
4193                 0x4c000, 0x4c078,
4194                 0x4c0c0, 0x4c278,
4195                 0x4c2c0, 0x4c478,
4196                 0x4c4c0, 0x4c678,
4197                 0x4c6c0, 0x4c878,
4198                 0x4c8c0, 0x4c9fc,
4199                 0x4d000, 0x4d068,
4200                 0x4d080, 0x4d084,
4201                 0x4d0a0, 0x4d0b0,
4202                 0x4d200, 0x4d268,
4203                 0x4d280, 0x4d284,
4204                 0x4d2a0, 0x4d2b0,
4205                 0x4e0c0, 0x4e0e4,
4206                 0x4f000, 0x4f08c,
4207                 0x4f200, 0x4f250,
4208                 0x4f400, 0x4f420,
4209                 0x4f600, 0x4f618,
4210                 0x4f800, 0x4f814,
4211                 0x50000, 0x500cc,
4212                 0x50400, 0x50400,
4213                 0x50800, 0x508cc,
4214                 0x50c00, 0x50c00,
4215                 0x51000, 0x5101c,
4216                 0x51300, 0x51308,
4217         };
4218
4219         if (is_t4(sc)) {
4220                 reg_ranges = &t4_reg_ranges[0];
4221                 n = nitems(t4_reg_ranges);
4222         } else {
4223                 reg_ranges = &t5_reg_ranges[0];
4224                 n = nitems(t5_reg_ranges);
4225         }
4226
4227         regs->version = chip_id(sc) | chip_rev(sc) << 10;
4228         for (i = 0; i < n; i += 2)
4229                 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
4230 }
4231
4232 static void
4233 cxgbe_tick(void *arg)
4234 {
4235         struct port_info *pi = arg;
4236         struct adapter *sc = pi->adapter;
4237         struct ifnet *ifp = pi->ifp;
4238         struct sge_txq *txq;
4239         int i, drops;
4240         struct port_stats *s = &pi->stats;
4241
4242         PORT_LOCK(pi);
4243         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4244                 PORT_UNLOCK(pi);
4245                 return; /* without scheduling another callout */
4246         }
4247
4248         t4_get_port_stats(sc, pi->tx_chan, s);
4249
4250         ifp->if_opackets = s->tx_frames - s->tx_pause;
4251         ifp->if_ipackets = s->rx_frames - s->rx_pause;
4252         ifp->if_obytes = s->tx_octets - s->tx_pause * 64;
4253         ifp->if_ibytes = s->rx_octets - s->rx_pause * 64;
4254         ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause;
4255         ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause;
4256         ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
4257             s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
4258             s->rx_trunc3;
4259         for (i = 0; i < 4; i++) {
4260                 if (pi->rx_chan_map & (1 << i)) {
4261                         uint32_t v;
4262
4263                         /*
4264                          * XXX: indirect reads from the same ADDR/DATA pair can
4265                          * race with each other.
4266                          */
4267                         t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
4268                             1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4269                         ifp->if_iqdrops += v;
4270                 }
4271         }
4272
4273         drops = s->tx_drop;
4274         for_each_txq(pi, i, txq)
4275                 drops += txq->br->br_drops;
4276         ifp->if_snd.ifq_drops = drops;
4277
4278         ifp->if_oerrors = s->tx_error_frames;
4279         ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long +
4280             s->rx_fcs_err + s->rx_len_err;
4281
4282         callout_schedule(&pi->tick, hz);
4283         PORT_UNLOCK(pi);
4284 }
4285
4286 static void
4287 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4288 {
4289         struct ifnet *vlan;
4290
4291         if (arg != ifp || ifp->if_type != IFT_ETHER)
4292                 return;
4293
4294         vlan = VLAN_DEVAT(ifp, vid);
4295         VLAN_SETCOOKIE(vlan, ifp);
4296 }
4297
4298 static int
4299 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4300 {
4301
4302 #ifdef INVARIANTS
4303         panic("%s: opcode 0x%02x on iq %p with payload %p",
4304             __func__, rss->opcode, iq, m);
4305 #else
4306         log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
4307             __func__, rss->opcode, iq, m);
4308         m_freem(m);
4309 #endif
4310         return (EDOOFUS);
4311 }
4312
4313 int
4314 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
4315 {
4316         uintptr_t *loc, new;
4317
4318         if (opcode >= nitems(sc->cpl_handler))
4319                 return (EINVAL);
4320
4321         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
4322         loc = (uintptr_t *) &sc->cpl_handler[opcode];
4323         atomic_store_rel_ptr(loc, new);
4324
4325         return (0);
4326 }
4327
4328 static int
4329 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
4330 {
4331
4332 #ifdef INVARIANTS
4333         panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
4334 #else
4335         log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
4336             __func__, iq, ctrl);
4337 #endif
4338         return (EDOOFUS);
4339 }
4340
4341 int
4342 t4_register_an_handler(struct adapter *sc, an_handler_t h)
4343 {
4344         uintptr_t *loc, new;
4345
4346         new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
4347         loc = (uintptr_t *) &sc->an_handler;
4348         atomic_store_rel_ptr(loc, new);
4349
4350         return (0);
4351 }
4352
4353 static int
4354 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
4355 {
4356         const struct cpl_fw6_msg *cpl =
4357             __containerof(rpl, struct cpl_fw6_msg, data[0]);
4358
4359 #ifdef INVARIANTS
4360         panic("%s: fw_msg type %d", __func__, cpl->type);
4361 #else
4362         log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
4363 #endif
4364         return (EDOOFUS);
4365 }
4366
4367 int
4368 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
4369 {
4370         uintptr_t *loc, new;
4371
4372         if (type >= nitems(sc->fw_msg_handler))
4373                 return (EINVAL);
4374
4375         /*
4376          * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
4377          * handler dispatch table.  Reject any attempt to install a handler for
4378          * this subtype.
4379          */
4380         if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
4381                 return (EINVAL);
4382
4383         new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
4384         loc = (uintptr_t *) &sc->fw_msg_handler[type];
4385         atomic_store_rel_ptr(loc, new);
4386
4387         return (0);
4388 }
4389
4390 static int
4391 t4_sysctls(struct adapter *sc)
4392 {
4393         struct sysctl_ctx_list *ctx;
4394         struct sysctl_oid *oid;
4395         struct sysctl_oid_list *children, *c0;
4396         static char *caps[] = {
4397                 "\20\1PPP\2QFC\3DCBX",                  /* caps[0] linkcaps */
4398                 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL"        /* caps[1] niccaps */
4399                     "\6HASHFILTER\7ETHOFLD",
4400                 "\20\1TOE",                             /* caps[2] toecaps */
4401                 "\20\1RDDP\2RDMAC",                     /* caps[3] rdmacaps */
4402                 "\20\1INITIATOR_PDU\2TARGET_PDU"        /* caps[4] iscsicaps */
4403                     "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
4404                     "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
4405                 "\20\1INITIATOR\2TARGET\3CTRL_OFLD"     /* caps[5] fcoecaps */
4406                     "\4PO_INITIAOR\5PO_TARGET"
4407         };
4408         static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
4409
4410         ctx = device_get_sysctl_ctx(sc->dev);
4411
4412         /*
4413          * dev.t4nex.X.
4414          */
4415         oid = device_get_sysctl_tree(sc->dev);
4416         c0 = children = SYSCTL_CHILDREN(oid);
4417
4418         sc->sc_do_rxcopy = 1;
4419         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
4420             &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
4421
4422         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
4423             sc->params.nports, "# of ports");
4424
4425         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
4426             NULL, chip_rev(sc), "chip hardware revision");
4427
4428         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
4429             CTLFLAG_RD, sc->fw_version, 0, "firmware version");
4430
4431         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
4432             CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
4433
4434         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
4435             sc->cfcsum, "config file checksum");
4436
4437         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
4438             CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
4439             sysctl_bitfield, "A", "available doorbells");
4440
4441         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
4442             CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
4443             sysctl_bitfield, "A", "available link capabilities");
4444
4445         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
4446             CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
4447             sysctl_bitfield, "A", "available NIC capabilities");
4448
4449         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
4450             CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
4451             sysctl_bitfield, "A", "available TCP offload capabilities");
4452
4453         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
4454             CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
4455             sysctl_bitfield, "A", "available RDMA capabilities");
4456
4457         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
4458             CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
4459             sysctl_bitfield, "A", "available iSCSI capabilities");
4460
4461         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
4462             CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
4463             sysctl_bitfield, "A", "available FCoE capabilities");
4464
4465         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
4466             sc->params.vpd.cclk, "core clock frequency (in KHz)");
4467
4468         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
4469             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
4470             sizeof(sc->sge.timer_val), sysctl_int_array, "A",
4471             "interrupt holdoff timer values (us)");
4472
4473         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
4474             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
4475             sizeof(sc->sge.counter_val), sysctl_int_array, "A",
4476             "interrupt holdoff packet counter values");
4477
4478         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
4479             NULL, sc->tids.nftids, "number of filters");
4480
4481         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
4482             CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
4483             "chip temperature (in Celsius)");
4484
4485         t4_sge_sysctls(sc, ctx, children);
4486
4487         sc->lro_timeout = 100;
4488         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
4489             &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
4490
4491 #ifdef SBUF_DRAIN
4492         /*
4493          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
4494          */
4495         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
4496             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
4497             "logs and miscellaneous information");
4498         children = SYSCTL_CHILDREN(oid);
4499
4500         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
4501             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4502             sysctl_cctrl, "A", "congestion control");
4503
4504         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
4505             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4506             sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
4507
4508         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
4509             CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
4510             sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
4511
4512         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
4513             CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
4514             sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
4515
4516         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
4517             CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
4518             sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
4519
4520         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
4521             CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
4522             sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
4523
4524         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
4525             CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
4526             sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
4527
4528         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
4529             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4530             sysctl_cim_la, "A", "CIM logic analyzer");
4531
4532         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
4533             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4534             sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
4535
4536         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
4537             CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
4538             sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
4539
4540         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
4541             CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
4542             sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
4543
4544         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
4545             CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
4546             sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
4547
4548         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
4549             CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
4550             sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
4551
4552         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
4553             CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
4554             sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
4555
4556         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
4557             CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
4558             sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
4559
4560         if (is_t5(sc)) {
4561                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
4562                     CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
4563                     sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
4564
4565                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
4566                     CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
4567                     sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
4568         }
4569
4570         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
4571             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4572             sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
4573
4574         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
4575             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4576             sysctl_cim_qcfg, "A", "CIM queue configuration");
4577
4578         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
4579             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4580             sysctl_cpl_stats, "A", "CPL statistics");
4581
4582         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
4583             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4584             sysctl_ddp_stats, "A", "DDP statistics");
4585
4586         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
4587             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4588             sysctl_devlog, "A", "firmware's device log");
4589
4590         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
4591             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4592             sysctl_fcoe_stats, "A", "FCoE statistics");
4593
4594         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
4595             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4596             sysctl_hw_sched, "A", "hardware scheduler ");
4597
4598         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
4599             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4600             sysctl_l2t, "A", "hardware L2 table");
4601
4602         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
4603             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4604             sysctl_lb_stats, "A", "loopback statistics");
4605
4606         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
4607             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4608             sysctl_meminfo, "A", "memory regions");
4609
4610         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
4611             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4612             sysctl_mps_tcam, "A", "MPS TCAM entries");
4613
4614         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
4615             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4616             sysctl_path_mtus, "A", "path MTUs");
4617
4618         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
4619             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4620             sysctl_pm_stats, "A", "PM statistics");
4621
4622         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
4623             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4624             sysctl_rdma_stats, "A", "RDMA statistics");
4625
4626         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
4627             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4628             sysctl_tcp_stats, "A", "TCP statistics");
4629
4630         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
4631             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4632             sysctl_tids, "A", "TID information");
4633
4634         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
4635             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4636             sysctl_tp_err_stats, "A", "TP error statistics");
4637
4638         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
4639             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4640             sysctl_tp_la, "A", "TP logic analyzer");
4641
4642         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
4643             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4644             sysctl_tx_rate, "A", "Tx rate");
4645
4646         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
4647             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4648             sysctl_ulprx_la, "A", "ULPRX logic analyzer");
4649
4650         if (is_t5(sc)) {
4651                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
4652                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4653                     sysctl_wcwr_stats, "A", "write combined work requests");
4654         }
4655 #endif
4656
4657 #ifdef TCP_OFFLOAD
4658         if (is_offload(sc)) {
4659                 /*
4660                  * dev.t4nex.X.toe.
4661                  */
4662                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
4663                     NULL, "TOE parameters");
4664                 children = SYSCTL_CHILDREN(oid);
4665
4666                 sc->tt.sndbuf = 256 * 1024;
4667                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
4668                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
4669
4670                 sc->tt.ddp = 0;
4671                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
4672                     &sc->tt.ddp, 0, "DDP allowed");
4673
4674                 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
4675                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
4676                     &sc->tt.indsz, 0, "DDP max indicate size allowed");
4677
4678                 sc->tt.ddp_thres =
4679                     G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
4680                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
4681                     &sc->tt.ddp_thres, 0, "DDP threshold");
4682
4683                 sc->tt.rx_coalesce = 1;
4684                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
4685                     CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
4686         }
4687 #endif
4688
4689
4690         return (0);
4691 }
4692
4693 static int
4694 cxgbe_sysctls(struct port_info *pi)
4695 {
4696         struct sysctl_ctx_list *ctx;
4697         struct sysctl_oid *oid;
4698         struct sysctl_oid_list *children;
4699         struct adapter *sc = pi->adapter;
4700
4701         ctx = device_get_sysctl_ctx(pi->dev);
4702
4703         /*
4704          * dev.cxgbe.X.
4705          */
4706         oid = device_get_sysctl_tree(pi->dev);
4707         children = SYSCTL_CHILDREN(oid);
4708
4709         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
4710            CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
4711         if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
4712                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
4713                     CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
4714                     "PHY temperature (in Celsius)");
4715                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
4716                     CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
4717                     "PHY firmware version");
4718         }
4719         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
4720             &pi->nrxq, 0, "# of rx queues");
4721         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
4722             &pi->ntxq, 0, "# of tx queues");
4723         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
4724             &pi->first_rxq, 0, "index of first rx queue");
4725         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
4726             &pi->first_txq, 0, "index of first tx queue");
4727         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
4728             CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU",
4729             "Reserve queue 0 for non-flowid packets");
4730
4731 #ifdef TCP_OFFLOAD
4732         if (is_offload(sc)) {
4733                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
4734                     &pi->nofldrxq, 0,
4735                     "# of rx queues for offloaded TCP connections");
4736                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
4737                     &pi->nofldtxq, 0,
4738                     "# of tx queues for offloaded TCP connections");
4739                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
4740                     CTLFLAG_RD, &pi->first_ofld_rxq, 0,
4741                     "index of first TOE rx queue");
4742                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
4743                     CTLFLAG_RD, &pi->first_ofld_txq, 0,
4744                     "index of first TOE tx queue");
4745         }
4746 #endif
4747 #ifdef DEV_NETMAP
4748         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
4749             &pi->nnmrxq, 0, "# of rx queues for netmap");
4750         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
4751             &pi->nnmtxq, 0, "# of tx queues for netmap");
4752         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
4753             CTLFLAG_RD, &pi->first_nm_rxq, 0,
4754             "index of first netmap rx queue");
4755         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
4756             CTLFLAG_RD, &pi->first_nm_txq, 0,
4757             "index of first netmap tx queue");
4758 #endif
4759
4760         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
4761             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
4762             "holdoff timer index");
4763         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
4764             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
4765             "holdoff packet counter index");
4766
4767         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
4768             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
4769             "rx queue size");
4770         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
4771             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
4772             "tx queue size");
4773
4774         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
4775             CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings,
4776             "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
4777
4778         /*
4779          * dev.cxgbe.X.stats.
4780          */
4781         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
4782             NULL, "port statistics");
4783         children = SYSCTL_CHILDREN(oid);
4784
4785 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
4786         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
4787             CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
4788             sysctl_handle_t4_reg64, "QU", desc)
4789
4790         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
4791             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
4792         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
4793             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
4794         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
4795             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
4796         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
4797             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
4798         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
4799             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
4800         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
4801             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
4802         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
4803             "# of tx frames in this range",
4804             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
4805         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
4806             "# of tx frames in this range",
4807             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
4808         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
4809             "# of tx frames in this range",
4810             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
4811         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
4812             "# of tx frames in this range",
4813             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
4814         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
4815             "# of tx frames in this range",
4816             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
4817         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
4818             "# of tx frames in this range",
4819             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
4820         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
4821             "# of tx frames in this range",
4822             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
4823         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
4824             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
4825         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
4826             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
4827         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
4828             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
4829         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
4830             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
4831         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
4832             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
4833         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
4834             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
4835         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
4836             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
4837         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
4838             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
4839         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
4840             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
4841         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
4842             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
4843
4844         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
4845             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
4846         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
4847             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
4848         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
4849             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
4850         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
4851             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
4852         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
4853             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
4854         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
4855             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
4856         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
4857             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
4858         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
4859             "# of frames received with bad FCS",
4860             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
4861         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
4862             "# of frames received with length error",
4863             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
4864         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
4865             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
4866         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
4867             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
4868         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
4869             "# of rx frames in this range",
4870             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
4871         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
4872             "# of rx frames in this range",
4873             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
4874         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
4875             "# of rx frames in this range",
4876             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
4877         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
4878             "# of rx frames in this range",
4879             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
4880         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
4881             "# of rx frames in this range",
4882             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
4883         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
4884             "# of rx frames in this range",
4885             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
4886         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
4887             "# of rx frames in this range",
4888             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
4889         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
4890             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
4891         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
4892             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
4893         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
4894             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
4895         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
4896             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
4897         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
4898             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
4899         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
4900             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
4901         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
4902             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
4903         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
4904             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
4905         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
4906             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
4907
4908 #undef SYSCTL_ADD_T4_REG64
4909
4910 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
4911         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
4912             &pi->stats.name, desc)
4913
4914         /* We get these from port_stats and they may be stale by upto 1s */
4915         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
4916             "# drops due to buffer-group 0 overflows");
4917         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
4918             "# drops due to buffer-group 1 overflows");
4919         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
4920             "# drops due to buffer-group 2 overflows");
4921         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
4922             "# drops due to buffer-group 3 overflows");
4923         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
4924             "# of buffer-group 0 truncated packets");
4925         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
4926             "# of buffer-group 1 truncated packets");
4927         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
4928             "# of buffer-group 2 truncated packets");
4929         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
4930             "# of buffer-group 3 truncated packets");
4931
4932 #undef SYSCTL_ADD_T4_PORTSTAT
4933
4934         return (0);
4935 }
4936
4937 static int
4938 sysctl_int_array(SYSCTL_HANDLER_ARGS)
4939 {
4940         int rc, *i;
4941         struct sbuf sb;
4942
4943         sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
4944         for (i = arg1; arg2; arg2 -= sizeof(int), i++)
4945                 sbuf_printf(&sb, "%d ", *i);
4946         sbuf_trim(&sb);
4947         sbuf_finish(&sb);
4948         rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
4949         sbuf_delete(&sb);
4950         return (rc);
4951 }
4952
4953 static int
4954 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
4955 {
4956         int rc;
4957         struct sbuf *sb;
4958
4959         rc = sysctl_wire_old_buffer(req, 0);
4960         if (rc != 0)
4961                 return(rc);
4962
4963         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
4964         if (sb == NULL)
4965                 return (ENOMEM);
4966
4967         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
4968         rc = sbuf_finish(sb);
4969         sbuf_delete(sb);
4970
4971         return (rc);
4972 }
4973
4974 static int
4975 sysctl_btphy(SYSCTL_HANDLER_ARGS)
4976 {
4977         struct port_info *pi = arg1;
4978         int op = arg2;
4979         struct adapter *sc = pi->adapter;
4980         u_int v;
4981         int rc;
4982
4983         rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt");
4984         if (rc)
4985                 return (rc);
4986         /* XXX: magic numbers */
4987         rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
4988             &v);
4989         end_synchronized_op(sc, 0);
4990         if (rc)
4991                 return (rc);
4992         if (op == 0)
4993                 v /= 256;
4994
4995         rc = sysctl_handle_int(oidp, &v, 0, req);
4996         return (rc);
4997 }
4998
4999 static int
5000 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
5001 {
5002         struct port_info *pi = arg1;
5003         int rc, val;
5004
5005         val = pi->rsrv_noflowq;
5006         rc = sysctl_handle_int(oidp, &val, 0, req);
5007         if (rc != 0 || req->newptr == NULL)
5008                 return (rc);
5009
5010         if ((val >= 1) && (pi->ntxq > 1))
5011                 pi->rsrv_noflowq = 1;
5012         else
5013                 pi->rsrv_noflowq = 0;
5014
5015         return (rc);
5016 }
5017
5018 static int
5019 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
5020 {
5021         struct port_info *pi = arg1;
5022         struct adapter *sc = pi->adapter;
5023         int idx, rc, i;
5024         struct sge_rxq *rxq;
5025 #ifdef TCP_OFFLOAD
5026         struct sge_ofld_rxq *ofld_rxq;
5027 #endif
5028         uint8_t v;
5029
5030         idx = pi->tmr_idx;
5031
5032         rc = sysctl_handle_int(oidp, &idx, 0, req);
5033         if (rc != 0 || req->newptr == NULL)
5034                 return (rc);
5035
5036         if (idx < 0 || idx >= SGE_NTIMERS)
5037                 return (EINVAL);
5038
5039         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5040             "t4tmr");
5041         if (rc)
5042                 return (rc);
5043
5044         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
5045         for_each_rxq(pi, i, rxq) {
5046 #ifdef atomic_store_rel_8
5047                 atomic_store_rel_8(&rxq->iq.intr_params, v);
5048 #else
5049                 rxq->iq.intr_params = v;
5050 #endif
5051         }
5052 #ifdef TCP_OFFLOAD
5053         for_each_ofld_rxq(pi, i, ofld_rxq) {
5054 #ifdef atomic_store_rel_8
5055                 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
5056 #else
5057                 ofld_rxq->iq.intr_params = v;
5058 #endif
5059         }
5060 #endif
5061         pi->tmr_idx = idx;
5062
5063         end_synchronized_op(sc, LOCK_HELD);
5064         return (0);
5065 }
5066
5067 static int
5068 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
5069 {
5070         struct port_info *pi = arg1;
5071         struct adapter *sc = pi->adapter;
5072         int idx, rc;
5073
5074         idx = pi->pktc_idx;
5075
5076         rc = sysctl_handle_int(oidp, &idx, 0, req);
5077         if (rc != 0 || req->newptr == NULL)
5078                 return (rc);
5079
5080         if (idx < -1 || idx >= SGE_NCOUNTERS)
5081                 return (EINVAL);
5082
5083         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5084             "t4pktc");
5085         if (rc)
5086                 return (rc);
5087
5088         if (pi->flags & PORT_INIT_DONE)
5089                 rc = EBUSY; /* cannot be changed once the queues are created */
5090         else
5091                 pi->pktc_idx = idx;
5092
5093         end_synchronized_op(sc, LOCK_HELD);
5094         return (rc);
5095 }
5096
5097 static int
5098 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
5099 {
5100         struct port_info *pi = arg1;
5101         struct adapter *sc = pi->adapter;
5102         int qsize, rc;
5103
5104         qsize = pi->qsize_rxq;
5105
5106         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5107         if (rc != 0 || req->newptr == NULL)
5108                 return (rc);
5109
5110         if (qsize < 128 || (qsize & 7))
5111                 return (EINVAL);
5112
5113         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5114             "t4rxqs");
5115         if (rc)
5116                 return (rc);
5117
5118         if (pi->flags & PORT_INIT_DONE)
5119                 rc = EBUSY; /* cannot be changed once the queues are created */
5120         else
5121                 pi->qsize_rxq = qsize;
5122
5123         end_synchronized_op(sc, LOCK_HELD);
5124         return (rc);
5125 }
5126
5127 static int
5128 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
5129 {
5130         struct port_info *pi = arg1;
5131         struct adapter *sc = pi->adapter;
5132         int qsize, rc;
5133
5134         qsize = pi->qsize_txq;
5135
5136         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5137         if (rc != 0 || req->newptr == NULL)
5138                 return (rc);
5139
5140         /* bufring size must be powerof2 */
5141         if (qsize < 128 || !powerof2(qsize))
5142                 return (EINVAL);
5143
5144         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5145             "t4txqs");
5146         if (rc)
5147                 return (rc);
5148
5149         if (pi->flags & PORT_INIT_DONE)
5150                 rc = EBUSY; /* cannot be changed once the queues are created */
5151         else
5152                 pi->qsize_txq = qsize;
5153
5154         end_synchronized_op(sc, LOCK_HELD);
5155         return (rc);
5156 }
5157
5158 static int
5159 sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
5160 {
5161         struct port_info *pi = arg1;
5162         struct adapter *sc = pi->adapter;
5163         struct link_config *lc = &pi->link_cfg;
5164         int rc;
5165
5166         if (req->newptr == NULL) {
5167                 struct sbuf *sb;
5168                 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX";
5169
5170                 rc = sysctl_wire_old_buffer(req, 0);
5171                 if (rc != 0)
5172                         return(rc);
5173
5174                 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5175                 if (sb == NULL)
5176                         return (ENOMEM);
5177
5178                 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits);
5179                 rc = sbuf_finish(sb);
5180                 sbuf_delete(sb);
5181         } else {
5182                 char s[2];
5183                 int n;
5184
5185                 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX));
5186                 s[1] = 0;
5187
5188                 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
5189                 if (rc != 0)
5190                         return(rc);
5191
5192                 if (s[1] != 0)
5193                         return (EINVAL);
5194                 if (s[0] < '0' || s[0] > '9')
5195                         return (EINVAL);        /* not a number */
5196                 n = s[0] - '0';
5197                 if (n & ~(PAUSE_TX | PAUSE_RX))
5198                         return (EINVAL);        /* some other bit is set too */
5199
5200                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE");
5201                 if (rc)
5202                         return (rc);
5203                 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) {
5204                         int link_ok = lc->link_ok;
5205
5206                         lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX);
5207                         lc->requested_fc |= n;
5208                         rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc);
5209                         lc->link_ok = link_ok;  /* restore */
5210                 }
5211                 end_synchronized_op(sc, 0);
5212         }
5213
5214         return (rc);
5215 }
5216
5217 static int
5218 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
5219 {
5220         struct adapter *sc = arg1;
5221         int reg = arg2;
5222         uint64_t val;
5223
5224         val = t4_read_reg64(sc, reg);
5225
5226         return (sysctl_handle_64(oidp, &val, 0, req));
5227 }
5228
5229 static int
5230 sysctl_temperature(SYSCTL_HANDLER_ARGS)
5231 {
5232         struct adapter *sc = arg1;
5233         int rc, t;
5234         uint32_t param, val;
5235
5236         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
5237         if (rc)
5238                 return (rc);
5239         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5240             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5241             V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
5242         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
5243         end_synchronized_op(sc, 0);
5244         if (rc)
5245                 return (rc);
5246
5247         /* unknown is returned as 0 but we display -1 in that case */
5248         t = val == 0 ? -1 : val;
5249
5250         rc = sysctl_handle_int(oidp, &t, 0, req);
5251         return (rc);
5252 }
5253
5254 #ifdef SBUF_DRAIN
5255 static int
5256 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
5257 {
5258         struct adapter *sc = arg1;
5259         struct sbuf *sb;
5260         int rc, i;
5261         uint16_t incr[NMTUS][NCCTRL_WIN];
5262         static const char *dec_fac[] = {
5263                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
5264                 "0.9375"
5265         };
5266
5267         rc = sysctl_wire_old_buffer(req, 0);
5268         if (rc != 0)
5269                 return (rc);
5270
5271         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5272         if (sb == NULL)
5273                 return (ENOMEM);
5274
5275         t4_read_cong_tbl(sc, incr);
5276
5277         for (i = 0; i < NCCTRL_WIN; ++i) {
5278                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
5279                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
5280                     incr[5][i], incr[6][i], incr[7][i]);
5281                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
5282                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
5283                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
5284                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
5285         }
5286
5287         rc = sbuf_finish(sb);
5288         sbuf_delete(sb);
5289
5290         return (rc);
5291 }
5292
5293 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
5294         "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",   /* ibq's */
5295         "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
5296         "SGE0-RX", "SGE1-RX"    /* additional obq's (T5 onwards) */
5297 };
5298
5299 static int
5300 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
5301 {
5302         struct adapter *sc = arg1;
5303         struct sbuf *sb;
5304         int rc, i, n, qid = arg2;
5305         uint32_t *buf, *p;
5306         char *qtype;
5307         u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
5308
5309         KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
5310             ("%s: bad qid %d\n", __func__, qid));
5311
5312         if (qid < CIM_NUM_IBQ) {
5313                 /* inbound queue */
5314                 qtype = "IBQ";
5315                 n = 4 * CIM_IBQ_SIZE;
5316                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5317                 rc = t4_read_cim_ibq(sc, qid, buf, n);
5318         } else {
5319                 /* outbound queue */
5320                 qtype = "OBQ";
5321                 qid -= CIM_NUM_IBQ;
5322                 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
5323                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5324                 rc = t4_read_cim_obq(sc, qid, buf, n);
5325         }
5326
5327         if (rc < 0) {
5328                 rc = -rc;
5329                 goto done;
5330         }
5331         n = rc * sizeof(uint32_t);      /* rc has # of words actually read */
5332
5333         rc = sysctl_wire_old_buffer(req, 0);
5334         if (rc != 0)
5335                 goto done;
5336
5337         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5338         if (sb == NULL) {
5339                 rc = ENOMEM;
5340                 goto done;
5341         }
5342
5343         sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
5344         for (i = 0, p = buf; i < n; i += 16, p += 4)
5345                 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
5346                     p[2], p[3]);
5347
5348         rc = sbuf_finish(sb);
5349         sbuf_delete(sb);
5350 done:
5351         free(buf, M_CXGBE);
5352         return (rc);
5353 }
5354
5355 static int
5356 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
5357 {
5358         struct adapter *sc = arg1;
5359         u_int cfg;
5360         struct sbuf *sb;
5361         uint32_t *buf, *p;
5362         int rc;
5363
5364         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
5365         if (rc != 0)
5366                 return (rc);
5367
5368         rc = sysctl_wire_old_buffer(req, 0);
5369         if (rc != 0)
5370                 return (rc);
5371
5372         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5373         if (sb == NULL)
5374                 return (ENOMEM);
5375
5376         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
5377             M_ZERO | M_WAITOK);
5378
5379         rc = -t4_cim_read_la(sc, buf, NULL);
5380         if (rc != 0)
5381                 goto done;
5382
5383         sbuf_printf(sb, "Status   Data      PC%s",
5384             cfg & F_UPDBGLACAPTPCONLY ? "" :
5385             "     LS0Stat  LS0Addr             LS0Data");
5386
5387         KASSERT((sc->params.cim_la_size & 7) == 0,
5388             ("%s: p will walk off the end of buf", __func__));
5389
5390         for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
5391                 if (cfg & F_UPDBGLACAPTPCONLY) {
5392                         sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
5393                             p[6], p[7]);
5394                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
5395                             (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
5396                             p[4] & 0xff, p[5] >> 8);
5397                         sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
5398                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5399                             p[1] & 0xf, p[2] >> 4);
5400                 } else {
5401                         sbuf_printf(sb,
5402                             "\n  %02x   %x%07x %x%07x %08x %08x "
5403                             "%08x%08x%08x%08x",
5404                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5405                             p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
5406                             p[6], p[7]);
5407                 }
5408         }
5409
5410         rc = sbuf_finish(sb);
5411         sbuf_delete(sb);
5412 done:
5413         free(buf, M_CXGBE);
5414         return (rc);
5415 }
5416
5417 static int
5418 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
5419 {
5420         struct adapter *sc = arg1;
5421         u_int i;
5422         struct sbuf *sb;
5423         uint32_t *buf, *p;
5424         int rc;
5425
5426         rc = sysctl_wire_old_buffer(req, 0);
5427         if (rc != 0)
5428                 return (rc);
5429
5430         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5431         if (sb == NULL)
5432                 return (ENOMEM);
5433
5434         buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
5435             M_ZERO | M_WAITOK);
5436
5437         t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
5438         p = buf;
5439
5440         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5441                 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
5442                     p[1], p[0]);
5443         }
5444
5445         sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
5446         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5447                 sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
5448                     (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
5449                     (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
5450                     (p[1] >> 2) | ((p[2] & 3) << 30),
5451                     (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
5452                     p[0] & 1);
5453         }
5454
5455         rc = sbuf_finish(sb);
5456         sbuf_delete(sb);
5457         free(buf, M_CXGBE);
5458         return (rc);
5459 }
5460
5461 static int
5462 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
5463 {
5464         struct adapter *sc = arg1;
5465         u_int i;
5466         struct sbuf *sb;
5467         uint32_t *buf, *p;
5468         int rc;
5469
5470         rc = sysctl_wire_old_buffer(req, 0);
5471         if (rc != 0)
5472                 return (rc);
5473
5474         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5475         if (sb == NULL)
5476                 return (ENOMEM);
5477
5478         buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
5479             M_ZERO | M_WAITOK);
5480
5481         t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
5482         p = buf;
5483
5484         sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
5485         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5486                 sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
5487                     (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
5488                     p[4], p[3], p[2], p[1], p[0]);
5489         }
5490
5491         sbuf_printf(sb, "\n\nCntl ID               Data");
5492         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5493                 sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
5494                     (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
5495         }
5496
5497         rc = sbuf_finish(sb);
5498         sbuf_delete(sb);
5499         free(buf, M_CXGBE);
5500         return (rc);
5501 }
5502
5503 static int
5504 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
5505 {
5506         struct adapter *sc = arg1;
5507         struct sbuf *sb;
5508         int rc, i;
5509         uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5510         uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5511         uint16_t thres[CIM_NUM_IBQ];
5512         uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
5513         uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
5514         u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
5515
5516         if (is_t4(sc)) {
5517                 cim_num_obq = CIM_NUM_OBQ;
5518                 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
5519                 obq_rdaddr = A_UP_OBQ_0_REALADDR;
5520         } else {
5521                 cim_num_obq = CIM_NUM_OBQ_T5;
5522                 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
5523                 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
5524         }
5525         nq = CIM_NUM_IBQ + cim_num_obq;
5526
5527         rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
5528         if (rc == 0)
5529                 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
5530         if (rc != 0)
5531                 return (rc);
5532
5533         t4_read_cimq_cfg(sc, base, size, thres);
5534
5535         rc = sysctl_wire_old_buffer(req, 0);
5536         if (rc != 0)
5537                 return (rc);
5538
5539         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5540         if (sb == NULL)
5541                 return (ENOMEM);
5542
5543         sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
5544
5545         for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
5546                 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
5547                     qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
5548                     G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5549                     G_QUEREMFLITS(p[2]) * 16);
5550         for ( ; i < nq; i++, p += 4, wr += 2)
5551                 sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
5552                     base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
5553                     wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5554                     G_QUEREMFLITS(p[2]) * 16);
5555
5556         rc = sbuf_finish(sb);
5557         sbuf_delete(sb);
5558
5559         return (rc);
5560 }
5561
5562 static int
5563 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
5564 {
5565         struct adapter *sc = arg1;
5566         struct sbuf *sb;
5567         int rc;
5568         struct tp_cpl_stats stats;
5569
5570         rc = sysctl_wire_old_buffer(req, 0);
5571         if (rc != 0)
5572                 return (rc);
5573
5574         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5575         if (sb == NULL)
5576                 return (ENOMEM);
5577
5578         t4_tp_get_cpl_stats(sc, &stats);
5579
5580         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
5581             "channel 3\n");
5582         sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
5583                    stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
5584         sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
5585                    stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
5586
5587         rc = sbuf_finish(sb);
5588         sbuf_delete(sb);
5589
5590         return (rc);
5591 }
5592
5593 static int
5594 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
5595 {
5596         struct adapter *sc = arg1;
5597         struct sbuf *sb;
5598         int rc;
5599         struct tp_usm_stats stats;
5600
5601         rc = sysctl_wire_old_buffer(req, 0);
5602         if (rc != 0)
5603                 return(rc);
5604
5605         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5606         if (sb == NULL)
5607                 return (ENOMEM);
5608
5609         t4_get_usm_stats(sc, &stats);
5610
5611         sbuf_printf(sb, "Frames: %u\n", stats.frames);
5612         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
5613         sbuf_printf(sb, "Drops:  %u", stats.drops);
5614
5615         rc = sbuf_finish(sb);
5616         sbuf_delete(sb);
5617
5618         return (rc);
5619 }
5620
5621 const char *devlog_level_strings[] = {
5622         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
5623         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
5624         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
5625         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
5626         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
5627         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
5628 };
5629
5630 const char *devlog_facility_strings[] = {
5631         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
5632         [FW_DEVLOG_FACILITY_CF]         = "CF",
5633         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
5634         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
5635         [FW_DEVLOG_FACILITY_RES]        = "RES",
5636         [FW_DEVLOG_FACILITY_HW]         = "HW",
5637         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
5638         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
5639         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
5640         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
5641         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
5642         [FW_DEVLOG_FACILITY_VI]         = "VI",
5643         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
5644         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
5645         [FW_DEVLOG_FACILITY_TM]         = "TM",
5646         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
5647         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
5648         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
5649         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
5650         [FW_DEVLOG_FACILITY_RI]         = "RI",
5651         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
5652         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
5653         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
5654         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE"
5655 };
5656
5657 static int
5658 sysctl_devlog(SYSCTL_HANDLER_ARGS)
5659 {
5660         struct adapter *sc = arg1;
5661         struct devlog_params *dparams = &sc->params.devlog;
5662         struct fw_devlog_e *buf, *e;
5663         int i, j, rc, nentries, first = 0, m;
5664         struct sbuf *sb;
5665         uint64_t ftstamp = UINT64_MAX;
5666
5667         if (dparams->start == 0) {
5668                 dparams->memtype = FW_MEMTYPE_EDC0;
5669                 dparams->start = 0x84000;
5670                 dparams->size = 32768;
5671         }
5672
5673         nentries = dparams->size / sizeof(struct fw_devlog_e);
5674
5675         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
5676         if (buf == NULL)
5677                 return (ENOMEM);
5678
5679         m = fwmtype_to_hwmtype(dparams->memtype);
5680         rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
5681         if (rc != 0)
5682                 goto done;
5683
5684         for (i = 0; i < nentries; i++) {
5685                 e = &buf[i];
5686
5687                 if (e->timestamp == 0)
5688                         break;  /* end */
5689
5690                 e->timestamp = be64toh(e->timestamp);
5691                 e->seqno = be32toh(e->seqno);
5692                 for (j = 0; j < 8; j++)
5693                         e->params[j] = be32toh(e->params[j]);
5694
5695                 if (e->timestamp < ftstamp) {
5696                         ftstamp = e->timestamp;
5697                         first = i;
5698                 }
5699         }
5700
5701         if (buf[first].timestamp == 0)
5702                 goto done;      /* nothing in the log */
5703
5704         rc = sysctl_wire_old_buffer(req, 0);
5705         if (rc != 0)
5706                 goto done;
5707
5708         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5709         if (sb == NULL) {
5710                 rc = ENOMEM;
5711                 goto done;
5712         }
5713         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
5714             "Seq#", "Tstamp", "Level", "Facility", "Message");
5715
5716         i = first;
5717         do {
5718                 e = &buf[i];
5719                 if (e->timestamp == 0)
5720                         break;  /* end */
5721
5722                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
5723                     e->seqno, e->timestamp,
5724                     (e->level < nitems(devlog_level_strings) ?
5725                         devlog_level_strings[e->level] : "UNKNOWN"),
5726                     (e->facility < nitems(devlog_facility_strings) ?
5727                         devlog_facility_strings[e->facility] : "UNKNOWN"));
5728                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
5729                     e->params[2], e->params[3], e->params[4],
5730                     e->params[5], e->params[6], e->params[7]);
5731
5732                 if (++i == nentries)
5733                         i = 0;
5734         } while (i != first);
5735
5736         rc = sbuf_finish(sb);
5737         sbuf_delete(sb);
5738 done:
5739         free(buf, M_CXGBE);
5740         return (rc);
5741 }
5742
5743 static int
5744 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
5745 {
5746         struct adapter *sc = arg1;
5747         struct sbuf *sb;
5748         int rc;
5749         struct tp_fcoe_stats stats[4];
5750
5751         rc = sysctl_wire_old_buffer(req, 0);
5752         if (rc != 0)
5753                 return (rc);
5754
5755         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5756         if (sb == NULL)
5757                 return (ENOMEM);
5758
5759         t4_get_fcoe_stats(sc, 0, &stats[0]);
5760         t4_get_fcoe_stats(sc, 1, &stats[1]);
5761         t4_get_fcoe_stats(sc, 2, &stats[2]);
5762         t4_get_fcoe_stats(sc, 3, &stats[3]);
5763
5764         sbuf_printf(sb, "                   channel 0        channel 1        "
5765             "channel 2        channel 3\n");
5766         sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
5767             stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
5768             stats[3].octetsDDP);
5769         sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
5770             stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
5771         sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
5772             stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
5773             stats[3].framesDrop);
5774
5775         rc = sbuf_finish(sb);
5776         sbuf_delete(sb);
5777
5778         return (rc);
5779 }
5780
5781 static int
5782 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
5783 {
5784         struct adapter *sc = arg1;
5785         struct sbuf *sb;
5786         int rc, i;
5787         unsigned int map, kbps, ipg, mode;
5788         unsigned int pace_tab[NTX_SCHED];
5789
5790         rc = sysctl_wire_old_buffer(req, 0);
5791         if (rc != 0)
5792                 return (rc);
5793
5794         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5795         if (sb == NULL)
5796                 return (ENOMEM);
5797
5798         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
5799         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
5800         t4_read_pace_tbl(sc, pace_tab);
5801
5802         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
5803             "Class IPG (0.1 ns)   Flow IPG (us)");
5804
5805         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
5806                 t4_get_tx_sched(sc, i, &kbps, &ipg);
5807                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
5808                     (mode & (1 << i)) ? "flow" : "class", map & 3);
5809                 if (kbps)
5810                         sbuf_printf(sb, "%9u     ", kbps);
5811                 else
5812                         sbuf_printf(sb, " disabled     ");
5813
5814                 if (ipg)
5815                         sbuf_printf(sb, "%13u        ", ipg);
5816                 else
5817                         sbuf_printf(sb, "     disabled        ");
5818
5819                 if (pace_tab[i])
5820                         sbuf_printf(sb, "%10u", pace_tab[i]);
5821                 else
5822                         sbuf_printf(sb, "  disabled");
5823         }
5824
5825         rc = sbuf_finish(sb);
5826         sbuf_delete(sb);
5827
5828         return (rc);
5829 }
5830
5831 static int
5832 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
5833 {
5834         struct adapter *sc = arg1;
5835         struct sbuf *sb;
5836         int rc, i, j;
5837         uint64_t *p0, *p1;
5838         struct lb_port_stats s[2];
5839         static const char *stat_name[] = {
5840                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
5841                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
5842                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
5843                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
5844                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
5845                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
5846                 "BG2FramesTrunc:", "BG3FramesTrunc:"
5847         };
5848
5849         rc = sysctl_wire_old_buffer(req, 0);
5850         if (rc != 0)
5851                 return (rc);
5852
5853         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5854         if (sb == NULL)
5855                 return (ENOMEM);
5856
5857         memset(s, 0, sizeof(s));
5858
5859         for (i = 0; i < 4; i += 2) {
5860                 t4_get_lb_stats(sc, i, &s[0]);
5861                 t4_get_lb_stats(sc, i + 1, &s[1]);
5862
5863                 p0 = &s[0].octets;
5864                 p1 = &s[1].octets;
5865                 sbuf_printf(sb, "%s                       Loopback %u"
5866                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
5867
5868                 for (j = 0; j < nitems(stat_name); j++)
5869                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
5870                                    *p0++, *p1++);
5871         }
5872
5873         rc = sbuf_finish(sb);
5874         sbuf_delete(sb);
5875
5876         return (rc);
5877 }
5878
5879 static int
5880 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
5881 {
5882         int rc = 0;
5883         struct port_info *pi = arg1;
5884         struct sbuf *sb;
5885         static const char *linkdnreasons[] = {
5886                 "non-specific", "remote fault", "autoneg failed", "reserved3",
5887                 "PHY overheated", "unknown", "rx los", "reserved7"
5888         };
5889
5890         rc = sysctl_wire_old_buffer(req, 0);
5891         if (rc != 0)
5892                 return(rc);
5893         sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
5894         if (sb == NULL)
5895                 return (ENOMEM);
5896
5897         if (pi->linkdnrc < 0)
5898                 sbuf_printf(sb, "n/a");
5899         else if (pi->linkdnrc < nitems(linkdnreasons))
5900                 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
5901         else
5902                 sbuf_printf(sb, "%d", pi->linkdnrc);
5903
5904         rc = sbuf_finish(sb);
5905         sbuf_delete(sb);
5906
5907         return (rc);
5908 }
5909
5910 struct mem_desc {
5911         unsigned int base;
5912         unsigned int limit;
5913         unsigned int idx;
5914 };
5915
5916 static int
5917 mem_desc_cmp(const void *a, const void *b)
5918 {
5919         return ((const struct mem_desc *)a)->base -
5920                ((const struct mem_desc *)b)->base;
5921 }
5922
5923 static void
5924 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
5925     unsigned int to)
5926 {
5927         unsigned int size;
5928
5929         size = to - from + 1;
5930         if (size == 0)
5931                 return;
5932
5933         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
5934         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
5935 }
5936
5937 static int
5938 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
5939 {
5940         struct adapter *sc = arg1;
5941         struct sbuf *sb;
5942         int rc, i, n;
5943         uint32_t lo, hi, used, alloc;
5944         static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
5945         static const char *region[] = {
5946                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
5947                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
5948                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
5949                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
5950                 "RQUDP region:", "PBL region:", "TXPBL region:",
5951                 "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
5952                 "On-chip queues:"
5953         };
5954         struct mem_desc avail[4];
5955         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
5956         struct mem_desc *md = mem;
5957
5958         rc = sysctl_wire_old_buffer(req, 0);
5959         if (rc != 0)
5960                 return (rc);
5961
5962         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5963         if (sb == NULL)
5964                 return (ENOMEM);
5965
5966         for (i = 0; i < nitems(mem); i++) {
5967                 mem[i].limit = 0;
5968                 mem[i].idx = i;
5969         }
5970
5971         /* Find and sort the populated memory ranges */
5972         i = 0;
5973         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
5974         if (lo & F_EDRAM0_ENABLE) {
5975                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
5976                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
5977                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
5978                 avail[i].idx = 0;
5979                 i++;
5980         }
5981         if (lo & F_EDRAM1_ENABLE) {
5982                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
5983                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
5984                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
5985                 avail[i].idx = 1;
5986                 i++;
5987         }
5988         if (lo & F_EXT_MEM_ENABLE) {
5989                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
5990                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
5991                 avail[i].limit = avail[i].base +
5992                     (G_EXT_MEM_SIZE(hi) << 20);
5993                 avail[i].idx = is_t4(sc) ? 2 : 3;       /* Call it MC for T4 */
5994                 i++;
5995         }
5996         if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
5997                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
5998                 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
5999                 avail[i].limit = avail[i].base +
6000                     (G_EXT_MEM1_SIZE(hi) << 20);
6001                 avail[i].idx = 4;
6002                 i++;
6003         }
6004         if (!i)                                    /* no memory available */
6005                 return 0;
6006         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
6007
6008         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
6009         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
6010         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
6011         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
6012         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
6013         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
6014         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
6015         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
6016         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
6017
6018         /* the next few have explicit upper bounds */
6019         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
6020         md->limit = md->base - 1 +
6021                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
6022                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
6023         md++;
6024
6025         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
6026         md->limit = md->base - 1 +
6027                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
6028                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
6029         md++;
6030
6031         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6032                 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
6033                 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
6034                 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
6035         } else {
6036                 md->base = 0;
6037                 md->idx = nitems(region);  /* hide it */
6038         }
6039         md++;
6040
6041 #define ulp_region(reg) \
6042         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
6043         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
6044
6045         ulp_region(RX_ISCSI);
6046         ulp_region(RX_TDDP);
6047         ulp_region(TX_TPT);
6048         ulp_region(RX_STAG);
6049         ulp_region(RX_RQ);
6050         ulp_region(RX_RQUDP);
6051         ulp_region(RX_PBL);
6052         ulp_region(TX_PBL);
6053 #undef ulp_region
6054
6055         md->base = 0;
6056         md->idx = nitems(region);
6057         if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
6058                 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
6059                 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
6060                     A_SGE_DBVFIFO_SIZE))) << 2) - 1;
6061         }
6062         md++;
6063
6064         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
6065         md->limit = md->base + sc->tids.ntids - 1;
6066         md++;
6067         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
6068         md->limit = md->base + sc->tids.ntids - 1;
6069         md++;
6070
6071         md->base = sc->vres.ocq.start;
6072         if (sc->vres.ocq.size)
6073                 md->limit = md->base + sc->vres.ocq.size - 1;
6074         else
6075                 md->idx = nitems(region);  /* hide it */
6076         md++;
6077
6078         /* add any address-space holes, there can be up to 3 */
6079         for (n = 0; n < i - 1; n++)
6080                 if (avail[n].limit < avail[n + 1].base)
6081                         (md++)->base = avail[n].limit;
6082         if (avail[n].limit)
6083                 (md++)->base = avail[n].limit;
6084
6085         n = md - mem;
6086         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
6087
6088         for (lo = 0; lo < i; lo++)
6089                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
6090                                 avail[lo].limit - 1);
6091
6092         sbuf_printf(sb, "\n");
6093         for (i = 0; i < n; i++) {
6094                 if (mem[i].idx >= nitems(region))
6095                         continue;                        /* skip holes */
6096                 if (!mem[i].limit)
6097                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
6098                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
6099                                 mem[i].limit);
6100         }
6101
6102         sbuf_printf(sb, "\n");
6103         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
6104         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
6105         mem_region_show(sb, "uP RAM:", lo, hi);
6106
6107         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
6108         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
6109         mem_region_show(sb, "uP Extmem2:", lo, hi);
6110
6111         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
6112         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
6113                    G_PMRXMAXPAGE(lo),
6114                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
6115                    (lo & F_PMRXNUMCHN) ? 2 : 1);
6116
6117         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
6118         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
6119         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
6120                    G_PMTXMAXPAGE(lo),
6121                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
6122                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
6123         sbuf_printf(sb, "%u p-structs\n",
6124                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
6125
6126         for (i = 0; i < 4; i++) {
6127                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
6128                 if (is_t4(sc)) {
6129                         used = G_USED(lo);
6130                         alloc = G_ALLOC(lo);
6131                 } else {
6132                         used = G_T5_USED(lo);
6133                         alloc = G_T5_ALLOC(lo);
6134                 }
6135                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
6136                            i, used, alloc);
6137         }
6138         for (i = 0; i < 4; i++) {
6139                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
6140                 if (is_t4(sc)) {
6141                         used = G_USED(lo);
6142                         alloc = G_ALLOC(lo);
6143                 } else {
6144                         used = G_T5_USED(lo);
6145                         alloc = G_T5_ALLOC(lo);
6146                 }
6147                 sbuf_printf(sb,
6148                            "\nLoopback %d using %u pages out of %u allocated",
6149                            i, used, alloc);
6150         }
6151
6152         rc = sbuf_finish(sb);
6153         sbuf_delete(sb);
6154
6155         return (rc);
6156 }
6157
6158 static inline void
6159 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
6160 {
6161         *mask = x | y;
6162         y = htobe64(y);
6163         memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
6164 }
6165
6166 static int
6167 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
6168 {
6169         struct adapter *sc = arg1;
6170         struct sbuf *sb;
6171         int rc, i, n;
6172
6173         rc = sysctl_wire_old_buffer(req, 0);
6174         if (rc != 0)
6175                 return (rc);
6176
6177         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6178         if (sb == NULL)
6179                 return (ENOMEM);
6180
6181         sbuf_printf(sb,
6182             "Idx  Ethernet address     Mask     Vld Ports PF"
6183             "  VF              Replication             P0 P1 P2 P3  ML");
6184         n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
6185             NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
6186         for (i = 0; i < n; i++) {
6187                 uint64_t tcamx, tcamy, mask;
6188                 uint32_t cls_lo, cls_hi;
6189                 uint8_t addr[ETHER_ADDR_LEN];
6190
6191                 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
6192                 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
6193                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
6194                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
6195
6196                 if (tcamx & tcamy)
6197                         continue;
6198
6199                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
6200                 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
6201                            "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
6202                            addr[3], addr[4], addr[5], (uintmax_t)mask,
6203                            (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
6204                            G_PORTMAP(cls_hi), G_PF(cls_lo),
6205                            (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
6206
6207                 if (cls_lo & F_REPLICATE) {
6208                         struct fw_ldst_cmd ldst_cmd;
6209
6210                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
6211                         ldst_cmd.op_to_addrspace =
6212                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
6213                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
6214                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
6215                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
6216                         ldst_cmd.u.mps.fid_ctl =
6217                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
6218                                 V_FW_LDST_CMD_CTL(i));
6219
6220                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
6221                             "t4mps");
6222                         if (rc)
6223                                 break;
6224                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
6225                             sizeof(ldst_cmd), &ldst_cmd);
6226                         end_synchronized_op(sc, 0);
6227
6228                         if (rc != 0) {
6229                                 sbuf_printf(sb,
6230                                     " ------------ error %3u ------------", rc);
6231                                 rc = 0;
6232                         } else {
6233                                 sbuf_printf(sb, " %08x %08x %08x %08x",
6234                                     be32toh(ldst_cmd.u.mps.rplc127_96),
6235                                     be32toh(ldst_cmd.u.mps.rplc95_64),
6236                                     be32toh(ldst_cmd.u.mps.rplc63_32),
6237                                     be32toh(ldst_cmd.u.mps.rplc31_0));
6238                         }
6239                 } else
6240                         sbuf_printf(sb, "%36s", "");
6241
6242                 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
6243                     G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
6244                     G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
6245         }
6246
6247         if (rc)
6248                 (void) sbuf_finish(sb);
6249         else
6250                 rc = sbuf_finish(sb);
6251         sbuf_delete(sb);
6252
6253         return (rc);
6254 }
6255
6256 static int
6257 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
6258 {
6259         struct adapter *sc = arg1;
6260         struct sbuf *sb;
6261         int rc;
6262         uint16_t mtus[NMTUS];
6263
6264         rc = sysctl_wire_old_buffer(req, 0);
6265         if (rc != 0)
6266                 return (rc);
6267
6268         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6269         if (sb == NULL)
6270                 return (ENOMEM);
6271
6272         t4_read_mtu_tbl(sc, mtus, NULL);
6273
6274         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
6275             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
6276             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
6277             mtus[14], mtus[15]);
6278
6279         rc = sbuf_finish(sb);
6280         sbuf_delete(sb);
6281
6282         return (rc);
6283 }
6284
6285 static int
6286 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
6287 {
6288         struct adapter *sc = arg1;
6289         struct sbuf *sb;
6290         int rc, i;
6291         uint32_t cnt[PM_NSTATS];
6292         uint64_t cyc[PM_NSTATS];
6293         static const char *rx_stats[] = {
6294                 "Read:", "Write bypass:", "Write mem:", "Flush:"
6295         };
6296         static const char *tx_stats[] = {
6297                 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
6298         };
6299
6300         rc = sysctl_wire_old_buffer(req, 0);
6301         if (rc != 0)
6302                 return (rc);
6303
6304         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6305         if (sb == NULL)
6306                 return (ENOMEM);
6307
6308         t4_pmtx_get_stats(sc, cnt, cyc);
6309         sbuf_printf(sb, "                Tx pcmds             Tx bytes");
6310         for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
6311                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
6312                     cyc[i]);
6313
6314         t4_pmrx_get_stats(sc, cnt, cyc);
6315         sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
6316         for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
6317                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
6318                     cyc[i]);
6319
6320         rc = sbuf_finish(sb);
6321         sbuf_delete(sb);
6322
6323         return (rc);
6324 }
6325
6326 static int
6327 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
6328 {
6329         struct adapter *sc = arg1;
6330         struct sbuf *sb;
6331         int rc;
6332         struct tp_rdma_stats stats;
6333
6334         rc = sysctl_wire_old_buffer(req, 0);
6335         if (rc != 0)
6336                 return (rc);
6337
6338         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6339         if (sb == NULL)
6340                 return (ENOMEM);
6341
6342         t4_tp_get_rdma_stats(sc, &stats);
6343         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
6344         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
6345
6346         rc = sbuf_finish(sb);
6347         sbuf_delete(sb);
6348
6349         return (rc);
6350 }
6351
6352 static int
6353 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
6354 {
6355         struct adapter *sc = arg1;
6356         struct sbuf *sb;
6357         int rc;
6358         struct tp_tcp_stats v4, v6;
6359
6360         rc = sysctl_wire_old_buffer(req, 0);
6361         if (rc != 0)
6362                 return (rc);
6363
6364         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6365         if (sb == NULL)
6366                 return (ENOMEM);
6367
6368         t4_tp_get_tcp_stats(sc, &v4, &v6);
6369         sbuf_printf(sb,
6370             "                                IP                 IPv6\n");
6371         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
6372             v4.tcpOutRsts, v6.tcpOutRsts);
6373         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
6374             v4.tcpInSegs, v6.tcpInSegs);
6375         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
6376             v4.tcpOutSegs, v6.tcpOutSegs);
6377         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
6378             v4.tcpRetransSegs, v6.tcpRetransSegs);
6379
6380         rc = sbuf_finish(sb);
6381         sbuf_delete(sb);
6382
6383         return (rc);
6384 }
6385
6386 static int
6387 sysctl_tids(SYSCTL_HANDLER_ARGS)
6388 {
6389         struct adapter *sc = arg1;
6390         struct sbuf *sb;
6391         int rc;
6392         struct tid_info *t = &sc->tids;
6393
6394         rc = sysctl_wire_old_buffer(req, 0);
6395         if (rc != 0)
6396                 return (rc);
6397
6398         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6399         if (sb == NULL)
6400                 return (ENOMEM);
6401
6402         if (t->natids) {
6403                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
6404                     t->atids_in_use);
6405         }
6406
6407         if (t->ntids) {
6408                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6409                         uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
6410
6411                         if (b) {
6412                                 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
6413                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6414                                     t->ntids - 1);
6415                         } else {
6416                                 sbuf_printf(sb, "TID range: %u-%u",
6417                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6418                                     t->ntids - 1);
6419                         }
6420                 } else
6421                         sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
6422                 sbuf_printf(sb, ", in use: %u\n",
6423                     atomic_load_acq_int(&t->tids_in_use));
6424         }
6425
6426         if (t->nstids) {
6427                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
6428                     t->stid_base + t->nstids - 1, t->stids_in_use);
6429         }
6430
6431         if (t->nftids) {
6432                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
6433                     t->ftid_base + t->nftids - 1);
6434         }
6435
6436         if (t->netids) {
6437                 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
6438                     t->etid_base + t->netids - 1);
6439         }
6440
6441         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
6442             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
6443             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
6444
6445         rc = sbuf_finish(sb);
6446         sbuf_delete(sb);
6447
6448         return (rc);
6449 }
6450
6451 static int
6452 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
6453 {
6454         struct adapter *sc = arg1;
6455         struct sbuf *sb;
6456         int rc;
6457         struct tp_err_stats stats;
6458
6459         rc = sysctl_wire_old_buffer(req, 0);
6460         if (rc != 0)
6461                 return (rc);
6462
6463         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6464         if (sb == NULL)
6465                 return (ENOMEM);
6466
6467         t4_tp_get_err_stats(sc, &stats);
6468
6469         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
6470                       "channel 3\n");
6471         sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
6472             stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
6473             stats.macInErrs[3]);
6474         sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
6475             stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
6476             stats.hdrInErrs[3]);
6477         sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
6478             stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
6479             stats.tcpInErrs[3]);
6480         sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
6481             stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
6482             stats.tcp6InErrs[3]);
6483         sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
6484             stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
6485             stats.tnlCongDrops[3]);
6486         sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
6487             stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
6488             stats.tnlTxDrops[3]);
6489         sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
6490             stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
6491             stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
6492         sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
6493             stats.ofldChanDrops[0], stats.ofldChanDrops[1],
6494             stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
6495         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
6496             stats.ofldNoNeigh, stats.ofldCongDefer);
6497
6498         rc = sbuf_finish(sb);
6499         sbuf_delete(sb);
6500
6501         return (rc);
6502 }
6503
6504 struct field_desc {
6505         const char *name;
6506         u_int start;
6507         u_int width;
6508 };
6509
6510 static void
6511 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
6512 {
6513         char buf[32];
6514         int line_size = 0;
6515
6516         while (f->name) {
6517                 uint64_t mask = (1ULL << f->width) - 1;
6518                 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
6519                     ((uintmax_t)v >> f->start) & mask);
6520
6521                 if (line_size + len >= 79) {
6522                         line_size = 8;
6523                         sbuf_printf(sb, "\n        ");
6524                 }
6525                 sbuf_printf(sb, "%s ", buf);
6526                 line_size += len + 1;
6527                 f++;
6528         }
6529         sbuf_printf(sb, "\n");
6530 }
6531
6532 static struct field_desc tp_la0[] = {
6533         { "RcfOpCodeOut", 60, 4 },
6534         { "State", 56, 4 },
6535         { "WcfState", 52, 4 },
6536         { "RcfOpcSrcOut", 50, 2 },
6537         { "CRxError", 49, 1 },
6538         { "ERxError", 48, 1 },
6539         { "SanityFailed", 47, 1 },
6540         { "SpuriousMsg", 46, 1 },
6541         { "FlushInputMsg", 45, 1 },
6542         { "FlushInputCpl", 44, 1 },
6543         { "RssUpBit", 43, 1 },
6544         { "RssFilterHit", 42, 1 },
6545         { "Tid", 32, 10 },
6546         { "InitTcb", 31, 1 },
6547         { "LineNumber", 24, 7 },
6548         { "Emsg", 23, 1 },
6549         { "EdataOut", 22, 1 },
6550         { "Cmsg", 21, 1 },
6551         { "CdataOut", 20, 1 },
6552         { "EreadPdu", 19, 1 },
6553         { "CreadPdu", 18, 1 },
6554         { "TunnelPkt", 17, 1 },
6555         { "RcfPeerFin", 16, 1 },
6556         { "RcfReasonOut", 12, 4 },
6557         { "TxCchannel", 10, 2 },
6558         { "RcfTxChannel", 8, 2 },
6559         { "RxEchannel", 6, 2 },
6560         { "RcfRxChannel", 5, 1 },
6561         { "RcfDataOutSrdy", 4, 1 },
6562         { "RxDvld", 3, 1 },
6563         { "RxOoDvld", 2, 1 },
6564         { "RxCongestion", 1, 1 },
6565         { "TxCongestion", 0, 1 },
6566         { NULL }
6567 };
6568
6569 static struct field_desc tp_la1[] = {
6570         { "CplCmdIn", 56, 8 },
6571         { "CplCmdOut", 48, 8 },
6572         { "ESynOut", 47, 1 },
6573         { "EAckOut", 46, 1 },
6574         { "EFinOut", 45, 1 },
6575         { "ERstOut", 44, 1 },
6576         { "SynIn", 43, 1 },
6577         { "AckIn", 42, 1 },
6578         { "FinIn", 41, 1 },
6579         { "RstIn", 40, 1 },
6580         { "DataIn", 39, 1 },
6581         { "DataInVld", 38, 1 },
6582         { "PadIn", 37, 1 },
6583         { "RxBufEmpty", 36, 1 },
6584         { "RxDdp", 35, 1 },
6585         { "RxFbCongestion", 34, 1 },
6586         { "TxFbCongestion", 33, 1 },
6587         { "TxPktSumSrdy", 32, 1 },
6588         { "RcfUlpType", 28, 4 },
6589         { "Eread", 27, 1 },
6590         { "Ebypass", 26, 1 },
6591         { "Esave", 25, 1 },
6592         { "Static0", 24, 1 },
6593         { "Cread", 23, 1 },
6594         { "Cbypass", 22, 1 },
6595         { "Csave", 21, 1 },
6596         { "CPktOut", 20, 1 },
6597         { "RxPagePoolFull", 18, 2 },
6598         { "RxLpbkPkt", 17, 1 },
6599         { "TxLpbkPkt", 16, 1 },
6600         { "RxVfValid", 15, 1 },
6601         { "SynLearned", 14, 1 },
6602         { "SetDelEntry", 13, 1 },
6603         { "SetInvEntry", 12, 1 },
6604         { "CpcmdDvld", 11, 1 },
6605         { "CpcmdSave", 10, 1 },
6606         { "RxPstructsFull", 8, 2 },
6607         { "EpcmdDvld", 7, 1 },
6608         { "EpcmdFlush", 6, 1 },
6609         { "EpcmdTrimPrefix", 5, 1 },
6610         { "EpcmdTrimPostfix", 4, 1 },
6611         { "ERssIp4Pkt", 3, 1 },
6612         { "ERssIp6Pkt", 2, 1 },
6613         { "ERssTcpUdpPkt", 1, 1 },
6614         { "ERssFceFipPkt", 0, 1 },
6615         { NULL }
6616 };
6617
6618 static struct field_desc tp_la2[] = {
6619         { "CplCmdIn", 56, 8 },
6620         { "MpsVfVld", 55, 1 },
6621         { "MpsPf", 52, 3 },
6622         { "MpsVf", 44, 8 },
6623         { "SynIn", 43, 1 },
6624         { "AckIn", 42, 1 },
6625         { "FinIn", 41, 1 },
6626         { "RstIn", 40, 1 },
6627         { "DataIn", 39, 1 },
6628         { "DataInVld", 38, 1 },
6629         { "PadIn", 37, 1 },
6630         { "RxBufEmpty", 36, 1 },
6631         { "RxDdp", 35, 1 },
6632         { "RxFbCongestion", 34, 1 },
6633         { "TxFbCongestion", 33, 1 },
6634         { "TxPktSumSrdy", 32, 1 },
6635         { "RcfUlpType", 28, 4 },
6636         { "Eread", 27, 1 },
6637         { "Ebypass", 26, 1 },
6638         { "Esave", 25, 1 },
6639         { "Static0", 24, 1 },
6640         { "Cread", 23, 1 },
6641         { "Cbypass", 22, 1 },
6642         { "Csave", 21, 1 },
6643         { "CPktOut", 20, 1 },
6644         { "RxPagePoolFull", 18, 2 },
6645         { "RxLpbkPkt", 17, 1 },
6646         { "TxLpbkPkt", 16, 1 },
6647         { "RxVfValid", 15, 1 },
6648         { "SynLearned", 14, 1 },
6649         { "SetDelEntry", 13, 1 },
6650         { "SetInvEntry", 12, 1 },
6651         { "CpcmdDvld", 11, 1 },
6652         { "CpcmdSave", 10, 1 },
6653         { "RxPstructsFull", 8, 2 },
6654         { "EpcmdDvld", 7, 1 },
6655         { "EpcmdFlush", 6, 1 },
6656         { "EpcmdTrimPrefix", 5, 1 },
6657         { "EpcmdTrimPostfix", 4, 1 },
6658         { "ERssIp4Pkt", 3, 1 },
6659         { "ERssIp6Pkt", 2, 1 },
6660         { "ERssTcpUdpPkt", 1, 1 },
6661         { "ERssFceFipPkt", 0, 1 },
6662         { NULL }
6663 };
6664
6665 static void
6666 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
6667 {
6668
6669         field_desc_show(sb, *p, tp_la0);
6670 }
6671
6672 static void
6673 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
6674 {
6675
6676         if (idx)
6677                 sbuf_printf(sb, "\n");
6678         field_desc_show(sb, p[0], tp_la0);
6679         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6680                 field_desc_show(sb, p[1], tp_la0);
6681 }
6682
6683 static void
6684 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
6685 {
6686
6687         if (idx)
6688                 sbuf_printf(sb, "\n");
6689         field_desc_show(sb, p[0], tp_la0);
6690         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6691                 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
6692 }
6693
6694 static int
6695 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
6696 {
6697         struct adapter *sc = arg1;
6698         struct sbuf *sb;
6699         uint64_t *buf, *p;
6700         int rc;
6701         u_int i, inc;
6702         void (*show_func)(struct sbuf *, uint64_t *, int);
6703
6704         rc = sysctl_wire_old_buffer(req, 0);
6705         if (rc != 0)
6706                 return (rc);
6707
6708         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6709         if (sb == NULL)
6710                 return (ENOMEM);
6711
6712         buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
6713
6714         t4_tp_read_la(sc, buf, NULL);
6715         p = buf;
6716
6717         switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
6718         case 2:
6719                 inc = 2;
6720                 show_func = tp_la_show2;
6721                 break;
6722         case 3:
6723                 inc = 2;
6724                 show_func = tp_la_show3;
6725                 break;
6726         default:
6727                 inc = 1;
6728                 show_func = tp_la_show;
6729         }
6730
6731         for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
6732                 (*show_func)(sb, p, i);
6733
6734         rc = sbuf_finish(sb);
6735         sbuf_delete(sb);
6736         free(buf, M_CXGBE);
6737         return (rc);
6738 }
6739
6740 static int
6741 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
6742 {
6743         struct adapter *sc = arg1;
6744         struct sbuf *sb;
6745         int rc;
6746         u64 nrate[NCHAN], orate[NCHAN];
6747
6748         rc = sysctl_wire_old_buffer(req, 0);
6749         if (rc != 0)
6750                 return (rc);
6751
6752         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6753         if (sb == NULL)
6754                 return (ENOMEM);
6755
6756         t4_get_chan_txrate(sc, nrate, orate);
6757         sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
6758                  "channel 3\n");
6759         sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
6760             nrate[0], nrate[1], nrate[2], nrate[3]);
6761         sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
6762             orate[0], orate[1], orate[2], orate[3]);
6763
6764         rc = sbuf_finish(sb);
6765         sbuf_delete(sb);
6766
6767         return (rc);
6768 }
6769
6770 static int
6771 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
6772 {
6773         struct adapter *sc = arg1;
6774         struct sbuf *sb;
6775         uint32_t *buf, *p;
6776         int rc, i;
6777
6778         rc = sysctl_wire_old_buffer(req, 0);
6779         if (rc != 0)
6780                 return (rc);
6781
6782         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6783         if (sb == NULL)
6784                 return (ENOMEM);
6785
6786         buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
6787             M_ZERO | M_WAITOK);
6788
6789         t4_ulprx_read_la(sc, buf);
6790         p = buf;
6791
6792         sbuf_printf(sb, "      Pcmd        Type   Message"
6793             "                Data");
6794         for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
6795                 sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
6796                     p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
6797         }
6798
6799         rc = sbuf_finish(sb);
6800         sbuf_delete(sb);
6801         free(buf, M_CXGBE);
6802         return (rc);
6803 }
6804
6805 static int
6806 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
6807 {
6808         struct adapter *sc = arg1;
6809         struct sbuf *sb;
6810         int rc, v;
6811
6812         rc = sysctl_wire_old_buffer(req, 0);
6813         if (rc != 0)
6814                 return (rc);
6815
6816         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6817         if (sb == NULL)
6818                 return (ENOMEM);
6819
6820         v = t4_read_reg(sc, A_SGE_STAT_CFG);
6821         if (G_STATSOURCE_T5(v) == 7) {
6822                 if (G_STATMODE(v) == 0) {
6823                         sbuf_printf(sb, "total %d, incomplete %d",
6824                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6825                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6826                 } else if (G_STATMODE(v) == 1) {
6827                         sbuf_printf(sb, "total %d, data overflow %d",
6828                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6829                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6830                 }
6831         }
6832         rc = sbuf_finish(sb);
6833         sbuf_delete(sb);
6834
6835         return (rc);
6836 }
6837 #endif
6838
6839 static inline void
6840 txq_start(struct ifnet *ifp, struct sge_txq *txq)
6841 {
6842         struct buf_ring *br;
6843         struct mbuf *m;
6844
6845         TXQ_LOCK_ASSERT_OWNED(txq);
6846
6847         br = txq->br;
6848         m = txq->m ? txq->m : drbr_dequeue(ifp, br);
6849         if (m)
6850                 t4_eth_tx(ifp, txq, m);
6851 }
6852
6853 void
6854 t4_tx_callout(void *arg)
6855 {
6856         struct sge_eq *eq = arg;
6857         struct adapter *sc;
6858
6859         if (EQ_TRYLOCK(eq) == 0)
6860                 goto reschedule;
6861
6862         if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) {
6863                 EQ_UNLOCK(eq);
6864 reschedule:
6865                 if (__predict_true(!(eq->flags && EQ_DOOMED)))
6866                         callout_schedule(&eq->tx_callout, 1);
6867                 return;
6868         }
6869
6870         EQ_LOCK_ASSERT_OWNED(eq);
6871
6872         if (__predict_true((eq->flags & EQ_DOOMED) == 0)) {
6873
6874                 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
6875                         struct sge_txq *txq = arg;
6876                         struct port_info *pi = txq->ifp->if_softc;
6877
6878                         sc = pi->adapter;
6879                 } else {
6880                         struct sge_wrq *wrq = arg;
6881
6882                         sc = wrq->adapter;
6883                 }
6884
6885                 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task);
6886         }
6887
6888         EQ_UNLOCK(eq);
6889 }
6890
6891 void
6892 t4_tx_task(void *arg, int count)
6893 {
6894         struct sge_eq *eq = arg;
6895
6896         EQ_LOCK(eq);
6897         if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
6898                 struct sge_txq *txq = arg;
6899                 txq_start(txq->ifp, txq);
6900         } else {
6901                 struct sge_wrq *wrq = arg;
6902                 t4_wrq_tx_locked(wrq->adapter, wrq, NULL);
6903         }
6904         EQ_UNLOCK(eq);
6905 }
6906
6907 static uint32_t
6908 fconf_to_mode(uint32_t fconf)
6909 {
6910         uint32_t mode;
6911
6912         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
6913             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
6914
6915         if (fconf & F_FRAGMENTATION)
6916                 mode |= T4_FILTER_IP_FRAGMENT;
6917
6918         if (fconf & F_MPSHITTYPE)
6919                 mode |= T4_FILTER_MPS_HIT_TYPE;
6920
6921         if (fconf & F_MACMATCH)
6922                 mode |= T4_FILTER_MAC_IDX;
6923
6924         if (fconf & F_ETHERTYPE)
6925                 mode |= T4_FILTER_ETH_TYPE;
6926
6927         if (fconf & F_PROTOCOL)
6928                 mode |= T4_FILTER_IP_PROTO;
6929
6930         if (fconf & F_TOS)
6931                 mode |= T4_FILTER_IP_TOS;
6932
6933         if (fconf & F_VLAN)
6934                 mode |= T4_FILTER_VLAN;
6935
6936         if (fconf & F_VNIC_ID)
6937                 mode |= T4_FILTER_VNIC;
6938
6939         if (fconf & F_PORT)
6940                 mode |= T4_FILTER_PORT;
6941
6942         if (fconf & F_FCOE)
6943                 mode |= T4_FILTER_FCoE;
6944
6945         return (mode);
6946 }
6947
6948 static uint32_t
6949 mode_to_fconf(uint32_t mode)
6950 {
6951         uint32_t fconf = 0;
6952
6953         if (mode & T4_FILTER_IP_FRAGMENT)
6954                 fconf |= F_FRAGMENTATION;
6955
6956         if (mode & T4_FILTER_MPS_HIT_TYPE)
6957                 fconf |= F_MPSHITTYPE;
6958
6959         if (mode & T4_FILTER_MAC_IDX)
6960                 fconf |= F_MACMATCH;
6961
6962         if (mode & T4_FILTER_ETH_TYPE)
6963                 fconf |= F_ETHERTYPE;
6964
6965         if (mode & T4_FILTER_IP_PROTO)
6966                 fconf |= F_PROTOCOL;
6967
6968         if (mode & T4_FILTER_IP_TOS)
6969                 fconf |= F_TOS;
6970
6971         if (mode & T4_FILTER_VLAN)
6972                 fconf |= F_VLAN;
6973
6974         if (mode & T4_FILTER_VNIC)
6975                 fconf |= F_VNIC_ID;
6976
6977         if (mode & T4_FILTER_PORT)
6978                 fconf |= F_PORT;
6979
6980         if (mode & T4_FILTER_FCoE)
6981                 fconf |= F_FCOE;
6982
6983         return (fconf);
6984 }
6985
6986 static uint32_t
6987 fspec_to_fconf(struct t4_filter_specification *fs)
6988 {
6989         uint32_t fconf = 0;
6990
6991         if (fs->val.frag || fs->mask.frag)
6992                 fconf |= F_FRAGMENTATION;
6993
6994         if (fs->val.matchtype || fs->mask.matchtype)
6995                 fconf |= F_MPSHITTYPE;
6996
6997         if (fs->val.macidx || fs->mask.macidx)
6998                 fconf |= F_MACMATCH;
6999
7000         if (fs->val.ethtype || fs->mask.ethtype)
7001                 fconf |= F_ETHERTYPE;
7002
7003         if (fs->val.proto || fs->mask.proto)
7004                 fconf |= F_PROTOCOL;
7005
7006         if (fs->val.tos || fs->mask.tos)
7007                 fconf |= F_TOS;
7008
7009         if (fs->val.vlan_vld || fs->mask.vlan_vld)
7010                 fconf |= F_VLAN;
7011
7012         if (fs->val.vnic_vld || fs->mask.vnic_vld)
7013                 fconf |= F_VNIC_ID;
7014
7015         if (fs->val.iport || fs->mask.iport)
7016                 fconf |= F_PORT;
7017
7018         if (fs->val.fcoe || fs->mask.fcoe)
7019                 fconf |= F_FCOE;
7020
7021         return (fconf);
7022 }
7023
7024 static int
7025 get_filter_mode(struct adapter *sc, uint32_t *mode)
7026 {
7027         int rc;
7028         uint32_t fconf;
7029
7030         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7031             "t4getfm");
7032         if (rc)
7033                 return (rc);
7034
7035         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
7036             A_TP_VLAN_PRI_MAP);
7037
7038         if (sc->params.tp.vlan_pri_map != fconf) {
7039                 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
7040                     device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
7041                     fconf);
7042                 sc->params.tp.vlan_pri_map = fconf;
7043         }
7044
7045         *mode = fconf_to_mode(sc->params.tp.vlan_pri_map);
7046
7047         end_synchronized_op(sc, LOCK_HELD);
7048         return (0);
7049 }
7050
7051 static int
7052 set_filter_mode(struct adapter *sc, uint32_t mode)
7053 {
7054         uint32_t fconf;
7055         int rc;
7056
7057         fconf = mode_to_fconf(mode);
7058
7059         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7060             "t4setfm");
7061         if (rc)
7062                 return (rc);
7063
7064         if (sc->tids.ftids_in_use > 0) {
7065                 rc = EBUSY;
7066                 goto done;
7067         }
7068
7069 #ifdef TCP_OFFLOAD
7070         if (sc->offload_map) {
7071                 rc = EBUSY;
7072                 goto done;
7073         }
7074 #endif
7075
7076 #ifdef notyet
7077         rc = -t4_set_filter_mode(sc, fconf);
7078         if (rc == 0)
7079                 sc->filter_mode = fconf;
7080 #else
7081         rc = ENOTSUP;
7082 #endif
7083
7084 done:
7085         end_synchronized_op(sc, LOCK_HELD);
7086         return (rc);
7087 }
7088
7089 static inline uint64_t
7090 get_filter_hits(struct adapter *sc, uint32_t fid)
7091 {
7092         uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
7093         uint64_t hits;
7094
7095         memwin_info(sc, 0, &mw_base, NULL);
7096         off = position_memwin(sc, 0,
7097             tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
7098         if (is_t4(sc)) {
7099                 hits = t4_read_reg64(sc, mw_base + off + 16);
7100                 hits = be64toh(hits);
7101         } else {
7102                 hits = t4_read_reg(sc, mw_base + off + 24);
7103                 hits = be32toh(hits);
7104         }
7105
7106         return (hits);
7107 }
7108
7109 static int
7110 get_filter(struct adapter *sc, struct t4_filter *t)
7111 {
7112         int i, rc, nfilters = sc->tids.nftids;
7113         struct filter_entry *f;
7114
7115         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7116             "t4getf");
7117         if (rc)
7118                 return (rc);
7119
7120         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
7121             t->idx >= nfilters) {
7122                 t->idx = 0xffffffff;
7123                 goto done;
7124         }
7125
7126         f = &sc->tids.ftid_tab[t->idx];
7127         for (i = t->idx; i < nfilters; i++, f++) {
7128                 if (f->valid) {
7129                         t->idx = i;
7130                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
7131                         t->smtidx = f->smtidx;
7132                         if (f->fs.hitcnts)
7133                                 t->hits = get_filter_hits(sc, t->idx);
7134                         else
7135                                 t->hits = UINT64_MAX;
7136                         t->fs = f->fs;
7137
7138                         goto done;
7139                 }
7140         }
7141
7142         t->idx = 0xffffffff;
7143 done:
7144         end_synchronized_op(sc, LOCK_HELD);
7145         return (0);
7146 }
7147
7148 static int
7149 set_filter(struct adapter *sc, struct t4_filter *t)
7150 {
7151         unsigned int nfilters, nports;
7152         struct filter_entry *f;
7153         int i, rc;
7154
7155         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
7156         if (rc)
7157                 return (rc);
7158
7159         nfilters = sc->tids.nftids;
7160         nports = sc->params.nports;
7161
7162         if (nfilters == 0) {
7163                 rc = ENOTSUP;
7164                 goto done;
7165         }
7166
7167         if (!(sc->flags & FULL_INIT_DONE)) {
7168                 rc = EAGAIN;
7169                 goto done;
7170         }
7171
7172         if (t->idx >= nfilters) {
7173                 rc = EINVAL;
7174                 goto done;
7175         }
7176
7177         /* Validate against the global filter mode */
7178         if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
7179             sc->params.tp.vlan_pri_map) {
7180                 rc = E2BIG;
7181                 goto done;
7182         }
7183
7184         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
7185                 rc = EINVAL;
7186                 goto done;
7187         }
7188
7189         if (t->fs.val.iport >= nports) {
7190                 rc = EINVAL;
7191                 goto done;
7192         }
7193
7194         /* Can't specify an iq if not steering to it */
7195         if (!t->fs.dirsteer && t->fs.iq) {
7196                 rc = EINVAL;
7197                 goto done;
7198         }
7199
7200         /* IPv6 filter idx must be 4 aligned */
7201         if (t->fs.type == 1 &&
7202             ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
7203                 rc = EINVAL;
7204                 goto done;
7205         }
7206
7207         if (sc->tids.ftid_tab == NULL) {
7208                 KASSERT(sc->tids.ftids_in_use == 0,
7209                     ("%s: no memory allocated but filters_in_use > 0",
7210                     __func__));
7211
7212                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
7213                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
7214                 if (sc->tids.ftid_tab == NULL) {
7215                         rc = ENOMEM;
7216                         goto done;
7217                 }
7218                 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
7219         }
7220
7221         for (i = 0; i < 4; i++) {
7222                 f = &sc->tids.ftid_tab[t->idx + i];
7223
7224                 if (f->pending || f->valid) {
7225                         rc = EBUSY;
7226                         goto done;
7227                 }
7228                 if (f->locked) {
7229                         rc = EPERM;
7230                         goto done;
7231                 }
7232
7233                 if (t->fs.type == 0)
7234                         break;
7235         }
7236
7237         f = &sc->tids.ftid_tab[t->idx];
7238         f->fs = t->fs;
7239
7240         rc = set_filter_wr(sc, t->idx);
7241 done:
7242         end_synchronized_op(sc, 0);
7243
7244         if (rc == 0) {
7245                 mtx_lock(&sc->tids.ftid_lock);
7246                 for (;;) {
7247                         if (f->pending == 0) {
7248                                 rc = f->valid ? 0 : EIO;
7249                                 break;
7250                         }
7251
7252                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7253                             PCATCH, "t4setfw", 0)) {
7254                                 rc = EINPROGRESS;
7255                                 break;
7256                         }
7257                 }
7258                 mtx_unlock(&sc->tids.ftid_lock);
7259         }
7260         return (rc);
7261 }
7262
7263 static int
7264 del_filter(struct adapter *sc, struct t4_filter *t)
7265 {
7266         unsigned int nfilters;
7267         struct filter_entry *f;
7268         int rc;
7269
7270         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
7271         if (rc)
7272                 return (rc);
7273
7274         nfilters = sc->tids.nftids;
7275
7276         if (nfilters == 0) {
7277                 rc = ENOTSUP;
7278                 goto done;
7279         }
7280
7281         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
7282             t->idx >= nfilters) {
7283                 rc = EINVAL;
7284                 goto done;
7285         }
7286
7287         if (!(sc->flags & FULL_INIT_DONE)) {
7288                 rc = EAGAIN;
7289                 goto done;
7290         }
7291
7292         f = &sc->tids.ftid_tab[t->idx];
7293
7294         if (f->pending) {
7295                 rc = EBUSY;
7296                 goto done;
7297         }
7298         if (f->locked) {
7299                 rc = EPERM;
7300                 goto done;
7301         }
7302
7303         if (f->valid) {
7304                 t->fs = f->fs;  /* extra info for the caller */
7305                 rc = del_filter_wr(sc, t->idx);
7306         }
7307
7308 done:
7309         end_synchronized_op(sc, 0);
7310
7311         if (rc == 0) {
7312                 mtx_lock(&sc->tids.ftid_lock);
7313                 for (;;) {
7314                         if (f->pending == 0) {
7315                                 rc = f->valid ? EIO : 0;
7316                                 break;
7317                         }
7318
7319                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7320                             PCATCH, "t4delfw", 0)) {
7321                                 rc = EINPROGRESS;
7322                                 break;
7323                         }
7324                 }
7325                 mtx_unlock(&sc->tids.ftid_lock);
7326         }
7327
7328         return (rc);
7329 }
7330
7331 static void
7332 clear_filter(struct filter_entry *f)
7333 {
7334         if (f->l2t)
7335                 t4_l2t_release(f->l2t);
7336
7337         bzero(f, sizeof (*f));
7338 }
7339
7340 static int
7341 set_filter_wr(struct adapter *sc, int fidx)
7342 {
7343         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7344         struct wrqe *wr;
7345         struct fw_filter_wr *fwr;
7346         unsigned int ftid;
7347
7348         ASSERT_SYNCHRONIZED_OP(sc);
7349
7350         if (f->fs.newdmac || f->fs.newvlan) {
7351                 /* This filter needs an L2T entry; allocate one. */
7352                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
7353                 if (f->l2t == NULL)
7354                         return (EAGAIN);
7355                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
7356                     f->fs.dmac)) {
7357                         t4_l2t_release(f->l2t);
7358                         f->l2t = NULL;
7359                         return (ENOMEM);
7360                 }
7361         }
7362
7363         ftid = sc->tids.ftid_base + fidx;
7364
7365         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7366         if (wr == NULL)
7367                 return (ENOMEM);
7368
7369         fwr = wrtod(wr);
7370         bzero(fwr, sizeof (*fwr));
7371
7372         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
7373         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
7374         fwr->tid_to_iq =
7375             htobe32(V_FW_FILTER_WR_TID(ftid) |
7376                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
7377                 V_FW_FILTER_WR_NOREPLY(0) |
7378                 V_FW_FILTER_WR_IQ(f->fs.iq));
7379         fwr->del_filter_to_l2tix =
7380             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
7381                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
7382                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
7383                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
7384                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
7385                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
7386                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
7387                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
7388                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
7389                     f->fs.newvlan == VLAN_REWRITE) |
7390                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
7391                     f->fs.newvlan == VLAN_REWRITE) |
7392                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
7393                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
7394                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
7395                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
7396         fwr->ethtype = htobe16(f->fs.val.ethtype);
7397         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
7398         fwr->frag_to_ovlan_vldm =
7399             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
7400                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
7401                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
7402                 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
7403                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
7404                 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
7405         fwr->smac_sel = 0;
7406         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
7407             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
7408         fwr->maci_to_matchtypem =
7409             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
7410                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
7411                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
7412                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
7413                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
7414                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
7415                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
7416                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
7417         fwr->ptcl = f->fs.val.proto;
7418         fwr->ptclm = f->fs.mask.proto;
7419         fwr->ttyp = f->fs.val.tos;
7420         fwr->ttypm = f->fs.mask.tos;
7421         fwr->ivlan = htobe16(f->fs.val.vlan);
7422         fwr->ivlanm = htobe16(f->fs.mask.vlan);
7423         fwr->ovlan = htobe16(f->fs.val.vnic);
7424         fwr->ovlanm = htobe16(f->fs.mask.vnic);
7425         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
7426         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
7427         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
7428         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
7429         fwr->lp = htobe16(f->fs.val.dport);
7430         fwr->lpm = htobe16(f->fs.mask.dport);
7431         fwr->fp = htobe16(f->fs.val.sport);
7432         fwr->fpm = htobe16(f->fs.mask.sport);
7433         if (f->fs.newsmac)
7434                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
7435
7436         f->pending = 1;
7437         sc->tids.ftids_in_use++;
7438
7439         t4_wrq_tx(sc, wr);
7440         return (0);
7441 }
7442
7443 static int
7444 del_filter_wr(struct adapter *sc, int fidx)
7445 {
7446         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7447         struct wrqe *wr;
7448         struct fw_filter_wr *fwr;
7449         unsigned int ftid;
7450
7451         ftid = sc->tids.ftid_base + fidx;
7452
7453         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7454         if (wr == NULL)
7455                 return (ENOMEM);
7456         fwr = wrtod(wr);
7457         bzero(fwr, sizeof (*fwr));
7458
7459         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
7460
7461         f->pending = 1;
7462         t4_wrq_tx(sc, wr);
7463         return (0);
7464 }
7465
7466 int
7467 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
7468 {
7469         struct adapter *sc = iq->adapter;
7470         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
7471         unsigned int idx = GET_TID(rpl);
7472         unsigned int rc;
7473         struct filter_entry *f;
7474
7475         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
7476             rss->opcode));
7477
7478         if (is_ftid(sc, idx)) {
7479
7480                 idx -= sc->tids.ftid_base;
7481                 f = &sc->tids.ftid_tab[idx];
7482                 rc = G_COOKIE(rpl->cookie);
7483
7484                 mtx_lock(&sc->tids.ftid_lock);
7485                 if (rc == FW_FILTER_WR_FLT_ADDED) {
7486                         KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
7487                             __func__, idx));
7488                         f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
7489                         f->pending = 0;  /* asynchronous setup completed */
7490                         f->valid = 1;
7491                 } else {
7492                         if (rc != FW_FILTER_WR_FLT_DELETED) {
7493                                 /* Add or delete failed, display an error */
7494                                 log(LOG_ERR,
7495                                     "filter %u setup failed with error %u\n",
7496                                     idx, rc);
7497                         }
7498
7499                         clear_filter(f);
7500                         sc->tids.ftids_in_use--;
7501                 }
7502                 wakeup(&sc->tids.ftid_tab);
7503                 mtx_unlock(&sc->tids.ftid_lock);
7504         }
7505
7506         return (0);
7507 }
7508
7509 static int
7510 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
7511 {
7512         int rc;
7513
7514         if (cntxt->cid > M_CTXTQID)
7515                 return (EINVAL);
7516
7517         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
7518             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
7519                 return (EINVAL);
7520
7521         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
7522         if (rc)
7523                 return (rc);
7524
7525         if (sc->flags & FW_OK) {
7526                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
7527                     &cntxt->data[0]);
7528                 if (rc == 0)
7529                         goto done;
7530         }
7531
7532         /*
7533          * Read via firmware failed or wasn't even attempted.  Read directly via
7534          * the backdoor.
7535          */
7536         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
7537 done:
7538         end_synchronized_op(sc, 0);
7539         return (rc);
7540 }
7541
7542 static int
7543 load_fw(struct adapter *sc, struct t4_data *fw)
7544 {
7545         int rc;
7546         uint8_t *fw_data;
7547
7548         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
7549         if (rc)
7550                 return (rc);
7551
7552         if (sc->flags & FULL_INIT_DONE) {
7553                 rc = EBUSY;
7554                 goto done;
7555         }
7556
7557         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
7558         if (fw_data == NULL) {
7559                 rc = ENOMEM;
7560                 goto done;
7561         }
7562
7563         rc = copyin(fw->data, fw_data, fw->len);
7564         if (rc == 0)
7565                 rc = -t4_load_fw(sc, fw_data, fw->len);
7566
7567         free(fw_data, M_CXGBE);
7568 done:
7569         end_synchronized_op(sc, 0);
7570         return (rc);
7571 }
7572
7573 static int
7574 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
7575 {
7576         uint32_t addr, off, remaining, i, n;
7577         uint32_t *buf, *b;
7578         uint32_t mw_base, mw_aperture;
7579         int rc;
7580         uint8_t *dst;
7581
7582         rc = validate_mem_range(sc, mr->addr, mr->len);
7583         if (rc != 0)
7584                 return (rc);
7585
7586         memwin_info(sc, win, &mw_base, &mw_aperture);
7587         buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
7588         addr = mr->addr;
7589         remaining = mr->len;
7590         dst = (void *)mr->data;
7591
7592         while (remaining) {
7593                 off = position_memwin(sc, win, addr);
7594
7595                 /* number of bytes that we'll copy in the inner loop */
7596                 n = min(remaining, mw_aperture - off);
7597                 for (i = 0; i < n; i += 4)
7598                         *b++ = t4_read_reg(sc, mw_base + off + i);
7599
7600                 rc = copyout(buf, dst, n);
7601                 if (rc != 0)
7602                         break;
7603
7604                 b = buf;
7605                 dst += n;
7606                 remaining -= n;
7607                 addr += n;
7608         }
7609
7610         free(buf, M_CXGBE);
7611         return (rc);
7612 }
7613
7614 static int
7615 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
7616 {
7617         int rc;
7618
7619         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
7620                 return (EINVAL);
7621
7622         if (i2cd->len > sizeof(i2cd->data))
7623                 return (EFBIG);
7624
7625         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
7626         if (rc)
7627                 return (rc);
7628         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
7629             i2cd->offset, i2cd->len, &i2cd->data[0]);
7630         end_synchronized_op(sc, 0);
7631
7632         return (rc);
7633 }
7634
7635 static int
7636 in_range(int val, int lo, int hi)
7637 {
7638
7639         return (val < 0 || (val <= hi && val >= lo));
7640 }
7641
7642 static int
7643 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
7644 {
7645         int fw_subcmd, fw_type, rc;
7646
7647         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
7648         if (rc)
7649                 return (rc);
7650
7651         if (!(sc->flags & FULL_INIT_DONE)) {
7652                 rc = EAGAIN;
7653                 goto done;
7654         }
7655
7656         /*
7657          * Translate the cxgbetool parameters into T4 firmware parameters.  (The
7658          * sub-command and type are in common locations.)
7659          */
7660         if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
7661                 fw_subcmd = FW_SCHED_SC_CONFIG;
7662         else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
7663                 fw_subcmd = FW_SCHED_SC_PARAMS;
7664         else {
7665                 rc = EINVAL;
7666                 goto done;
7667         }
7668         if (p->type == SCHED_CLASS_TYPE_PACKET)
7669                 fw_type = FW_SCHED_TYPE_PKTSCHED;
7670         else {
7671                 rc = EINVAL;
7672                 goto done;
7673         }
7674
7675         if (fw_subcmd == FW_SCHED_SC_CONFIG) {
7676                 /* Vet our parameters ..*/
7677                 if (p->u.config.minmax < 0) {
7678                         rc = EINVAL;
7679                         goto done;
7680                 }
7681
7682                 /* And pass the request to the firmware ...*/
7683                 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1);
7684                 goto done;
7685         }
7686
7687         if (fw_subcmd == FW_SCHED_SC_PARAMS) {
7688                 int fw_level;
7689                 int fw_mode;
7690                 int fw_rateunit;
7691                 int fw_ratemode;
7692
7693                 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
7694                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
7695                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
7696                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
7697                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
7698                         fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
7699                 else {
7700                         rc = EINVAL;
7701                         goto done;
7702                 }
7703
7704                 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
7705                         fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
7706                 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
7707                         fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
7708                 else {
7709                         rc = EINVAL;
7710                         goto done;
7711                 }
7712
7713                 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
7714                         fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
7715                 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
7716                         fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
7717                 else {
7718                         rc = EINVAL;
7719                         goto done;
7720                 }
7721
7722                 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
7723                         fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
7724                 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
7725                         fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
7726                 else {
7727                         rc = EINVAL;
7728                         goto done;
7729                 }
7730
7731                 /* Vet our parameters ... */
7732                 if (!in_range(p->u.params.channel, 0, 3) ||
7733                     !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
7734                     !in_range(p->u.params.minrate, 0, 10000000) ||
7735                     !in_range(p->u.params.maxrate, 0, 10000000) ||
7736                     !in_range(p->u.params.weight, 0, 100)) {
7737                         rc = ERANGE;
7738                         goto done;
7739                 }
7740
7741                 /*
7742                  * Translate any unset parameters into the firmware's
7743                  * nomenclature and/or fail the call if the parameters
7744                  * are required ...
7745                  */
7746                 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
7747                     p->u.params.channel < 0 || p->u.params.cl < 0) {
7748                         rc = EINVAL;
7749                         goto done;
7750                 }
7751                 if (p->u.params.minrate < 0)
7752                         p->u.params.minrate = 0;
7753                 if (p->u.params.maxrate < 0) {
7754                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7755                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7756                                 rc = EINVAL;
7757                                 goto done;
7758                         } else
7759                                 p->u.params.maxrate = 0;
7760                 }
7761                 if (p->u.params.weight < 0) {
7762                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
7763                                 rc = EINVAL;
7764                                 goto done;
7765                         } else
7766                                 p->u.params.weight = 0;
7767                 }
7768                 if (p->u.params.pktsize < 0) {
7769                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7770                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7771                                 rc = EINVAL;
7772                                 goto done;
7773                         } else
7774                                 p->u.params.pktsize = 0;
7775                 }
7776
7777                 /* See what the firmware thinks of the request ... */
7778                 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
7779                     fw_rateunit, fw_ratemode, p->u.params.channel,
7780                     p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
7781                     p->u.params.weight, p->u.params.pktsize, 1);
7782                 goto done;
7783         }
7784
7785         rc = EINVAL;
7786 done:
7787         end_synchronized_op(sc, 0);
7788         return (rc);
7789 }
7790
7791 static int
7792 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
7793 {
7794         struct port_info *pi = NULL;
7795         struct sge_txq *txq;
7796         uint32_t fw_mnem, fw_queue, fw_class;
7797         int i, rc;
7798
7799         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
7800         if (rc)
7801                 return (rc);
7802
7803         if (!(sc->flags & FULL_INIT_DONE)) {
7804                 rc = EAGAIN;
7805                 goto done;
7806         }
7807
7808         if (p->port >= sc->params.nports) {
7809                 rc = EINVAL;
7810                 goto done;
7811         }
7812
7813         pi = sc->port[p->port];
7814         if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
7815                 rc = EINVAL;
7816                 goto done;
7817         }
7818
7819         /*
7820          * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
7821          * Scheduling Class in this case).
7822          */
7823         fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
7824             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
7825         fw_class = p->cl < 0 ? 0xffffffff : p->cl;
7826
7827         /*
7828          * If op.queue is non-negative, then we're only changing the scheduling
7829          * on a single specified TX queue.
7830          */
7831         if (p->queue >= 0) {
7832                 txq = &sc->sge.txq[pi->first_txq + p->queue];
7833                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7834                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7835                     &fw_class);
7836                 goto done;
7837         }
7838
7839         /*
7840          * Change the scheduling on all the TX queues for the
7841          * interface.
7842          */
7843         for_each_txq(pi, i, txq) {
7844                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7845                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7846                     &fw_class);
7847                 if (rc)
7848                         goto done;
7849         }
7850
7851         rc = 0;
7852 done:
7853         end_synchronized_op(sc, 0);
7854         return (rc);
7855 }
7856
7857 int
7858 t4_os_find_pci_capability(struct adapter *sc, int cap)
7859 {
7860         int i;
7861
7862         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
7863 }
7864
7865 int
7866 t4_os_pci_save_state(struct adapter *sc)
7867 {
7868         device_t dev;
7869         struct pci_devinfo *dinfo;
7870
7871         dev = sc->dev;
7872         dinfo = device_get_ivars(dev);
7873
7874         pci_cfg_save(dev, dinfo, 0);
7875         return (0);
7876 }
7877
7878 int
7879 t4_os_pci_restore_state(struct adapter *sc)
7880 {
7881         device_t dev;
7882         struct pci_devinfo *dinfo;
7883
7884         dev = sc->dev;
7885         dinfo = device_get_ivars(dev);
7886
7887         pci_cfg_restore(dev, dinfo);
7888         return (0);
7889 }
7890
7891 void
7892 t4_os_portmod_changed(const struct adapter *sc, int idx)
7893 {
7894         struct port_info *pi = sc->port[idx];
7895         static const char *mod_str[] = {
7896                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
7897         };
7898
7899         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
7900                 if_printf(pi->ifp, "transceiver unplugged.\n");
7901         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
7902                 if_printf(pi->ifp, "unknown transceiver inserted.\n");
7903         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
7904                 if_printf(pi->ifp, "unsupported transceiver inserted.\n");
7905         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
7906                 if_printf(pi->ifp, "%s transceiver inserted.\n",
7907                     mod_str[pi->mod_type]);
7908         } else {
7909                 if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
7910                     pi->mod_type);
7911         }
7912 }
7913
7914 void
7915 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
7916 {
7917         struct port_info *pi = sc->port[idx];
7918         struct ifnet *ifp = pi->ifp;
7919
7920         if (link_stat) {
7921                 pi->linkdnrc = -1;
7922                 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
7923                 if_link_state_change(ifp, LINK_STATE_UP);
7924         } else {
7925                 if (reason >= 0)
7926                         pi->linkdnrc = reason;
7927                 if_link_state_change(ifp, LINK_STATE_DOWN);
7928         }
7929 }
7930
7931 void
7932 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
7933 {
7934         struct adapter *sc;
7935
7936         sx_slock(&t4_list_lock);
7937         SLIST_FOREACH(sc, &t4_list, link) {
7938                 /*
7939                  * func should not make any assumptions about what state sc is
7940                  * in - the only guarantee is that sc->sc_lock is a valid lock.
7941                  */
7942                 func(sc, arg);
7943         }
7944         sx_sunlock(&t4_list_lock);
7945 }
7946
7947 static int
7948 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
7949 {
7950        return (0);
7951 }
7952
7953 static int
7954 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
7955 {
7956        return (0);
7957 }
7958
7959 static int
7960 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
7961     struct thread *td)
7962 {
7963         int rc;
7964         struct adapter *sc = dev->si_drv1;
7965
7966         rc = priv_check(td, PRIV_DRIVER);
7967         if (rc != 0)
7968                 return (rc);
7969
7970         switch (cmd) {
7971         case CHELSIO_T4_GETREG: {
7972                 struct t4_reg *edata = (struct t4_reg *)data;
7973
7974                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
7975                         return (EFAULT);
7976
7977                 if (edata->size == 4)
7978                         edata->val = t4_read_reg(sc, edata->addr);
7979                 else if (edata->size == 8)
7980                         edata->val = t4_read_reg64(sc, edata->addr);
7981                 else
7982                         return (EINVAL);
7983
7984                 break;
7985         }
7986         case CHELSIO_T4_SETREG: {
7987                 struct t4_reg *edata = (struct t4_reg *)data;
7988
7989                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
7990                         return (EFAULT);
7991
7992                 if (edata->size == 4) {
7993                         if (edata->val & 0xffffffff00000000)
7994                                 return (EINVAL);
7995                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
7996                 } else if (edata->size == 8)
7997                         t4_write_reg64(sc, edata->addr, edata->val);
7998                 else
7999                         return (EINVAL);
8000                 break;
8001         }
8002         case CHELSIO_T4_REGDUMP: {
8003                 struct t4_regdump *regs = (struct t4_regdump *)data;
8004                 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
8005                 uint8_t *buf;
8006
8007                 if (regs->len < reglen) {
8008                         regs->len = reglen; /* hint to the caller */
8009                         return (ENOBUFS);
8010                 }
8011
8012                 regs->len = reglen;
8013                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
8014                 t4_get_regs(sc, regs, buf);
8015                 rc = copyout(buf, regs->data, reglen);
8016                 free(buf, M_CXGBE);
8017                 break;
8018         }
8019         case CHELSIO_T4_GET_FILTER_MODE:
8020                 rc = get_filter_mode(sc, (uint32_t *)data);
8021                 break;
8022         case CHELSIO_T4_SET_FILTER_MODE:
8023                 rc = set_filter_mode(sc, *(uint32_t *)data);
8024                 break;
8025         case CHELSIO_T4_GET_FILTER:
8026                 rc = get_filter(sc, (struct t4_filter *)data);
8027                 break;
8028         case CHELSIO_T4_SET_FILTER:
8029                 rc = set_filter(sc, (struct t4_filter *)data);
8030                 break;
8031         case CHELSIO_T4_DEL_FILTER:
8032                 rc = del_filter(sc, (struct t4_filter *)data);
8033                 break;
8034         case CHELSIO_T4_GET_SGE_CONTEXT:
8035                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
8036                 break;
8037         case CHELSIO_T4_LOAD_FW:
8038                 rc = load_fw(sc, (struct t4_data *)data);
8039                 break;
8040         case CHELSIO_T4_GET_MEM:
8041                 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
8042                 break;
8043         case CHELSIO_T4_GET_I2C:
8044                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
8045                 break;
8046         case CHELSIO_T4_CLEAR_STATS: {
8047                 int i;
8048                 u_int port_id = *(uint32_t *)data;
8049                 struct port_info *pi;
8050
8051                 if (port_id >= sc->params.nports)
8052                         return (EINVAL);
8053                 pi = sc->port[port_id];
8054
8055                 /* MAC stats */
8056                 t4_clr_port_stats(sc, pi->tx_chan);
8057
8058                 if (pi->flags & PORT_INIT_DONE) {
8059                         struct sge_rxq *rxq;
8060                         struct sge_txq *txq;
8061                         struct sge_wrq *wrq;
8062
8063                         for_each_rxq(pi, i, rxq) {
8064 #if defined(INET) || defined(INET6)
8065                                 rxq->lro.lro_queued = 0;
8066                                 rxq->lro.lro_flushed = 0;
8067 #endif
8068                                 rxq->rxcsum = 0;
8069                                 rxq->vlan_extraction = 0;
8070                         }
8071
8072                         for_each_txq(pi, i, txq) {
8073                                 txq->txcsum = 0;
8074                                 txq->tso_wrs = 0;
8075                                 txq->vlan_insertion = 0;
8076                                 txq->imm_wrs = 0;
8077                                 txq->sgl_wrs = 0;
8078                                 txq->txpkt_wrs = 0;
8079                                 txq->txpkts_wrs = 0;
8080                                 txq->txpkts_pkts = 0;
8081                                 txq->br->br_drops = 0;
8082                                 txq->no_dmamap = 0;
8083                                 txq->no_desc = 0;
8084                         }
8085
8086 #ifdef TCP_OFFLOAD
8087                         /* nothing to clear for each ofld_rxq */
8088
8089                         for_each_ofld_txq(pi, i, wrq) {
8090                                 wrq->tx_wrs = 0;
8091                                 wrq->no_desc = 0;
8092                         }
8093 #endif
8094                         wrq = &sc->sge.ctrlq[pi->port_id];
8095                         wrq->tx_wrs = 0;
8096                         wrq->no_desc = 0;
8097                 }
8098                 break;
8099         }
8100         case CHELSIO_T4_SCHED_CLASS:
8101                 rc = set_sched_class(sc, (struct t4_sched_params *)data);
8102                 break;
8103         case CHELSIO_T4_SCHED_QUEUE:
8104                 rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
8105                 break;
8106         case CHELSIO_T4_GET_TRACER:
8107                 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
8108                 break;
8109         case CHELSIO_T4_SET_TRACER:
8110                 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
8111                 break;
8112         default:
8113                 rc = EINVAL;
8114         }
8115
8116         return (rc);
8117 }
8118
8119 #ifdef TCP_OFFLOAD
8120 void
8121 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask,
8122     const unsigned int *pgsz_order)
8123 {
8124         struct port_info *pi = ifp->if_softc;
8125         struct adapter *sc = pi->adapter;
8126
8127         t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
8128         t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |
8129                 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) |
8130                 V_HPZ3(pgsz_order[3]));
8131 }
8132
8133 static int
8134 toe_capability(struct port_info *pi, int enable)
8135 {
8136         int rc;
8137         struct adapter *sc = pi->adapter;
8138
8139         ASSERT_SYNCHRONIZED_OP(sc);
8140
8141         if (!is_offload(sc))
8142                 return (ENODEV);
8143
8144         if (enable) {
8145                 if (!(sc->flags & FULL_INIT_DONE)) {
8146                         rc = cxgbe_init_synchronized(pi);
8147                         if (rc)
8148                                 return (rc);
8149                 }
8150
8151                 if (isset(&sc->offload_map, pi->port_id))
8152                         return (0);
8153
8154                 if (!(sc->flags & TOM_INIT_DONE)) {
8155                         rc = t4_activate_uld(sc, ULD_TOM);
8156                         if (rc == EAGAIN) {
8157                                 log(LOG_WARNING,
8158                                     "You must kldload t4_tom.ko before trying "
8159                                     "to enable TOE on a cxgbe interface.\n");
8160                         }
8161                         if (rc != 0)
8162                                 return (rc);
8163                         KASSERT(sc->tom_softc != NULL,
8164                             ("%s: TOM activated but softc NULL", __func__));
8165                         KASSERT(sc->flags & TOM_INIT_DONE,
8166                             ("%s: TOM activated but flag not set", __func__));
8167                 }
8168
8169                 setbit(&sc->offload_map, pi->port_id);
8170         } else {
8171                 if (!isset(&sc->offload_map, pi->port_id))
8172                         return (0);
8173
8174                 KASSERT(sc->flags & TOM_INIT_DONE,
8175                     ("%s: TOM never initialized?", __func__));
8176                 clrbit(&sc->offload_map, pi->port_id);
8177         }
8178
8179         return (0);
8180 }
8181
8182 /*
8183  * Add an upper layer driver to the global list.
8184  */
8185 int
8186 t4_register_uld(struct uld_info *ui)
8187 {
8188         int rc = 0;
8189         struct uld_info *u;
8190
8191         sx_xlock(&t4_uld_list_lock);
8192         SLIST_FOREACH(u, &t4_uld_list, link) {
8193             if (u->uld_id == ui->uld_id) {
8194                     rc = EEXIST;
8195                     goto done;
8196             }
8197         }
8198
8199         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
8200         ui->refcount = 0;
8201 done:
8202         sx_xunlock(&t4_uld_list_lock);
8203         return (rc);
8204 }
8205
8206 int
8207 t4_unregister_uld(struct uld_info *ui)
8208 {
8209         int rc = EINVAL;
8210         struct uld_info *u;
8211
8212         sx_xlock(&t4_uld_list_lock);
8213
8214         SLIST_FOREACH(u, &t4_uld_list, link) {
8215             if (u == ui) {
8216                     if (ui->refcount > 0) {
8217                             rc = EBUSY;
8218                             goto done;
8219                     }
8220
8221                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
8222                     rc = 0;
8223                     goto done;
8224             }
8225         }
8226 done:
8227         sx_xunlock(&t4_uld_list_lock);
8228         return (rc);
8229 }
8230
8231 int
8232 t4_activate_uld(struct adapter *sc, int id)
8233 {
8234         int rc = EAGAIN;
8235         struct uld_info *ui;
8236
8237         ASSERT_SYNCHRONIZED_OP(sc);
8238
8239         sx_slock(&t4_uld_list_lock);
8240
8241         SLIST_FOREACH(ui, &t4_uld_list, link) {
8242                 if (ui->uld_id == id) {
8243                         rc = ui->activate(sc);
8244                         if (rc == 0)
8245                                 ui->refcount++;
8246                         goto done;
8247                 }
8248         }
8249 done:
8250         sx_sunlock(&t4_uld_list_lock);
8251
8252         return (rc);
8253 }
8254
8255 int
8256 t4_deactivate_uld(struct adapter *sc, int id)
8257 {
8258         int rc = EINVAL;
8259         struct uld_info *ui;
8260
8261         ASSERT_SYNCHRONIZED_OP(sc);
8262
8263         sx_slock(&t4_uld_list_lock);
8264
8265         SLIST_FOREACH(ui, &t4_uld_list, link) {
8266                 if (ui->uld_id == id) {
8267                         rc = ui->deactivate(sc);
8268                         if (rc == 0)
8269                                 ui->refcount--;
8270                         goto done;
8271                 }
8272         }
8273 done:
8274         sx_sunlock(&t4_uld_list_lock);
8275
8276         return (rc);
8277 }
8278 #endif
8279
8280 /*
8281  * Come up with reasonable defaults for some of the tunables, provided they're
8282  * not set by the user (in which case we'll use the values as is).
8283  */
8284 static void
8285 tweak_tunables(void)
8286 {
8287         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
8288
8289         if (t4_ntxq10g < 1)
8290                 t4_ntxq10g = min(nc, NTXQ_10G);
8291
8292         if (t4_ntxq1g < 1)
8293                 t4_ntxq1g = min(nc, NTXQ_1G);
8294
8295         if (t4_nrxq10g < 1)
8296                 t4_nrxq10g = min(nc, NRXQ_10G);
8297
8298         if (t4_nrxq1g < 1)
8299                 t4_nrxq1g = min(nc, NRXQ_1G);
8300
8301 #ifdef TCP_OFFLOAD
8302         if (t4_nofldtxq10g < 1)
8303                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
8304
8305         if (t4_nofldtxq1g < 1)
8306                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
8307
8308         if (t4_nofldrxq10g < 1)
8309                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
8310
8311         if (t4_nofldrxq1g < 1)
8312                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
8313
8314         if (t4_toecaps_allowed == -1)
8315                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
8316 #else
8317         if (t4_toecaps_allowed == -1)
8318                 t4_toecaps_allowed = 0;
8319 #endif
8320
8321 #ifdef DEV_NETMAP
8322         if (t4_nnmtxq10g < 1)
8323                 t4_nnmtxq10g = min(nc, NNMTXQ_10G);
8324
8325         if (t4_nnmtxq1g < 1)
8326                 t4_nnmtxq1g = min(nc, NNMTXQ_1G);
8327
8328         if (t4_nnmrxq10g < 1)
8329                 t4_nnmrxq10g = min(nc, NNMRXQ_10G);
8330
8331         if (t4_nnmrxq1g < 1)
8332                 t4_nnmrxq1g = min(nc, NNMRXQ_1G);
8333 #endif
8334
8335         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
8336                 t4_tmr_idx_10g = TMR_IDX_10G;
8337
8338         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
8339                 t4_pktc_idx_10g = PKTC_IDX_10G;
8340
8341         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
8342                 t4_tmr_idx_1g = TMR_IDX_1G;
8343
8344         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
8345                 t4_pktc_idx_1g = PKTC_IDX_1G;
8346
8347         if (t4_qsize_txq < 128)
8348                 t4_qsize_txq = 128;
8349
8350         if (t4_qsize_rxq < 128)
8351                 t4_qsize_rxq = 128;
8352         while (t4_qsize_rxq & 7)
8353                 t4_qsize_rxq++;
8354
8355         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
8356 }
8357
8358 static struct sx mlu;   /* mod load unload */
8359 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
8360
8361 static int
8362 mod_event(module_t mod, int cmd, void *arg)
8363 {
8364         int rc = 0;
8365         static int loaded = 0;
8366
8367         switch (cmd) {
8368         case MOD_LOAD:
8369                 sx_xlock(&mlu);
8370                 if (loaded++ == 0) {
8371                         t4_sge_modload();
8372                         sx_init(&t4_list_lock, "T4/T5 adapters");
8373                         SLIST_INIT(&t4_list);
8374 #ifdef TCP_OFFLOAD
8375                         sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
8376                         SLIST_INIT(&t4_uld_list);
8377 #endif
8378                         t4_tracer_modload();
8379                         tweak_tunables();
8380                 }
8381                 sx_xunlock(&mlu);
8382                 break;
8383
8384         case MOD_UNLOAD:
8385                 sx_xlock(&mlu);
8386                 if (--loaded == 0) {
8387                         int tries;
8388
8389                         sx_slock(&t4_list_lock);
8390                         if (!SLIST_EMPTY(&t4_list)) {
8391                                 rc = EBUSY;
8392                                 sx_sunlock(&t4_list_lock);
8393                                 goto done_unload;
8394                         }
8395 #ifdef TCP_OFFLOAD
8396                         sx_slock(&t4_uld_list_lock);
8397                         if (!SLIST_EMPTY(&t4_uld_list)) {
8398                                 rc = EBUSY;
8399                                 sx_sunlock(&t4_uld_list_lock);
8400                                 sx_sunlock(&t4_list_lock);
8401                                 goto done_unload;
8402                         }
8403 #endif
8404                         tries = 0;
8405                         while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
8406                                 uprintf("%ju clusters with custom free routine "
8407                                     "still is use.\n", t4_sge_extfree_refs());
8408                                 pause("t4unload", 2 * hz);
8409                         }
8410 #ifdef TCP_OFFLOAD
8411                         sx_sunlock(&t4_uld_list_lock);
8412 #endif
8413                         sx_sunlock(&t4_list_lock);
8414
8415                         if (t4_sge_extfree_refs() == 0) {
8416                                 t4_tracer_modunload();
8417 #ifdef TCP_OFFLOAD
8418                                 sx_destroy(&t4_uld_list_lock);
8419 #endif
8420                                 sx_destroy(&t4_list_lock);
8421                                 t4_sge_modunload();
8422                                 loaded = 0;
8423                         } else {
8424                                 rc = EBUSY;
8425                                 loaded++;       /* undo earlier decrement */
8426                         }
8427                 }
8428 done_unload:
8429                 sx_xunlock(&mlu);
8430                 break;
8431         }
8432
8433         return (rc);
8434 }
8435
8436 static devclass_t t4_devclass, t5_devclass;
8437 static devclass_t cxgbe_devclass, cxl_devclass;
8438
8439 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
8440 MODULE_VERSION(t4nex, 1);
8441 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
8442
8443 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
8444 MODULE_VERSION(t5nex, 1);
8445 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
8446
8447 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
8448 MODULE_VERSION(cxgbe, 1);
8449
8450 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
8451 MODULE_VERSION(cxl, 1);