sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/conf.h>
  36 #include <sys/priv.h>
  37 #include <sys/kernel.h>
  38 #include <sys/bus.h>
  39 #include <sys/module.h>
  40 #include <sys/malloc.h>
  41 #include <sys/queue.h>
  42 #include <sys/taskqueue.h>
  43 #include <sys/pciio.h>
  44 #include <dev/pci/pcireg.h>
  45 #include <dev/pci/pcivar.h>
  46 #include <dev/pci/pci_private.h>
  47 #include <sys/firmware.h>
  48 #include <sys/sbuf.h>
  49 #include <sys/smp.h>
  50 #include <sys/socket.h>
  51 #include <sys/sockio.h>
  52 #include <sys/sysctl.h>
  53 #include <net/ethernet.h>
  54 #include <net/if.h>
  55 #include <net/if_types.h>
  56 #include <net/if_dl.h>
  57 #include <net/if_vlan_var.h>
  58 #if defined(__i386__) || defined(__amd64__)
  59 #include <vm/vm.h>
  60 #include <vm/pmap.h>
  61 #endif
  62
  63 #include "common/common.h"
  64 #include "common/t4_msg.h"
  65 #include "common/t4_regs.h"
  66 #include "common/t4_regs_values.h"
  67 #include "t4_ioctl.h"
  68 #include "t4_l2t.h"
  69 #include "t4_mp_ring.h"
  70
  71 /* T4 bus driver interface */
  72 static int t4_probe(device_t);
  73 static int t4_attach(device_t);
  74 static int t4_detach(device_t);
  75 static device_method_t t4_methods[] = {
  76         DEVMETHOD(device_probe,         t4_probe),
  77         DEVMETHOD(device_attach,        t4_attach),
  78         DEVMETHOD(device_detach,        t4_detach),
  79
  80         DEVMETHOD_END
  81 };
  82 static driver_t t4_driver = {
  83         "t4nex",
  84         t4_methods,
  85         sizeof(struct adapter)
  86 };
  87
  88
  89 /* T4 port (cxgbe) interface */
  90 static int cxgbe_probe(device_t);
  91 static int cxgbe_attach(device_t);
  92 static int cxgbe_detach(device_t);
  93 static device_method_t cxgbe_methods[] = {
  94         DEVMETHOD(device_probe,         cxgbe_probe),
  95         DEVMETHOD(device_attach,        cxgbe_attach),
  96         DEVMETHOD(device_detach,        cxgbe_detach),
  97         { 0, 0 }
  98 };
  99 static driver_t cxgbe_driver = {
 100         "cxgbe",
 101         cxgbe_methods,
 102         sizeof(struct port_info)
 103 };
 104
 105 static d_ioctl_t t4_ioctl;
 106 static d_open_t t4_open;
 107 static d_close_t t4_close;
 108
 109 static struct cdevsw t4_cdevsw = {
 110        .d_version = D_VERSION,
 111        .d_flags = 0,
 112        .d_open = t4_open,
 113        .d_close = t4_close,
 114        .d_ioctl = t4_ioctl,
 115        .d_name = "t4nex",
 116 };
 117
 118 /* T5 bus driver interface */
 119 static int t5_probe(device_t);
 120 static device_method_t t5_methods[] = {
 121         DEVMETHOD(device_probe,         t5_probe),
 122         DEVMETHOD(device_attach,        t4_attach),
 123         DEVMETHOD(device_detach,        t4_detach),
 124
 125         DEVMETHOD_END
 126 };
 127 static driver_t t5_driver = {
 128         "t5nex",
 129         t5_methods,
 130         sizeof(struct adapter)
 131 };
 132
 133
 134 /* T5 port (cxl) interface */
 135 static driver_t cxl_driver = {
 136         "cxl",
 137         cxgbe_methods,
 138         sizeof(struct port_info)
 139 };
 140
 141 static struct cdevsw t5_cdevsw = {
 142        .d_version = D_VERSION,
 143        .d_flags = 0,
 144        .d_open = t4_open,
 145        .d_close = t4_close,
 146        .d_ioctl = t4_ioctl,
 147        .d_name = "t5nex",
 148 };
 149
 150 /* ifnet + media interface */
 151 static void cxgbe_init(void *);
 152 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 153 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 154 static void cxgbe_qflush(struct ifnet *);
 155 static uint64_t cxgbe_get_counter(struct ifnet *, ift_counter);
 156 static int cxgbe_media_change(struct ifnet *);
 157 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 158
 159 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
 160
 161 /*
 162  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 163  * then ADAPTER_LOCK, then t4_uld_list_lock.
 164  */
 165 static struct sx t4_list_lock;
 166 SLIST_HEAD(, adapter) t4_list;
 167 #ifdef TCP_OFFLOAD
 168 static struct sx t4_uld_list_lock;
 169 SLIST_HEAD(, uld_info) t4_uld_list;
 170 #endif
 171
 172 /*
 173  * Tunables.  See tweak_tunables() too.
 174  *
 175  * Each tunable is set to a default value here if it's known at compile-time.
 176  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
 177  * provide a reasonable default when the driver is loaded.
 178  *
 179  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
 180  * T5 are under hw.cxl.
 181  */
 182
 183 /*
 184  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 185  */
 186 #define NTXQ_10G 16
 187 static int t4_ntxq10g = -1;
 188 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 189
 190 #define NRXQ_10G 8
 191 static int t4_nrxq10g = -1;
 192 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 193
 194 #define NTXQ_1G 4
 195 static int t4_ntxq1g = -1;
 196 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 197
 198 #define NRXQ_1G 2
 199 static int t4_nrxq1g = -1;
 200 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 201
 202 static int t4_rsrv_noflowq = 0;
 203 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
 204
 205 #ifdef TCP_OFFLOAD
 206 #define NOFLDTXQ_10G 8
 207 static int t4_nofldtxq10g = -1;
 208 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 209
 210 #define NOFLDRXQ_10G 2
 211 static int t4_nofldrxq10g = -1;
 212 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 213
 214 #define NOFLDTXQ_1G 2
 215 static int t4_nofldtxq1g = -1;
 216 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 217
 218 #define NOFLDRXQ_1G 1
 219 static int t4_nofldrxq1g = -1;
 220 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 221 #endif
 222
 223 #ifdef DEV_NETMAP
 224 #define NNMTXQ_10G 2
 225 static int t4_nnmtxq10g = -1;
 226 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g);
 227
 228 #define NNMRXQ_10G 2
 229 static int t4_nnmrxq10g = -1;
 230 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g);
 231
 232 #define NNMTXQ_1G 1
 233 static int t4_nnmtxq1g = -1;
 234 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g);
 235
 236 #define NNMRXQ_1G 1
 237 static int t4_nnmrxq1g = -1;
 238 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g);
 239 #endif
 240
 241 /*
 242  * Holdoff parameters for 10G and 1G ports.
 243  */
 244 #define TMR_IDX_10G 1
 245 static int t4_tmr_idx_10g = TMR_IDX_10G;
 246 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 247
 248 #define PKTC_IDX_10G (-1)
 249 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 250 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 251
 252 #define TMR_IDX_1G 1
 253 static int t4_tmr_idx_1g = TMR_IDX_1G;
 254 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 255
 256 #define PKTC_IDX_1G (-1)
 257 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 258 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 259
 260 /*
 261  * Size (# of entries) of each tx and rx queue.
 262  */
 263 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 264 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 265
 266 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 267 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 268
 269 /*
 270  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 271  */
 272 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 273 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 274
 275 /*
 276  * Configuration file.
 277  */
 278 #define DEFAULT_CF      "default"
 279 #define FLASH_CF        "flash"
 280 #define UWIRE_CF        "uwire"
 281 #define FPGA_CF         "fpga"
 282 static char t4_cfg_file[32] = DEFAULT_CF;
 283 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 284
 285 /*
 286  * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively).
 287  * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
 288  * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
 289  *            mark or when signalled to do so, 0 to never emit PAUSE.
 290  */
 291 static int t4_pause_settings = PAUSE_TX | PAUSE_RX;
 292 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings);
 293
 294 /*
 295  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
 296  * encouraged respectively).
 297  */
 298 static unsigned int t4_fw_install = 1;
 299 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 300
 301 /*
 302  * ASIC features that will be used.  Disable the ones you don't want so that the
 303  * chip resources aren't wasted on features that will not be used.
 304  */
 305 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 306 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 307
 308 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 309 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 310
 311 static int t4_toecaps_allowed = -1;
 312 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 313
 314 static int t4_rdmacaps_allowed = 0;
 315 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 316
 317 static int t4_iscsicaps_allowed = 0;
 318 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 319
 320 static int t4_fcoecaps_allowed = 0;
 321 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 322
 323 static int t5_write_combine = 0;
 324 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 325
 326 struct intrs_and_queues {
 327         uint16_t intr_type;     /* INTx, MSI, or MSI-X */
 328         uint16_t nirq;          /* Total # of vectors */
 329         uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
 330         uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */
 331         uint16_t ntxq10g;       /* # of NIC txq's for each 10G port */
 332         uint16_t nrxq10g;       /* # of NIC rxq's for each 10G port */
 333         uint16_t ntxq1g;        /* # of NIC txq's for each 1G port */
 334         uint16_t nrxq1g;        /* # of NIC rxq's for each 1G port */
 335         uint16_t rsrv_noflowq;  /* Flag whether to reserve queue 0 */
 336 #ifdef TCP_OFFLOAD
 337         uint16_t nofldtxq10g;   /* # of TOE txq's for each 10G port */
 338         uint16_t nofldrxq10g;   /* # of TOE rxq's for each 10G port */
 339         uint16_t nofldtxq1g;    /* # of TOE txq's for each 1G port */
 340         uint16_t nofldrxq1g;    /* # of TOE rxq's for each 1G port */
 341 #endif
 342 #ifdef DEV_NETMAP
 343         uint16_t nnmtxq10g;     /* # of netmap txq's for each 10G port */
 344         uint16_t nnmrxq10g;     /* # of netmap rxq's for each 10G port */
 345         uint16_t nnmtxq1g;      /* # of netmap txq's for each 1G port */
 346         uint16_t nnmrxq1g;      /* # of netmap rxq's for each 1G port */
 347 #endif
 348 };
 349
 350 struct filter_entry {
 351         uint32_t valid:1;       /* filter allocated and valid */
 352         uint32_t locked:1;      /* filter is administratively locked */
 353         uint32_t pending:1;     /* filter action is pending firmware reply */
 354         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 355         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 356
 357         struct t4_filter_specification fs;
 358 };
 359
 360 static int map_bars_0_and_4(struct adapter *);
 361 static int map_bar_2(struct adapter *);
 362 static void setup_memwin(struct adapter *);
 363 static int validate_mem_range(struct adapter *, uint32_t, int);
 364 static int fwmtype_to_hwmtype(int);
 365 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
 366     uint32_t *);
 367 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
 368 static uint32_t position_memwin(struct adapter *, int, uint32_t);
 369 static int cfg_itype_and_nqueues(struct adapter *, int, int,
 370     struct intrs_and_queues *);
 371 static int prep_firmware(struct adapter *);
 372 static int partition_resources(struct adapter *, const struct firmware *,
 373     const char *);
 374 static int get_params__pre_init(struct adapter *);
 375 static int get_params__post_init(struct adapter *);
 376 static int set_params__post_init(struct adapter *);
 377 static void t4_set_desc(struct adapter *);
 378 static void build_medialist(struct port_info *, struct ifmedia *);
 379 static int cxgbe_init_synchronized(struct port_info *);
 380 static int cxgbe_uninit_synchronized(struct port_info *);
 381 static int setup_intr_handlers(struct adapter *);
 382 static void quiesce_txq(struct adapter *, struct sge_txq *);
 383 static void quiesce_wrq(struct adapter *, struct sge_wrq *);
 384 static void quiesce_iq(struct adapter *, struct sge_iq *);
 385 static void quiesce_fl(struct adapter *, struct sge_fl *);
 386 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 387     driver_intr_t *, void *, char *);
 388 static int t4_free_irq(struct adapter *, struct irq *);
 389 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
 390     unsigned int);
 391 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 392 static void cxgbe_refresh_stats(struct adapter *, struct port_info *);
 393 static void cxgbe_tick(void *);
 394 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 395 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
 396     struct mbuf *);
 397 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 398 static int fw_msg_not_handled(struct adapter *, const __be64 *);
 399 static int t4_sysctls(struct adapter *);
 400 static int cxgbe_sysctls(struct port_info *);
 401 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 402 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 403 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
 404 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 405 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 406 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 407 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 408 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 409 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
 410 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 411 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 412 #ifdef SBUF_DRAIN
 413 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 414 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 415 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 416 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
 417 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
 418 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 419 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 420 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 421 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 422 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 423 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 424 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 425 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
 426 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 427 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
 428 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 429 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 430 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 431 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 432 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 433 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 434 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
 435 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 436 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 437 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 438 #endif
 439 static uint32_t fconf_to_mode(uint32_t);
 440 static uint32_t mode_to_fconf(uint32_t);
 441 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 442 static int get_filter_mode(struct adapter *, uint32_t *);
 443 static int set_filter_mode(struct adapter *, uint32_t);
 444 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 445 static int get_filter(struct adapter *, struct t4_filter *);
 446 static int set_filter(struct adapter *, struct t4_filter *);
 447 static int del_filter(struct adapter *, struct t4_filter *);
 448 static void clear_filter(struct filter_entry *);
 449 static int set_filter_wr(struct adapter *, int);
 450 static int del_filter_wr(struct adapter *, int);
 451 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 452 static int load_fw(struct adapter *, struct t4_data *);
 453 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 454 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 455 static int set_sched_class(struct adapter *, struct t4_sched_params *);
 456 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
 457 #ifdef TCP_OFFLOAD
 458 static int toe_capability(struct port_info *, int);
 459 #endif
 460 static int mod_event(module_t, int, void *);
 461
 462 struct {
 463         uint16_t device;
 464         char *desc;
 465 } t4_pciids[] = {
 466         {0xa000, "Chelsio Terminator 4 FPGA"},
 467         {0x4400, "Chelsio T440-dbg"},
 468         {0x4401, "Chelsio T420-CR"},
 469         {0x4402, "Chelsio T422-CR"},
 470         {0x4403, "Chelsio T440-CR"},
 471         {0x4404, "Chelsio T420-BCH"},
 472         {0x4405, "Chelsio T440-BCH"},
 473         {0x4406, "Chelsio T440-CH"},
 474         {0x4407, "Chelsio T420-SO"},
 475         {0x4408, "Chelsio T420-CX"},
 476         {0x4409, "Chelsio T420-BT"},
 477         {0x440a, "Chelsio T404-BT"},
 478         {0x440e, "Chelsio T440-LP-CR"},
 479 }, t5_pciids[] = {
 480         {0xb000, "Chelsio Terminator 5 FPGA"},
 481         {0x5400, "Chelsio T580-dbg"},
 482         {0x5401,  "Chelsio T520-CR"},           /* 2 x 10G */
 483         {0x5402,  "Chelsio T522-CR"},           /* 2 x 10G, 2 X 1G */
 484         {0x5403,  "Chelsio T540-CR"},           /* 4 x 10G */
 485         {0x5407,  "Chelsio T520-SO"},           /* 2 x 10G, nomem */
 486         {0x5409,  "Chelsio T520-BT"},           /* 2 x 10GBaseT */
 487         {0x540a,  "Chelsio T504-BT"},           /* 4 x 1G */
 488         {0x540d,  "Chelsio T580-CR"},           /* 2 x 40G */
 489         {0x540e,  "Chelsio T540-LP-CR"},        /* 4 x 10G */
 490         {0x5410,  "Chelsio T580-LP-CR"},        /* 2 x 40G */
 491         {0x5411,  "Chelsio T520-LL-CR"},        /* 2 x 10G */
 492         {0x5412,  "Chelsio T560-CR"},           /* 1 x 40G, 2 x 10G */
 493         {0x5414,  "Chelsio T580-LP-SO-CR"},     /* 2 x 40G, nomem */
 494         {0x5415,  "Chelsio T502-BT"},           /* 2 x 1G */
 495 #ifdef notyet
 496         {0x5404,  "Chelsio T520-BCH"},
 497         {0x5405,  "Chelsio T540-BCH"},
 498         {0x5406,  "Chelsio T540-CH"},
 499         {0x5408,  "Chelsio T520-CX"},
 500         {0x540b,  "Chelsio B520-SR"},
 501         {0x540c,  "Chelsio B504-BT"},
 502         {0x540f,  "Chelsio Amsterdam"},
 503         {0x5413,  "Chelsio T580-CHR"},
 504 #endif
 505 };
 506
 507 #ifdef TCP_OFFLOAD
 508 /*
 509  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 510  * exactly the same for both rxq and ofld_rxq.
 511  */
 512 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 513 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 514 #endif
 515
 516 /* No easy way to include t4_msg.h before adapter.h so we check this way */
 517 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
 518 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
 519
 520 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
 521
 522 static int
 523 t4_probe(device_t dev)
 524 {
 525         int i;
 526         uint16_t v = pci_get_vendor(dev);
 527         uint16_t d = pci_get_device(dev);
 528         uint8_t f = pci_get_function(dev);
 529
 530         if (v != PCI_VENDOR_ID_CHELSIO)
 531                 return (ENXIO);
 532
 533         /* Attach only to PF0 of the FPGA */
 534         if (d == 0xa000 && f != 0)
 535                 return (ENXIO);
 536
 537         for (i = 0; i < nitems(t4_pciids); i++) {
 538                 if (d == t4_pciids[i].device) {
 539                         device_set_desc(dev, t4_pciids[i].desc);
 540                         return (BUS_PROBE_DEFAULT);
 541                 }
 542         }
 543
 544         return (ENXIO);
 545 }
 546
 547 static int
 548 t5_probe(device_t dev)
 549 {
 550         int i;
 551         uint16_t v = pci_get_vendor(dev);
 552         uint16_t d = pci_get_device(dev);
 553         uint8_t f = pci_get_function(dev);
 554
 555         if (v != PCI_VENDOR_ID_CHELSIO)
 556                 return (ENXIO);
 557
 558         /* Attach only to PF0 of the FPGA */
 559         if (d == 0xb000 && f != 0)
 560                 return (ENXIO);
 561
 562         for (i = 0; i < nitems(t5_pciids); i++) {
 563                 if (d == t5_pciids[i].device) {
 564                         device_set_desc(dev, t5_pciids[i].desc);
 565                         return (BUS_PROBE_DEFAULT);
 566                 }
 567         }
 568
 569         return (ENXIO);
 570 }
 571
 572 static int
 573 t4_attach(device_t dev)
 574 {
 575         struct adapter *sc;
 576         int rc = 0, i, n10g, n1g, rqidx, tqidx;
 577         struct intrs_and_queues iaq;
 578         struct sge *s;
 579 #ifdef TCP_OFFLOAD
 580         int ofld_rqidx, ofld_tqidx;
 581 #endif
 582 #ifdef DEV_NETMAP
 583         int nm_rqidx, nm_tqidx;
 584 #endif
 585         const char *pcie_ts;
 586
 587         sc = device_get_softc(dev);
 588         sc->dev = dev;
 589
 590         pci_enable_busmaster(dev);
 591         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 592                 uint32_t v;
 593
 594                 pci_set_max_read_req(dev, 4096);
 595                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 596                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 597                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 598
 599                 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
 600         }
 601
 602         sc->traceq = -1;
 603         mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
 604         snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
 605             device_get_nameunit(dev));
 606
 607         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 608             device_get_nameunit(dev));
 609         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 610         sx_xlock(&t4_list_lock);
 611         SLIST_INSERT_HEAD(&t4_list, sc, link);
 612         sx_xunlock(&t4_list_lock);
 613
 614         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 615         TAILQ_INIT(&sc->sfl);
 616         callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
 617
 618         mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF);
 619
 620         rc = map_bars_0_and_4(sc);
 621         if (rc != 0)
 622                 goto done; /* error message displayed already */
 623
 624         /*
 625          * This is the real PF# to which we're attaching.  Works from within PCI
 626          * passthrough environments too, where pci_get_function() could return a
 627          * different PF# depending on the passthrough configuration.  We need to
 628          * use the real PF# in all our communication with the firmware.
 629          */
 630         sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 631         sc->mbox = sc->pf;
 632
 633         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 634         sc->an_handler = an_not_handled;
 635         for (i = 0; i < nitems(sc->cpl_handler); i++)
 636                 sc->cpl_handler[i] = cpl_not_handled;
 637         for (i = 0; i < nitems(sc->fw_msg_handler); i++)
 638                 sc->fw_msg_handler[i] = fw_msg_not_handled;
 639         t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 640         t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
 641         t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt);
 642         t4_init_sge_cpl_handlers(sc);
 643
 644         /* Prepare the adapter for operation */
 645         rc = -t4_prep_adapter(sc);
 646         if (rc != 0) {
 647                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 648                 goto done;
 649         }
 650
 651         /*
 652          * Do this really early, with the memory windows set up even before the
 653          * character device.  The userland tool's register i/o and mem read
 654          * will work even in "recovery mode".
 655          */
 656         setup_memwin(sc);
 657         sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
 658             device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
 659             device_get_nameunit(dev));
 660         if (sc->cdev == NULL)
 661                 device_printf(dev, "failed to create nexus char device.\n");
 662         else
 663                 sc->cdev->si_drv1 = sc;
 664
 665         /* Go no further if recovery mode has been requested. */
 666         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 667                 device_printf(dev, "recovery mode.\n");
 668                 goto done;
 669         }
 670
 671         /* Prepare the firmware for operation */
 672         rc = prep_firmware(sc);
 673         if (rc != 0)
 674                 goto done; /* error message displayed already */
 675
 676         rc = get_params__post_init(sc);
 677         if (rc != 0)
 678                 goto done; /* error message displayed already */
 679
 680         rc = set_params__post_init(sc);
 681         if (rc != 0)
 682                 goto done; /* error message displayed already */
 683
 684         rc = map_bar_2(sc);
 685         if (rc != 0)
 686                 goto done; /* error message displayed already */
 687
 688         rc = t4_create_dma_tag(sc);
 689         if (rc != 0)
 690                 goto done; /* error message displayed already */
 691
 692         /*
 693          * First pass over all the ports - allocate VIs and initialize some
 694          * basic parameters like mac address, port type, etc.  We also figure
 695          * out whether a port is 10G or 1G and use that information when
 696          * calculating how many interrupts to attempt to allocate.
 697          */
 698         n10g = n1g = 0;
 699         for_each_port(sc, i) {
 700                 struct port_info *pi;
 701
 702                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 703                 sc->port[i] = pi;
 704
 705                 /* These must be set before t4_port_init */
 706                 pi->adapter = sc;
 707                 pi->port_id = i;
 708
 709                 /* Allocate the vi and initialize parameters like mac addr */
 710                 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 711                 if (rc != 0) {
 712                         device_printf(dev, "unable to initialize port %d: %d\n",
 713                             i, rc);
 714                         free(pi, M_CXGBE);
 715                         sc->port[i] = NULL;
 716                         goto done;
 717                 }
 718
 719                 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX);
 720                 pi->link_cfg.requested_fc |= t4_pause_settings;
 721                 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX);
 722                 pi->link_cfg.fc |= t4_pause_settings;
 723
 724                 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
 725                 if (rc != 0) {
 726                         device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
 727                         free(pi, M_CXGBE);
 728                         sc->port[i] = NULL;
 729                         goto done;
 730                 }
 731
 732                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 733                     device_get_nameunit(dev), i);
 734                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 735                 sc->chan_map[pi->tx_chan] = i;
 736
 737                 if (is_10G_port(pi) || is_40G_port(pi)) {
 738                         n10g++;
 739                         pi->tmr_idx = t4_tmr_idx_10g;
 740                         pi->pktc_idx = t4_pktc_idx_10g;
 741                 } else {
 742                         n1g++;
 743                         pi->tmr_idx = t4_tmr_idx_1g;
 744                         pi->pktc_idx = t4_pktc_idx_1g;
 745                 }
 746
 747                 pi->xact_addr_filt = -1;
 748                 pi->linkdnrc = -1;
 749
 750                 pi->qsize_rxq = t4_qsize_rxq;
 751                 pi->qsize_txq = t4_qsize_txq;
 752
 753                 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
 754                 if (pi->dev == NULL) {
 755                         device_printf(dev,
 756                             "failed to add device for port %d.\n", i);
 757                         rc = ENXIO;
 758                         goto done;
 759                 }
 760                 device_set_softc(pi->dev, pi);
 761         }
 762
 763         /*
 764          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 765          */
 766         rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
 767         if (rc != 0)
 768                 goto done; /* error message displayed already */
 769
 770         sc->intr_type = iaq.intr_type;
 771         sc->intr_count = iaq.nirq;
 772
 773         s = &sc->sge;
 774         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 775         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 776         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
 777         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 778         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
 779 #ifdef TCP_OFFLOAD
 780         if (is_offload(sc)) {
 781                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 782                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 783                 s->neq += s->nofldtxq + s->nofldrxq;
 784                 s->niq += s->nofldrxq;
 785
 786                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 787                     M_CXGBE, M_ZERO | M_WAITOK);
 788                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 789                     M_CXGBE, M_ZERO | M_WAITOK);
 790         }
 791 #endif
 792 #ifdef DEV_NETMAP
 793         s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g;
 794         s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g;
 795         s->neq += s->nnmtxq + s->nnmrxq;
 796         s->niq += s->nnmrxq;
 797
 798         s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
 799             M_CXGBE, M_ZERO | M_WAITOK);
 800         s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
 801             M_CXGBE, M_ZERO | M_WAITOK);
 802 #endif
 803
 804         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 805             M_ZERO | M_WAITOK);
 806         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 807             M_ZERO | M_WAITOK);
 808         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 809             M_ZERO | M_WAITOK);
 810         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 811             M_ZERO | M_WAITOK);
 812         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 813             M_ZERO | M_WAITOK);
 814
 815         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 816             M_ZERO | M_WAITOK);
 817
 818         t4_init_l2t(sc, M_WAITOK);
 819
 820         /*
 821          * Second pass over the ports.  This time we know the number of rx and
 822          * tx queues that each port should get.
 823          */
 824         rqidx = tqidx = 0;
 825 #ifdef TCP_OFFLOAD
 826         ofld_rqidx = ofld_tqidx = 0;
 827 #endif
 828 #ifdef DEV_NETMAP
 829         nm_rqidx = nm_tqidx = 0;
 830 #endif
 831         for_each_port(sc, i) {
 832                 struct port_info *pi = sc->port[i];
 833
 834                 if (pi == NULL)
 835                         continue;
 836
 837                 pi->first_rxq = rqidx;
 838                 pi->first_txq = tqidx;
 839                 if (is_10G_port(pi) || is_40G_port(pi)) {
 840                         pi->flags |= iaq.intr_flags_10g;
 841                         pi->nrxq = iaq.nrxq10g;
 842                         pi->ntxq = iaq.ntxq10g;
 843                 } else {
 844                         pi->flags |= iaq.intr_flags_1g;
 845                         pi->nrxq = iaq.nrxq1g;
 846                         pi->ntxq = iaq.ntxq1g;
 847                 }
 848
 849                 if (pi->ntxq > 1)
 850                         pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
 851                 else
 852                         pi->rsrv_noflowq = 0;
 853
 854                 rqidx += pi->nrxq;
 855                 tqidx += pi->ntxq;
 856 #ifdef TCP_OFFLOAD
 857                 if (is_offload(sc)) {
 858                         pi->first_ofld_rxq = ofld_rqidx;
 859                         pi->first_ofld_txq = ofld_tqidx;
 860                         if (is_10G_port(pi) || is_40G_port(pi)) {
 861                                 pi->nofldrxq = iaq.nofldrxq10g;
 862                                 pi->nofldtxq = iaq.nofldtxq10g;
 863                         } else {
 864                                 pi->nofldrxq = iaq.nofldrxq1g;
 865                                 pi->nofldtxq = iaq.nofldtxq1g;
 866                         }
 867                         ofld_rqidx += pi->nofldrxq;
 868                         ofld_tqidx += pi->nofldtxq;
 869                 }
 870 #endif
 871 #ifdef DEV_NETMAP
 872                 pi->first_nm_rxq = nm_rqidx;
 873                 pi->first_nm_txq = nm_tqidx;
 874                 if (is_10G_port(pi) || is_40G_port(pi)) {
 875                         pi->nnmrxq = iaq.nnmrxq10g;
 876                         pi->nnmtxq = iaq.nnmtxq10g;
 877                 } else {
 878                         pi->nnmrxq = iaq.nnmrxq1g;
 879                         pi->nnmtxq = iaq.nnmtxq1g;
 880                 }
 881                 nm_rqidx += pi->nnmrxq;
 882                 nm_tqidx += pi->nnmtxq;
 883 #endif
 884         }
 885
 886         rc = setup_intr_handlers(sc);
 887         if (rc != 0) {
 888                 device_printf(dev,
 889                     "failed to setup interrupt handlers: %d\n", rc);
 890                 goto done;
 891         }
 892
 893         rc = bus_generic_attach(dev);
 894         if (rc != 0) {
 895                 device_printf(dev,
 896                     "failed to attach all child ports: %d\n", rc);
 897                 goto done;
 898         }
 899
 900         switch (sc->params.pci.speed) {
 901                 case 0x1:
 902                         pcie_ts = "2.5";
 903                         break;
 904                 case 0x2:
 905                         pcie_ts = "5.0";
 906                         break;
 907                 case 0x3:
 908                         pcie_ts = "8.0";
 909                         break;
 910                 default:
 911                         pcie_ts = "??";
 912                         break;
 913         }
 914         device_printf(dev,
 915             "PCIe x%d (%s GTS/s) (%d), %d ports, %d %s interrupt%s, %d eq, %d iq\n",
 916             sc->params.pci.width, pcie_ts, sc->params.pci.speed,
 917             sc->params.nports, sc->intr_count,
 918             sc->intr_type == INTR_MSIX ? "MSI-X" :
 919             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
 920             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
 921
 922         t4_set_desc(sc);
 923
 924 done:
 925         if (rc != 0 && sc->cdev) {
 926                 /* cdev was created and so cxgbetool works; recover that way. */
 927                 device_printf(dev,
 928                     "error during attach, adapter is now in recovery mode.\n");
 929                 rc = 0;
 930         }
 931
 932         if (rc != 0)
 933                 t4_detach(dev);
 934         else
 935                 t4_sysctls(sc);
 936
 937         return (rc);
 938 }
 939
 940 /*
 941  * Idempotent
 942  */
 943 static int
 944 t4_detach(device_t dev)
 945 {
 946         struct adapter *sc;
 947         struct port_info *pi;
 948         int i, rc;
 949
 950         sc = device_get_softc(dev);
 951
 952         if (sc->flags & FULL_INIT_DONE)
 953                 t4_intr_disable(sc);
 954
 955         if (sc->cdev) {
 956                 destroy_dev(sc->cdev);
 957                 sc->cdev = NULL;
 958         }
 959
 960         rc = bus_generic_detach(dev);
 961         if (rc) {
 962                 device_printf(dev,
 963                     "failed to detach child devices: %d\n", rc);
 964                 return (rc);
 965         }
 966
 967         for (i = 0; i < sc->intr_count; i++)
 968                 t4_free_irq(sc, &sc->irq[i]);
 969
 970         for (i = 0; i < MAX_NPORTS; i++) {
 971                 pi = sc->port[i];
 972                 if (pi) {
 973                         t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid);
 974                         if (pi->dev)
 975                                 device_delete_child(dev, pi->dev);
 976
 977                         mtx_destroy(&pi->pi_lock);
 978                         free(pi, M_CXGBE);
 979                 }
 980         }
 981
 982         if (sc->flags & FULL_INIT_DONE)
 983                 adapter_full_uninit(sc);
 984
 985         if (sc->flags & FW_OK)
 986                 t4_fw_bye(sc, sc->mbox);
 987
 988         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
 989                 pci_release_msi(dev);
 990
 991         if (sc->regs_res)
 992                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
 993                     sc->regs_res);
 994
 995         if (sc->udbs_res)
 996                 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
 997                     sc->udbs_res);
 998
 999         if (sc->msix_res)
1000                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
1001                     sc->msix_res);
1002
1003         if (sc->l2t)
1004                 t4_free_l2t(sc->l2t);
1005
1006 #ifdef TCP_OFFLOAD
1007         free(sc->sge.ofld_rxq, M_CXGBE);
1008         free(sc->sge.ofld_txq, M_CXGBE);
1009 #endif
1010 #ifdef DEV_NETMAP
1011         free(sc->sge.nm_rxq, M_CXGBE);
1012         free(sc->sge.nm_txq, M_CXGBE);
1013 #endif
1014         free(sc->irq, M_CXGBE);
1015         free(sc->sge.rxq, M_CXGBE);
1016         free(sc->sge.txq, M_CXGBE);
1017         free(sc->sge.ctrlq, M_CXGBE);
1018         free(sc->sge.iqmap, M_CXGBE);
1019         free(sc->sge.eqmap, M_CXGBE);
1020         free(sc->tids.ftid_tab, M_CXGBE);
1021         t4_destroy_dma_tag(sc);
1022         if (mtx_initialized(&sc->sc_lock)) {
1023                 sx_xlock(&t4_list_lock);
1024                 SLIST_REMOVE(&t4_list, sc, adapter, link);
1025                 sx_xunlock(&t4_list_lock);
1026                 mtx_destroy(&sc->sc_lock);
1027         }
1028
1029         if (mtx_initialized(&sc->tids.ftid_lock))
1030                 mtx_destroy(&sc->tids.ftid_lock);
1031         if (mtx_initialized(&sc->sfl_lock))
1032                 mtx_destroy(&sc->sfl_lock);
1033         if (mtx_initialized(&sc->ifp_lock))
1034                 mtx_destroy(&sc->ifp_lock);
1035         if (mtx_initialized(&sc->regwin_lock))
1036                 mtx_destroy(&sc->regwin_lock);
1037
1038         bzero(sc, sizeof(*sc));
1039
1040         return (0);
1041 }
1042
1043 static int
1044 cxgbe_probe(device_t dev)
1045 {
1046         char buf[128];
1047         struct port_info *pi = device_get_softc(dev);
1048
1049         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
1050         device_set_desc_copy(dev, buf);
1051
1052         return (BUS_PROBE_DEFAULT);
1053 }
1054
1055 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
1056     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1057     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
1058 #define T4_CAP_ENABLE (T4_CAP)
1059
1060 static int
1061 cxgbe_attach(device_t dev)
1062 {
1063         struct port_info *pi = device_get_softc(dev);
1064         struct ifnet *ifp;
1065         char *s;
1066         int n, o;
1067
1068         /* Allocate an ifnet and set it up */
1069         ifp = if_alloc(IFT_ETHER);
1070         if (ifp == NULL) {
1071                 device_printf(dev, "Cannot allocate ifnet\n");
1072                 return (ENOMEM);
1073         }
1074         pi->ifp = ifp;
1075         ifp->if_softc = pi;
1076
1077         callout_init(&pi->tick, CALLOUT_MPSAFE);
1078
1079         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1080         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1081
1082         ifp->if_init = cxgbe_init;
1083         ifp->if_ioctl = cxgbe_ioctl;
1084         ifp->if_transmit = cxgbe_transmit;
1085         ifp->if_qflush = cxgbe_qflush;
1086         ifp->if_get_counter = cxgbe_get_counter;
1087
1088         ifp->if_capabilities = T4_CAP;
1089 #ifdef TCP_OFFLOAD
1090         if (is_offload(pi->adapter))
1091                 ifp->if_capabilities |= IFCAP_TOE;
1092 #endif
1093         ifp->if_capenable = T4_CAP_ENABLE;
1094         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1095             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1096
1097         ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1098         ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS;
1099         ifp->if_hw_tsomaxsegsize = 65536;
1100
1101         /* Initialize ifmedia for this port */
1102         ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1103             cxgbe_media_status);
1104         build_medialist(pi, &pi->media);
1105
1106         pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1107             EVENTHANDLER_PRI_ANY);
1108
1109         ether_ifattach(ifp, pi->hw_addr);
1110
1111         n = 128;
1112         s = malloc(n, M_CXGBE, M_WAITOK);
1113         o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq);
1114         MPASS(n > o);
1115 #ifdef TCP_OFFLOAD
1116         if (is_offload(pi->adapter)) {
1117                 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)",
1118                     pi->nofldtxq, pi->nofldrxq);
1119                 MPASS(n > o);
1120         }
1121 #endif
1122 #ifdef DEV_NETMAP
1123         o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq,
1124             pi->nnmrxq);
1125         MPASS(n > o);
1126 #endif
1127         device_printf(dev, "%s\n", s);
1128         free(s, M_CXGBE);
1129
1130 #ifdef DEV_NETMAP
1131         /* nm_media handled here to keep implementation private to this file */
1132         ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change,
1133             cxgbe_media_status);
1134         build_medialist(pi, &pi->nm_media);
1135         create_netmap_ifnet(pi);        /* logs errors it something fails */
1136 #endif
1137         cxgbe_sysctls(pi);
1138
1139         return (0);
1140 }
1141
1142 static int
1143 cxgbe_detach(device_t dev)
1144 {
1145         struct port_info *pi = device_get_softc(dev);
1146         struct adapter *sc = pi->adapter;
1147         struct ifnet *ifp = pi->ifp;
1148
1149         /* Tell if_ioctl and if_init that the port is going away */
1150         ADAPTER_LOCK(sc);
1151         SET_DOOMED(pi);
1152         wakeup(&sc->flags);
1153         while (IS_BUSY(sc))
1154                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
1155         SET_BUSY(sc);
1156 #ifdef INVARIANTS
1157         sc->last_op = "t4detach";
1158         sc->last_op_thr = curthread;
1159 #endif
1160         ADAPTER_UNLOCK(sc);
1161
1162         if (pi->flags & HAS_TRACEQ) {
1163                 sc->traceq = -1;        /* cloner should not create ifnet */
1164                 t4_tracer_port_detach(sc);
1165         }
1166
1167         if (pi->vlan_c)
1168                 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
1169
1170         PORT_LOCK(pi);
1171         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1172         callout_stop(&pi->tick);
1173         PORT_UNLOCK(pi);
1174         callout_drain(&pi->tick);
1175
1176         /* Let detach proceed even if these fail. */
1177         cxgbe_uninit_synchronized(pi);
1178         port_full_uninit(pi);
1179
1180         ifmedia_removeall(&pi->media);
1181         ether_ifdetach(pi->ifp);
1182         if_free(pi->ifp);
1183
1184 #ifdef DEV_NETMAP
1185         /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */
1186         destroy_netmap_ifnet(pi);
1187 #endif
1188
1189         ADAPTER_LOCK(sc);
1190         CLR_BUSY(sc);
1191         wakeup(&sc->flags);
1192         ADAPTER_UNLOCK(sc);
1193
1194         return (0);
1195 }
1196
1197 static void
1198 cxgbe_init(void *arg)
1199 {
1200         struct port_info *pi = arg;
1201         struct adapter *sc = pi->adapter;
1202
1203         if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
1204                 return;
1205         cxgbe_init_synchronized(pi);
1206         end_synchronized_op(sc, 0);
1207 }
1208
1209 static int
1210 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1211 {
1212         int rc = 0, mtu, flags, can_sleep;
1213         struct port_info *pi = ifp->if_softc;
1214         struct adapter *sc = pi->adapter;
1215         struct ifreq *ifr = (struct ifreq *)data;
1216         uint32_t mask;
1217
1218         switch (cmd) {
1219         case SIOCSIFMTU:
1220                 mtu = ifr->ifr_mtu;
1221                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1222                         return (EINVAL);
1223
1224                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
1225                 if (rc)
1226                         return (rc);
1227                 ifp->if_mtu = mtu;
1228                 if (pi->flags & PORT_INIT_DONE) {
1229                         t4_update_fl_bufsize(ifp);
1230                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1231                                 rc = update_mac_settings(ifp, XGMAC_MTU);
1232                 }
1233                 end_synchronized_op(sc, 0);
1234                 break;
1235
1236         case SIOCSIFFLAGS:
1237                 can_sleep = 0;
1238 redo_sifflags:
1239                 rc = begin_synchronized_op(sc, pi,
1240                     can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg");
1241                 if (rc)
1242                         return (rc);
1243
1244                 if (ifp->if_flags & IFF_UP) {
1245                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1246                                 flags = pi->if_flags;
1247                                 if ((ifp->if_flags ^ flags) &
1248                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1249                                         if (can_sleep == 1) {
1250                                                 end_synchronized_op(sc, 0);
1251                                                 can_sleep = 0;
1252                                                 goto redo_sifflags;
1253                                         }
1254                                         rc = update_mac_settings(ifp,
1255                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1256                                 }
1257                         } else {
1258                                 if (can_sleep == 0) {
1259                                         end_synchronized_op(sc, LOCK_HELD);
1260                                         can_sleep = 1;
1261                                         goto redo_sifflags;
1262                                 }
1263                                 rc = cxgbe_init_synchronized(pi);
1264                         }
1265                         pi->if_flags = ifp->if_flags;
1266                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1267                         if (can_sleep == 0) {
1268                                 end_synchronized_op(sc, LOCK_HELD);
1269                                 can_sleep = 1;
1270                                 goto redo_sifflags;
1271                         }
1272                         rc = cxgbe_uninit_synchronized(pi);
1273                 }
1274                 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD);
1275                 break;
1276
1277         case SIOCADDMULTI:
1278         case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1279                 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
1280                 if (rc)
1281                         return (rc);
1282                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1283                         rc = update_mac_settings(ifp, XGMAC_MCADDRS);
1284                 end_synchronized_op(sc, LOCK_HELD);
1285                 break;
1286
1287         case SIOCSIFCAP:
1288                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
1289                 if (rc)
1290                         return (rc);
1291
1292                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1293                 if (mask & IFCAP_TXCSUM) {
1294                         ifp->if_capenable ^= IFCAP_TXCSUM;
1295                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1296
1297                         if (IFCAP_TSO4 & ifp->if_capenable &&
1298                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1299                                 ifp->if_capenable &= ~IFCAP_TSO4;
1300                                 if_printf(ifp,
1301                                     "tso4 disabled due to -txcsum.\n");
1302                         }
1303                 }
1304                 if (mask & IFCAP_TXCSUM_IPV6) {
1305                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1306                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1307
1308                         if (IFCAP_TSO6 & ifp->if_capenable &&
1309                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1310                                 ifp->if_capenable &= ~IFCAP_TSO6;
1311                                 if_printf(ifp,
1312                                     "tso6 disabled due to -txcsum6.\n");
1313                         }
1314                 }
1315                 if (mask & IFCAP_RXCSUM)
1316                         ifp->if_capenable ^= IFCAP_RXCSUM;
1317                 if (mask & IFCAP_RXCSUM_IPV6)
1318                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1319
1320                 /*
1321                  * Note that we leave CSUM_TSO alone (it is always set).  The
1322                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1323                  * sending a TSO request our way, so it's sufficient to toggle
1324                  * IFCAP_TSOx only.
1325                  */
1326                 if (mask & IFCAP_TSO4) {
1327                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1328                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1329                                 if_printf(ifp, "enable txcsum first.\n");
1330                                 rc = EAGAIN;
1331                                 goto fail;
1332                         }
1333                         ifp->if_capenable ^= IFCAP_TSO4;
1334                 }
1335                 if (mask & IFCAP_TSO6) {
1336                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1337                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1338                                 if_printf(ifp, "enable txcsum6 first.\n");
1339                                 rc = EAGAIN;
1340                                 goto fail;
1341                         }
1342                         ifp->if_capenable ^= IFCAP_TSO6;
1343                 }
1344                 if (mask & IFCAP_LRO) {
1345 #if defined(INET) || defined(INET6)
1346                         int i;
1347                         struct sge_rxq *rxq;
1348
1349                         ifp->if_capenable ^= IFCAP_LRO;
1350                         for_each_rxq(pi, i, rxq) {
1351                                 if (ifp->if_capenable & IFCAP_LRO)
1352                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1353                                 else
1354                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1355                         }
1356 #endif
1357                 }
1358 #ifdef TCP_OFFLOAD
1359                 if (mask & IFCAP_TOE) {
1360                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1361
1362                         rc = toe_capability(pi, enable);
1363                         if (rc != 0)
1364                                 goto fail;
1365
1366                         ifp->if_capenable ^= mask;
1367                 }
1368 #endif
1369                 if (mask & IFCAP_VLAN_HWTAGGING) {
1370                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1371                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1372                                 rc = update_mac_settings(ifp, XGMAC_VLANEX);
1373                 }
1374                 if (mask & IFCAP_VLAN_MTU) {
1375                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1376
1377                         /* Need to find out how to disable auto-mtu-inflation */
1378                 }
1379                 if (mask & IFCAP_VLAN_HWTSO)
1380                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1381                 if (mask & IFCAP_VLAN_HWCSUM)
1382                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1383
1384 #ifdef VLAN_CAPABILITIES
1385                 VLAN_CAPABILITIES(ifp);
1386 #endif
1387 fail:
1388                 end_synchronized_op(sc, 0);
1389                 break;
1390
1391         case SIOCSIFMEDIA:
1392         case SIOCGIFMEDIA:
1393                 ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1394                 break;
1395
1396         case SIOCGI2C: {
1397                 struct ifi2creq i2c;
1398
1399                 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
1400                 if (rc != 0)
1401                         break;
1402                 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
1403                         rc = EPERM;
1404                         break;
1405                 }
1406                 if (i2c.len > sizeof(i2c.data)) {
1407                         rc = EINVAL;
1408                         break;
1409                 }
1410                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4i2c");
1411                 if (rc)
1412                         return (rc);
1413                 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr,
1414                     i2c.offset, i2c.len, &i2c.data[0]);
1415                 end_synchronized_op(sc, 0);
1416                 if (rc == 0)
1417                         rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
1418                 break;
1419         }
1420
1421         default:
1422                 rc = ether_ioctl(ifp, cmd, data);
1423         }
1424
1425         return (rc);
1426 }
1427
1428 static int
1429 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1430 {
1431         struct port_info *pi = ifp->if_softc;
1432         struct adapter *sc = pi->adapter;
1433         struct sge_txq *txq;
1434         void *items[1];
1435         int rc;
1436
1437         M_ASSERTPKTHDR(m);
1438         MPASS(m->m_nextpkt == NULL);    /* not quite ready for this yet */
1439
1440         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1441                 m_freem(m);
1442                 return (ENETDOWN);
1443         }
1444
1445         rc = parse_pkt(&m);
1446         if (__predict_false(rc != 0)) {
1447                 MPASS(m == NULL);                       /* was freed already */
1448                 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */
1449                 return (rc);
1450         }
1451
1452         /* Select a txq. */
1453         txq = &sc->sge.txq[pi->first_txq];
1454         if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
1455                 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) +
1456                     pi->rsrv_noflowq);
1457
1458         items[0] = m;
1459         rc = mp_ring_enqueue(txq->r, items, 1, 4096);
1460         if (__predict_false(rc != 0))
1461                 m_freem(m);
1462
1463         return (rc);
1464 }
1465
1466 static void
1467 cxgbe_qflush(struct ifnet *ifp)
1468 {
1469         struct port_info *pi = ifp->if_softc;
1470         struct sge_txq *txq;
1471         int i;
1472
1473         /* queues do not exist if !PORT_INIT_DONE. */
1474         if (pi->flags & PORT_INIT_DONE) {
1475                 for_each_txq(pi, i, txq) {
1476                         TXQ_LOCK(txq);
1477                         txq->eq.flags &= ~EQ_ENABLED;
1478                         TXQ_UNLOCK(txq);
1479                         while (!mp_ring_is_idle(txq->r)) {
1480                                 mp_ring_check_drainage(txq->r, 0);
1481                                 pause("qflush", 1);
1482                         }
1483                 }
1484         }
1485         if_qflush(ifp);
1486 }
1487
1488 static uint64_t
1489 cxgbe_get_counter(struct ifnet *ifp, ift_counter c)
1490 {
1491         struct port_info *pi = ifp->if_softc;
1492         struct adapter *sc = pi->adapter;
1493         struct port_stats *s = &pi->stats;
1494
1495         cxgbe_refresh_stats(sc, pi);
1496
1497         switch (c) {
1498         case IFCOUNTER_IPACKETS:
1499                 return (s->rx_frames - s->rx_pause);
1500
1501         case IFCOUNTER_IERRORS:
1502                 return (s->rx_jabber + s->rx_runt + s->rx_too_long +
1503                     s->rx_fcs_err + s->rx_len_err);
1504
1505         case IFCOUNTER_OPACKETS:
1506                 return (s->tx_frames - s->tx_pause);
1507
1508         case IFCOUNTER_OERRORS:
1509                 return (s->tx_error_frames);
1510
1511         case IFCOUNTER_IBYTES:
1512                 return (s->rx_octets - s->rx_pause * 64);
1513
1514         case IFCOUNTER_OBYTES:
1515                 return (s->tx_octets - s->tx_pause * 64);
1516
1517         case IFCOUNTER_IMCASTS:
1518                 return (s->rx_mcast_frames - s->rx_pause);
1519
1520         case IFCOUNTER_OMCASTS:
1521                 return (s->tx_mcast_frames - s->tx_pause);
1522
1523         case IFCOUNTER_IQDROPS:
1524                 return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
1525                     s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
1526                     s->rx_trunc3 + pi->tnl_cong_drops);
1527
1528         case IFCOUNTER_OQDROPS: {
1529                 uint64_t drops;
1530
1531                 drops = s->tx_drop;
1532                 if (pi->flags & PORT_INIT_DONE) {
1533                         int i;
1534                         struct sge_txq *txq;
1535
1536                         for_each_txq(pi, i, txq)
1537                                 drops += counter_u64_fetch(txq->r->drops);
1538                 }
1539
1540                 return (drops);
1541
1542         }
1543
1544         default:
1545                 return (if_get_counter_default(ifp, c));
1546         }
1547 }
1548
1549 static int
1550 cxgbe_media_change(struct ifnet *ifp)
1551 {
1552         struct port_info *pi = ifp->if_softc;
1553
1554         device_printf(pi->dev, "%s unimplemented.\n", __func__);
1555
1556         return (EOPNOTSUPP);
1557 }
1558
1559 static void
1560 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1561 {
1562         struct port_info *pi = ifp->if_softc;
1563         struct ifmedia *media = NULL;
1564         struct ifmedia_entry *cur;
1565         int speed = pi->link_cfg.speed;
1566 #ifdef INVARIANTS
1567         int data = (pi->port_type << 8) | pi->mod_type;
1568 #endif
1569
1570         if (ifp == pi->ifp)
1571                 media = &pi->media;
1572 #ifdef DEV_NETMAP
1573         else if (ifp == pi->nm_ifp)
1574                 media = &pi->nm_media;
1575 #endif
1576         MPASS(media != NULL);
1577
1578         cur = media->ifm_cur;
1579         MPASS(cur->ifm_data == data);
1580
1581         ifmr->ifm_status = IFM_AVALID;
1582         if (!pi->link_cfg.link_ok)
1583                 return;
1584
1585         ifmr->ifm_status |= IFM_ACTIVE;
1586
1587         /* active and current will differ iff current media is autoselect. */
1588         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1589                 return;
1590
1591         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1592         if (speed == SPEED_10000)
1593                 ifmr->ifm_active |= IFM_10G_T;
1594         else if (speed == SPEED_1000)
1595                 ifmr->ifm_active |= IFM_1000_T;
1596         else if (speed == SPEED_100)
1597                 ifmr->ifm_active |= IFM_100_TX;
1598         else if (speed == SPEED_10)
1599                 ifmr->ifm_active |= IFM_10_T;
1600         else
1601                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1602                             speed));
1603 }
1604
1605 void
1606 t4_fatal_err(struct adapter *sc)
1607 {
1608         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1609         t4_intr_disable(sc);
1610         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1611             device_get_nameunit(sc->dev));
1612 }
1613
1614 static int
1615 map_bars_0_and_4(struct adapter *sc)
1616 {
1617         sc->regs_rid = PCIR_BAR(0);
1618         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1619             &sc->regs_rid, RF_ACTIVE);
1620         if (sc->regs_res == NULL) {
1621                 device_printf(sc->dev, "cannot map registers.\n");
1622                 return (ENXIO);
1623         }
1624         sc->bt = rman_get_bustag(sc->regs_res);
1625         sc->bh = rman_get_bushandle(sc->regs_res);
1626         sc->mmio_len = rman_get_size(sc->regs_res);
1627         setbit(&sc->doorbells, DOORBELL_KDB);
1628
1629         sc->msix_rid = PCIR_BAR(4);
1630         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1631             &sc->msix_rid, RF_ACTIVE);
1632         if (sc->msix_res == NULL) {
1633                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1634                 return (ENXIO);
1635         }
1636
1637         return (0);
1638 }
1639
1640 static int
1641 map_bar_2(struct adapter *sc)
1642 {
1643
1644         /*
1645          * T4: only iWARP driver uses the userspace doorbells.  There is no need
1646          * to map it if RDMA is disabled.
1647          */
1648         if (is_t4(sc) && sc->rdmacaps == 0)
1649                 return (0);
1650
1651         sc->udbs_rid = PCIR_BAR(2);
1652         sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1653             &sc->udbs_rid, RF_ACTIVE);
1654         if (sc->udbs_res == NULL) {
1655                 device_printf(sc->dev, "cannot map doorbell BAR.\n");
1656                 return (ENXIO);
1657         }
1658         sc->udbs_base = rman_get_virtual(sc->udbs_res);
1659
1660         if (is_t5(sc)) {
1661                 setbit(&sc->doorbells, DOORBELL_UDB);
1662 #if defined(__i386__) || defined(__amd64__)
1663                 if (t5_write_combine) {
1664                         int rc;
1665
1666                         /*
1667                          * Enable write combining on BAR2.  This is the
1668                          * userspace doorbell BAR and is split into 128B
1669                          * (UDBS_SEG_SIZE) doorbell regions, each associated
1670                          * with an egress queue.  The first 64B has the doorbell
1671                          * and the second 64B can be used to submit a tx work
1672                          * request with an implicit doorbell.
1673                          */
1674
1675                         rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
1676                             rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
1677                         if (rc == 0) {
1678                                 clrbit(&sc->doorbells, DOORBELL_UDB);
1679                                 setbit(&sc->doorbells, DOORBELL_WCWR);
1680                                 setbit(&sc->doorbells, DOORBELL_UDBWC);
1681                         } else {
1682                                 device_printf(sc->dev,
1683                                     "couldn't enable write combining: %d\n",
1684                                     rc);
1685                         }
1686
1687                         t4_write_reg(sc, A_SGE_STAT_CFG,
1688                             V_STATSOURCE_T5(7) | V_STATMODE(0));
1689                 }
1690 #endif
1691         }
1692
1693         return (0);
1694 }
1695
1696 static const struct memwin t4_memwin[] = {
1697         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1698         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1699         { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
1700 };
1701
1702 static const struct memwin t5_memwin[] = {
1703         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1704         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1705         { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1706 };
1707
1708 static void
1709 setup_memwin(struct adapter *sc)
1710 {
1711         const struct memwin *mw;
1712         int i, n;
1713         uint32_t bar0;
1714
1715         if (is_t4(sc)) {
1716                 /*
1717                  * Read low 32b of bar0 indirectly via the hardware backdoor
1718                  * mechanism.  Works from within PCI passthrough environments
1719                  * too, where rman_get_start() can return a different value.  We
1720                  * need to program the T4 memory window decoders with the actual
1721                  * addresses that will be coming across the PCIe link.
1722                  */
1723                 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1724                 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1725
1726                 mw = &t4_memwin[0];
1727                 n = nitems(t4_memwin);
1728         } else {
1729                 /* T5 uses the relative offset inside the PCIe BAR */
1730                 bar0 = 0;
1731
1732                 mw = &t5_memwin[0];
1733                 n = nitems(t5_memwin);
1734         }
1735
1736         for (i = 0; i < n; i++, mw++) {
1737                 t4_write_reg(sc,
1738                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
1739                     (mw->base + bar0) | V_BIR(0) |
1740                     V_WINDOW(ilog2(mw->aperture) - 10));
1741         }
1742
1743         /* flush */
1744         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1745 }
1746
1747 /*
1748  * Verify that the memory range specified by the addr/len pair is valid and lies
1749  * entirely within a single region (EDCx or MCx).
1750  */
1751 static int
1752 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
1753 {
1754         uint32_t em, addr_len, maddr, mlen;
1755
1756         /* Memory can only be accessed in naturally aligned 4 byte units */
1757         if (addr & 3 || len & 3 || len == 0)
1758                 return (EINVAL);
1759
1760         /* Enabled memories */
1761         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1762         if (em & F_EDRAM0_ENABLE) {
1763                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1764                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1765                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1766                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1767                     addr + len <= maddr + mlen)
1768                         return (0);
1769         }
1770         if (em & F_EDRAM1_ENABLE) {
1771                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1772                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1773                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1774                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1775                     addr + len <= maddr + mlen)
1776                         return (0);
1777         }
1778         if (em & F_EXT_MEM_ENABLE) {
1779                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1780                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1781                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1782                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1783                     addr + len <= maddr + mlen)
1784                         return (0);
1785         }
1786         if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
1787                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1788                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1789                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1790                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1791                     addr + len <= maddr + mlen)
1792                         return (0);
1793         }
1794
1795         return (EFAULT);
1796 }
1797
1798 static int
1799 fwmtype_to_hwmtype(int mtype)
1800 {
1801
1802         switch (mtype) {
1803         case FW_MEMTYPE_EDC0:
1804                 return (MEM_EDC0);
1805         case FW_MEMTYPE_EDC1:
1806                 return (MEM_EDC1);
1807         case FW_MEMTYPE_EXTMEM:
1808                 return (MEM_MC0);
1809         case FW_MEMTYPE_EXTMEM1:
1810                 return (MEM_MC1);
1811         default:
1812                 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
1813         }
1814 }
1815
1816 /*
1817  * Verify that the memory range specified by the memtype/offset/len pair is
1818  * valid and lies entirely within the memtype specified.  The global address of
1819  * the start of the range is returned in addr.
1820  */
1821 static int
1822 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1823     uint32_t *addr)
1824 {
1825         uint32_t em, addr_len, maddr, mlen;
1826
1827         /* Memory can only be accessed in naturally aligned 4 byte units */
1828         if (off & 3 || len & 3 || len == 0)
1829                 return (EINVAL);
1830
1831         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1832         switch (fwmtype_to_hwmtype(mtype)) {
1833         case MEM_EDC0:
1834                 if (!(em & F_EDRAM0_ENABLE))
1835                         return (EINVAL);
1836                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1837                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1838                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1839                 break;
1840         case MEM_EDC1:
1841                 if (!(em & F_EDRAM1_ENABLE))
1842                         return (EINVAL);
1843                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1844                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1845                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1846                 break;
1847         case MEM_MC:
1848                 if (!(em & F_EXT_MEM_ENABLE))
1849                         return (EINVAL);
1850                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1851                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1852                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1853                 break;
1854         case MEM_MC1:
1855                 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
1856                         return (EINVAL);
1857                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1858                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1859                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1860                 break;
1861         default:
1862                 return (EINVAL);
1863         }
1864
1865         if (mlen > 0 && off < mlen && off + len <= mlen) {
1866                 *addr = maddr + off;    /* global address */
1867                 return (0);
1868         }
1869
1870         return (EFAULT);
1871 }
1872
1873 static void
1874 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1875 {
1876         const struct memwin *mw;
1877
1878         if (is_t4(sc)) {
1879                 KASSERT(win >= 0 && win < nitems(t4_memwin),
1880                     ("%s: incorrect memwin# (%d)", __func__, win));
1881                 mw = &t4_memwin[win];
1882         } else {
1883                 KASSERT(win >= 0 && win < nitems(t5_memwin),
1884                     ("%s: incorrect memwin# (%d)", __func__, win));
1885                 mw = &t5_memwin[win];
1886         }
1887
1888         if (base != NULL)
1889                 *base = mw->base;
1890         if (aperture != NULL)
1891                 *aperture = mw->aperture;
1892 }
1893
1894 /*
1895  * Positions the memory window such that it can be used to access the specified
1896  * address in the chip's address space.  The return value is the offset of addr
1897  * from the start of the window.
1898  */
1899 static uint32_t
1900 position_memwin(struct adapter *sc, int n, uint32_t addr)
1901 {
1902         uint32_t start, pf;
1903         uint32_t reg;
1904
1905         KASSERT(n >= 0 && n <= 3,
1906             ("%s: invalid window %d.", __func__, n));
1907         KASSERT((addr & 3) == 0,
1908             ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
1909
1910         if (is_t4(sc)) {
1911                 pf = 0;
1912                 start = addr & ~0xf;    /* start must be 16B aligned */
1913         } else {
1914                 pf = V_PFNUM(sc->pf);
1915                 start = addr & ~0x7f;   /* start must be 128B aligned */
1916         }
1917         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1918
1919         t4_write_reg(sc, reg, start | pf);
1920         t4_read_reg(sc, reg);
1921
1922         return (addr - start);
1923 }
1924
1925 static int
1926 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1927     struct intrs_and_queues *iaq)
1928 {
1929         int rc, itype, navail, nrxq10g, nrxq1g, n;
1930         int nofldrxq10g = 0, nofldrxq1g = 0;
1931         int nnmrxq10g = 0, nnmrxq1g = 0;
1932
1933         bzero(iaq, sizeof(*iaq));
1934
1935         iaq->ntxq10g = t4_ntxq10g;
1936         iaq->ntxq1g = t4_ntxq1g;
1937         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1938         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1939         iaq->rsrv_noflowq = t4_rsrv_noflowq;
1940 #ifdef TCP_OFFLOAD
1941         if (is_offload(sc)) {
1942                 iaq->nofldtxq10g = t4_nofldtxq10g;
1943                 iaq->nofldtxq1g = t4_nofldtxq1g;
1944                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1945                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1946         }
1947 #endif
1948 #ifdef DEV_NETMAP
1949         iaq->nnmtxq10g = t4_nnmtxq10g;
1950         iaq->nnmtxq1g = t4_nnmtxq1g;
1951         iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g;
1952         iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g;
1953 #endif
1954
1955         for (itype = INTR_MSIX; itype; itype >>= 1) {
1956
1957                 if ((itype & t4_intr_types) == 0)
1958                         continue;       /* not allowed */
1959
1960                 if (itype == INTR_MSIX)
1961                         navail = pci_msix_count(sc->dev);
1962                 else if (itype == INTR_MSI)
1963                         navail = pci_msi_count(sc->dev);
1964                 else
1965                         navail = 1;
1966 restart:
1967                 if (navail == 0)
1968                         continue;
1969
1970                 iaq->intr_type = itype;
1971                 iaq->intr_flags_10g = 0;
1972                 iaq->intr_flags_1g = 0;
1973
1974                 /*
1975                  * Best option: an interrupt vector for errors, one for the
1976                  * firmware event queue, and one for every rxq (NIC, TOE, and
1977                  * netmap).
1978                  */
1979                 iaq->nirq = T4_EXTRA_INTR;
1980                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g);
1981                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g);
1982                 if (iaq->nirq <= navail &&
1983                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
1984                         iaq->intr_flags_10g = INTR_ALL;
1985                         iaq->intr_flags_1g = INTR_ALL;
1986                         goto allocate;
1987                 }
1988
1989                 /*
1990                  * Second best option: a vector for errors, one for the firmware
1991                  * event queue, and vectors for either all the NIC rx queues or
1992                  * all the TOE rx queues.  The queues that don't get vectors
1993                  * will forward their interrupts to those that do.
1994                  *
1995                  * Note: netmap rx queues cannot be created early and so they
1996                  * can't be setup to receive forwarded interrupts for others.
1997                  */
1998                 iaq->nirq = T4_EXTRA_INTR;
1999                 if (nrxq10g >= nofldrxq10g) {
2000                         iaq->intr_flags_10g = INTR_RXQ;
2001                         iaq->nirq += n10g * nrxq10g;
2002 #ifdef DEV_NETMAP
2003                         iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g);
2004 #endif
2005                 } else {
2006                         iaq->intr_flags_10g = INTR_OFLD_RXQ;
2007                         iaq->nirq += n10g * nofldrxq10g;
2008 #ifdef DEV_NETMAP
2009                         iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g);
2010 #endif
2011                 }
2012                 if (nrxq1g >= nofldrxq1g) {
2013                         iaq->intr_flags_1g = INTR_RXQ;
2014                         iaq->nirq += n1g * nrxq1g;
2015 #ifdef DEV_NETMAP
2016                         iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g);
2017 #endif
2018                 } else {
2019                         iaq->intr_flags_1g = INTR_OFLD_RXQ;
2020                         iaq->nirq += n1g * nofldrxq1g;
2021 #ifdef DEV_NETMAP
2022                         iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g);
2023 #endif
2024                 }
2025                 if (iaq->nirq <= navail &&
2026                     (itype != INTR_MSI || powerof2(iaq->nirq)))
2027                         goto allocate;
2028
2029                 /*
2030                  * Next best option: an interrupt vector for errors, one for the
2031                  * firmware event queue, and at least one per port.  At this
2032                  * point we know we'll have to downsize nrxq and/or nofldrxq
2033                  * and/or nnmrxq to fit what's available to us.
2034                  */
2035                 iaq->nirq = T4_EXTRA_INTR;
2036                 iaq->nirq += n10g + n1g;
2037                 if (iaq->nirq <= navail) {
2038                         int leftover = navail - iaq->nirq;
2039
2040                         if (n10g > 0) {
2041                                 int target = max(nrxq10g, nofldrxq10g);
2042
2043                                 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
2044                                     INTR_RXQ : INTR_OFLD_RXQ;
2045
2046                                 n = 1;
2047                                 while (n < target && leftover >= n10g) {
2048                                         leftover -= n10g;
2049                                         iaq->nirq += n10g;
2050                                         n++;
2051                                 }
2052                                 iaq->nrxq10g = min(n, nrxq10g);
2053 #ifdef TCP_OFFLOAD
2054                                 iaq->nofldrxq10g = min(n, nofldrxq10g);
2055 #endif
2056 #ifdef DEV_NETMAP
2057                                 iaq->nnmrxq10g = min(n, nnmrxq10g);
2058 #endif
2059                         }
2060
2061                         if (n1g > 0) {
2062                                 int target = max(nrxq1g, nofldrxq1g);
2063
2064                                 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
2065                                     INTR_RXQ : INTR_OFLD_RXQ;
2066
2067                                 n = 1;
2068                                 while (n < target && leftover >= n1g) {
2069                                         leftover -= n1g;
2070                                         iaq->nirq += n1g;
2071                                         n++;
2072                                 }
2073                                 iaq->nrxq1g = min(n, nrxq1g);
2074 #ifdef TCP_OFFLOAD
2075                                 iaq->nofldrxq1g = min(n, nofldrxq1g);
2076 #endif
2077 #ifdef DEV_NETMAP
2078                                 iaq->nnmrxq1g = min(n, nnmrxq1g);
2079 #endif
2080                         }
2081
2082                         if (itype != INTR_MSI || powerof2(iaq->nirq))
2083                                 goto allocate;
2084                 }
2085
2086                 /*
2087                  * Least desirable option: one interrupt vector for everything.
2088                  */
2089                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
2090                 iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
2091 #ifdef TCP_OFFLOAD
2092                 if (is_offload(sc))
2093                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2094 #endif
2095 #ifdef DEV_NETMAP
2096                 iaq->nnmrxq10g = iaq->nnmrxq1g = 1;
2097 #endif
2098
2099 allocate:
2100                 navail = iaq->nirq;
2101                 rc = 0;
2102                 if (itype == INTR_MSIX)
2103                         rc = pci_alloc_msix(sc->dev, &navail);
2104                 else if (itype == INTR_MSI)
2105                         rc = pci_alloc_msi(sc->dev, &navail);
2106
2107                 if (rc == 0) {
2108                         if (navail == iaq->nirq)
2109                                 return (0);
2110
2111                         /*
2112                          * Didn't get the number requested.  Use whatever number
2113                          * the kernel is willing to allocate (it's in navail).
2114                          */
2115                         device_printf(sc->dev, "fewer vectors than requested, "
2116                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
2117                             itype, iaq->nirq, navail);
2118                         pci_release_msi(sc->dev);
2119                         goto restart;
2120                 }
2121
2122                 device_printf(sc->dev,
2123                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
2124                     itype, rc, iaq->nirq, navail);
2125         }
2126
2127         device_printf(sc->dev,
2128             "failed to find a usable interrupt type.  "
2129             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
2130             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
2131
2132         return (ENXIO);
2133 }
2134
2135 #define FW_VERSION(chip) ( \
2136     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
2137     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
2138     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
2139     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
2140 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
2141
2142 struct fw_info {
2143         uint8_t chip;
2144         char *kld_name;
2145         char *fw_mod_name;
2146         struct fw_hdr fw_hdr;   /* XXX: waste of space, need a sparse struct */
2147 } fw_info[] = {
2148         {
2149                 .chip = CHELSIO_T4,
2150                 .kld_name = "t4fw_cfg",
2151                 .fw_mod_name = "t4fw",
2152                 .fw_hdr = {
2153                         .chip = FW_HDR_CHIP_T4,
2154                         .fw_ver = htobe32_const(FW_VERSION(T4)),
2155                         .intfver_nic = FW_INTFVER(T4, NIC),
2156                         .intfver_vnic = FW_INTFVER(T4, VNIC),
2157                         .intfver_ofld = FW_INTFVER(T4, OFLD),
2158                         .intfver_ri = FW_INTFVER(T4, RI),
2159                         .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
2160                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
2161                         .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
2162                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
2163                 },
2164         }, {
2165                 .chip = CHELSIO_T5,
2166                 .kld_name = "t5fw_cfg",
2167                 .fw_mod_name = "t5fw",
2168                 .fw_hdr = {
2169                         .chip = FW_HDR_CHIP_T5,
2170                         .fw_ver = htobe32_const(FW_VERSION(T5)),
2171                         .intfver_nic = FW_INTFVER(T5, NIC),
2172                         .intfver_vnic = FW_INTFVER(T5, VNIC),
2173                         .intfver_ofld = FW_INTFVER(T5, OFLD),
2174                         .intfver_ri = FW_INTFVER(T5, RI),
2175                         .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
2176                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
2177                         .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
2178                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
2179                 },
2180         }
2181 };
2182
2183 static struct fw_info *
2184 find_fw_info(int chip)
2185 {
2186         int i;
2187
2188         for (i = 0; i < nitems(fw_info); i++) {
2189                 if (fw_info[i].chip == chip)
2190                         return (&fw_info[i]);
2191         }
2192         return (NULL);
2193 }
2194
2195 /*
2196  * Is the given firmware API compatible with the one the driver was compiled
2197  * with?
2198  */
2199 static int
2200 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
2201 {
2202
2203         /* short circuit if it's the exact same firmware version */
2204         if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
2205                 return (1);
2206
2207         /*
2208          * XXX: Is this too conservative?  Perhaps I should limit this to the
2209          * features that are supported in the driver.
2210          */
2211 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
2212         if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
2213             SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
2214             SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
2215                 return (1);
2216 #undef SAME_INTF
2217
2218         return (0);
2219 }
2220
2221 /*
2222  * The firmware in the KLD is usable, but should it be installed?  This routine
2223  * explains itself in detail if it indicates the KLD firmware should be
2224  * installed.
2225  */
2226 static int
2227 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
2228 {
2229         const char *reason;
2230
2231         if (!card_fw_usable) {
2232                 reason = "incompatible or unusable";
2233                 goto install;
2234         }
2235
2236         if (k > c) {
2237                 reason = "older than the version bundled with this driver";
2238                 goto install;
2239         }
2240
2241         if (t4_fw_install == 2 && k != c) {
2242                 reason = "different than the version bundled with this driver";
2243                 goto install;
2244         }
2245
2246         return (0);
2247
2248 install:
2249         if (t4_fw_install == 0) {
2250                 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2251                     "but the driver is prohibited from installing a different "
2252                     "firmware on the card.\n",
2253                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2254                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2255
2256                 return (0);
2257         }
2258
2259         device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2260             "installing firmware %u.%u.%u.%u on card.\n",
2261             G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2262             G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2263             G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2264             G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2265
2266         return (1);
2267 }
2268 /*
2269  * Establish contact with the firmware and determine if we are the master driver
2270  * or not, and whether we are responsible for chip initialization.
2271  */
2272 static int
2273 prep_firmware(struct adapter *sc)
2274 {
2275         const struct firmware *fw = NULL, *default_cfg;
2276         int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2277         enum dev_state state;
2278         struct fw_info *fw_info;
2279         struct fw_hdr *card_fw;         /* fw on the card */
2280         const struct fw_hdr *kld_fw;    /* fw in the KLD */
2281         const struct fw_hdr *drv_fw;    /* fw header the driver was compiled
2282                                            against */
2283
2284         /* Contact firmware. */
2285         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2286         if (rc < 0 || state == DEV_STATE_ERR) {
2287                 rc = -rc;
2288                 device_printf(sc->dev,
2289                     "failed to connect to the firmware: %d, %d.\n", rc, state);
2290                 return (rc);
2291         }
2292         pf = rc;
2293         if (pf == sc->mbox)
2294                 sc->flags |= MASTER_PF;
2295         else if (state == DEV_STATE_UNINIT) {
2296                 /*
2297                  * We didn't get to be the master so we definitely won't be
2298                  * configuring the chip.  It's a bug if someone else hasn't
2299                  * configured it already.
2300                  */
2301                 device_printf(sc->dev, "couldn't be master(%d), "
2302                     "device not already initialized either(%d).\n", rc, state);
2303                 return (EDOOFUS);
2304         }
2305
2306         /* This is the firmware whose headers the driver was compiled against */
2307         fw_info = find_fw_info(chip_id(sc));
2308         if (fw_info == NULL) {
2309                 device_printf(sc->dev,
2310                     "unable to look up firmware information for chip %d.\n",
2311                     chip_id(sc));
2312                 return (EINVAL);
2313         }
2314         drv_fw = &fw_info->fw_hdr;
2315
2316         /*
2317          * The firmware KLD contains many modules.  The KLD name is also the
2318          * name of the module that contains the default config file.
2319          */
2320         default_cfg = firmware_get(fw_info->kld_name);
2321
2322         /* Read the header of the firmware on the card */
2323         card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2324         rc = -t4_read_flash(sc, FLASH_FW_START,
2325             sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2326         if (rc == 0)
2327                 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2328         else {
2329                 device_printf(sc->dev,
2330                     "Unable to read card's firmware header: %d\n", rc);
2331                 card_fw_usable = 0;
2332         }
2333
2334         /* This is the firmware in the KLD */
2335         fw = firmware_get(fw_info->fw_mod_name);
2336         if (fw != NULL) {
2337                 kld_fw = (const void *)fw->data;
2338                 kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2339         } else {
2340                 kld_fw = NULL;
2341                 kld_fw_usable = 0;
2342         }
2343
2344         if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2345             (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2346                 /*
2347                  * Common case: the firmware on the card is an exact match and
2348                  * the KLD is an exact match too, or the KLD is
2349                  * absent/incompatible.  Note that t4_fw_install = 2 is ignored
2350                  * here -- use cxgbetool loadfw if you want to reinstall the
2351                  * same firmware as the one on the card.
2352                  */
2353         } else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2354             should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2355             be32toh(card_fw->fw_ver))) {
2356
2357                 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
2358                 if (rc != 0) {
2359                         device_printf(sc->dev,
2360                             "failed to install firmware: %d\n", rc);
2361                         goto done;
2362                 }
2363
2364                 /* Installed successfully, update the cached header too. */
2365                 memcpy(card_fw, kld_fw, sizeof(*card_fw));
2366                 card_fw_usable = 1;
2367                 need_fw_reset = 0;      /* already reset as part of load_fw */
2368         }
2369
2370         if (!card_fw_usable) {
2371                 uint32_t d, c, k;
2372
2373                 d = ntohl(drv_fw->fw_ver);
2374                 c = ntohl(card_fw->fw_ver);
2375                 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
2376
2377                 device_printf(sc->dev, "Cannot find a usable firmware: "
2378                     "fw_install %d, chip state %d, "
2379                     "driver compiled with %d.%d.%d.%d, "
2380                     "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
2381                     t4_fw_install, state,
2382                     G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
2383                     G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
2384                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2385                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
2386                     G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2387                     G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2388                 rc = EINVAL;
2389                 goto done;
2390         }
2391
2392         /* We're using whatever's on the card and it's known to be good. */
2393         sc->params.fw_vers = ntohl(card_fw->fw_ver);
2394         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
2395             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2396             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2397             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2398             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2399         t4_get_tp_version(sc, &sc->params.tp_vers);
2400
2401         /* Reset device */
2402         if (need_fw_reset &&
2403             (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
2404                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
2405                 if (rc != ETIMEDOUT && rc != EIO)
2406                         t4_fw_bye(sc, sc->mbox);
2407                 goto done;
2408         }
2409         sc->flags |= FW_OK;
2410
2411         rc = get_params__pre_init(sc);
2412         if (rc != 0)
2413                 goto done; /* error message displayed already */
2414
2415         /* Partition adapter resources as specified in the config file. */
2416         if (state == DEV_STATE_UNINIT) {
2417
2418                 KASSERT(sc->flags & MASTER_PF,
2419                     ("%s: trying to change chip settings when not master.",
2420                     __func__));
2421
2422                 rc = partition_resources(sc, default_cfg, fw_info->kld_name);
2423                 if (rc != 0)
2424                         goto done;      /* error message displayed already */
2425
2426                 t4_tweak_chip_settings(sc);
2427
2428                 /* get basic stuff going */
2429                 rc = -t4_fw_initialize(sc, sc->mbox);
2430                 if (rc != 0) {
2431                         device_printf(sc->dev, "fw init failed: %d.\n", rc);
2432                         goto done;
2433                 }
2434         } else {
2435                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
2436                 sc->cfcsum = 0;
2437         }
2438
2439 done:
2440         free(card_fw, M_CXGBE);
2441         if (fw != NULL)
2442                 firmware_put(fw, FIRMWARE_UNLOAD);
2443         if (default_cfg != NULL)
2444                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
2445
2446         return (rc);
2447 }
2448
2449 #define FW_PARAM_DEV(param) \
2450         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2451          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2452 #define FW_PARAM_PFVF(param) \
2453         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2454          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2455
2456 /*
2457  * Partition chip resources for use between various PFs, VFs, etc.
2458  */
2459 static int
2460 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
2461     const char *name_prefix)
2462 {
2463         const struct firmware *cfg = NULL;
2464         int rc = 0;
2465         struct fw_caps_config_cmd caps;
2466         uint32_t mtype, moff, finicsum, cfcsum;
2467
2468         /*
2469          * Figure out what configuration file to use.  Pick the default config
2470          * file for the card if the user hasn't specified one explicitly.
2471          */
2472         snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
2473         if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
2474                 /* Card specific overrides go here. */
2475                 if (pci_get_device(sc->dev) == 0x440a)
2476                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
2477                 if (is_fpga(sc))
2478                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
2479         }
2480
2481         /*
2482          * We need to load another module if the profile is anything except
2483          * "default" or "flash".
2484          */
2485         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
2486             strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2487                 char s[32];
2488
2489                 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
2490                 cfg = firmware_get(s);
2491                 if (cfg == NULL) {
2492                         if (default_cfg != NULL) {
2493                                 device_printf(sc->dev,
2494                                     "unable to load module \"%s\" for "
2495                                     "configuration profile \"%s\", will use "
2496                                     "the default config file instead.\n",
2497                                     s, sc->cfg_file);
2498                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2499                                     "%s", DEFAULT_CF);
2500                         } else {
2501                                 device_printf(sc->dev,
2502                                     "unable to load module \"%s\" for "
2503                                     "configuration profile \"%s\", will use "
2504                                     "the config file on the card's flash "
2505                                     "instead.\n", s, sc->cfg_file);
2506                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2507                                     "%s", FLASH_CF);
2508                         }
2509                 }
2510         }
2511
2512         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
2513             default_cfg == NULL) {
2514                 device_printf(sc->dev,
2515                     "default config file not available, will use the config "
2516                     "file on the card's flash instead.\n");
2517                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
2518         }
2519
2520         if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2521                 u_int cflen, i, n;
2522                 const uint32_t *cfdata;
2523                 uint32_t param, val, addr, off, mw_base, mw_aperture;
2524
2525                 KASSERT(cfg != NULL || default_cfg != NULL,
2526                     ("%s: no config to upload", __func__));
2527
2528                 /*
2529                  * Ask the firmware where it wants us to upload the config file.
2530                  */
2531                 param = FW_PARAM_DEV(CF);
2532                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2533                 if (rc != 0) {
2534                         /* No support for config file?  Shouldn't happen. */
2535                         device_printf(sc->dev,
2536                             "failed to query config file location: %d.\n", rc);
2537                         goto done;
2538                 }
2539                 mtype = G_FW_PARAMS_PARAM_Y(val);
2540                 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
2541
2542                 /*
2543                  * XXX: sheer laziness.  We deliberately added 4 bytes of
2544                  * useless stuffing/comments at the end of the config file so
2545                  * it's ok to simply throw away the last remaining bytes when
2546                  * the config file is not an exact multiple of 4.  This also
2547                  * helps with the validate_mt_off_len check.
2548                  */
2549                 if (cfg != NULL) {
2550                         cflen = cfg->datasize & ~3;
2551                         cfdata = cfg->data;
2552                 } else {
2553                         cflen = default_cfg->datasize & ~3;
2554                         cfdata = default_cfg->data;
2555                 }
2556
2557                 if (cflen > FLASH_CFG_MAX_SIZE) {
2558                         device_printf(sc->dev,
2559                             "config file too long (%d, max allowed is %d).  "
2560                             "Will try to use the config on the card, if any.\n",
2561                             cflen, FLASH_CFG_MAX_SIZE);
2562                         goto use_config_on_flash;
2563                 }
2564
2565                 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
2566                 if (rc != 0) {
2567                         device_printf(sc->dev,
2568                             "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
2569                             "Will try to use the config on the card, if any.\n",
2570                             __func__, mtype, moff, cflen, rc);
2571                         goto use_config_on_flash;
2572                 }
2573
2574                 memwin_info(sc, 2, &mw_base, &mw_aperture);
2575                 while (cflen) {
2576                         off = position_memwin(sc, 2, addr);
2577                         n = min(cflen, mw_aperture - off);
2578                         for (i = 0; i < n; i += 4)
2579                                 t4_write_reg(sc, mw_base + off + i, *cfdata++);
2580                         cflen -= n;
2581                         addr += n;
2582                 }
2583         } else {
2584 use_config_on_flash:
2585                 mtype = FW_MEMTYPE_FLASH;
2586                 moff = t4_flash_cfg_addr(sc);
2587         }
2588
2589         bzero(&caps, sizeof(caps));
2590         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2591             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2592         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
2593             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
2594             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
2595         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2596         if (rc != 0) {
2597                 device_printf(sc->dev,
2598                     "failed to pre-process config file: %d "
2599                     "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
2600                 goto done;
2601         }
2602
2603         finicsum = be32toh(caps.finicsum);
2604         cfcsum = be32toh(caps.cfcsum);
2605         if (finicsum != cfcsum) {
2606                 device_printf(sc->dev,
2607                     "WARNING: config file checksum mismatch: %08x %08x\n",
2608                     finicsum, cfcsum);
2609         }
2610         sc->cfcsum = cfcsum;
2611
2612 #define LIMIT_CAPS(x) do { \
2613         caps.x &= htobe16(t4_##x##_allowed); \
2614 } while (0)
2615
2616         /*
2617          * Let the firmware know what features will (not) be used so it can tune
2618          * things accordingly.
2619          */
2620         LIMIT_CAPS(linkcaps);
2621         LIMIT_CAPS(niccaps);
2622         LIMIT_CAPS(toecaps);
2623         LIMIT_CAPS(rdmacaps);
2624         LIMIT_CAPS(iscsicaps);
2625         LIMIT_CAPS(fcoecaps);
2626 #undef LIMIT_CAPS
2627
2628         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2629             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
2630         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2631         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
2632         if (rc != 0) {
2633                 device_printf(sc->dev,
2634                     "failed to process config file: %d.\n", rc);
2635         }
2636 done:
2637         if (cfg != NULL)
2638                 firmware_put(cfg, FIRMWARE_UNLOAD);
2639         return (rc);
2640 }
2641
2642 /*
2643  * Retrieve parameters that are needed (or nice to have) very early.
2644  */
2645 static int
2646 get_params__pre_init(struct adapter *sc)
2647 {
2648         int rc;
2649         uint32_t param[2], val[2];
2650         struct fw_devlog_cmd cmd;
2651         struct devlog_params *dlog = &sc->params.devlog;
2652
2653         param[0] = FW_PARAM_DEV(PORTVEC);
2654         param[1] = FW_PARAM_DEV(CCLK);
2655         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2656         if (rc != 0) {
2657                 device_printf(sc->dev,
2658                     "failed to query parameters (pre_init): %d.\n", rc);
2659                 return (rc);
2660         }
2661
2662         sc->params.portvec = val[0];
2663         sc->params.nports = bitcount32(val[0]);
2664         sc->params.vpd.cclk = val[1];
2665
2666         /* Read device log parameters. */
2667         bzero(&cmd, sizeof(cmd));
2668         cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
2669             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2670         cmd.retval_len16 = htobe32(FW_LEN16(cmd));
2671         rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
2672         if (rc != 0) {
2673                 device_printf(sc->dev,
2674                     "failed to get devlog parameters: %d.\n", rc);
2675                 bzero(dlog, sizeof (*dlog));
2676                 rc = 0; /* devlog isn't critical for device operation */
2677         } else {
2678                 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
2679                 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
2680                 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
2681                 dlog->size = be32toh(cmd.memsize_devlog);
2682         }
2683
2684         return (rc);
2685 }
2686
2687 /*
2688  * Retrieve various parameters that are of interest to the driver.  The device
2689  * has been initialized by the firmware at this point.
2690  */
2691 static int
2692 get_params__post_init(struct adapter *sc)
2693 {
2694         int rc;
2695         uint32_t param[7], val[7];
2696         struct fw_caps_config_cmd caps;
2697
2698         param[0] = FW_PARAM_PFVF(IQFLINT_START);
2699         param[1] = FW_PARAM_PFVF(EQ_START);
2700         param[2] = FW_PARAM_PFVF(FILTER_START);
2701         param[3] = FW_PARAM_PFVF(FILTER_END);
2702         param[4] = FW_PARAM_PFVF(L2T_START);
2703         param[5] = FW_PARAM_PFVF(L2T_END);
2704         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2705         if (rc != 0) {
2706                 device_printf(sc->dev,
2707                     "failed to query parameters (post_init): %d.\n", rc);
2708                 return (rc);
2709         }
2710
2711         sc->sge.iq_start = val[0];
2712         sc->sge.eq_start = val[1];
2713         sc->tids.ftid_base = val[2];
2714         sc->tids.nftids = val[3] - val[2] + 1;
2715         sc->params.ftid_min = val[2];
2716         sc->params.ftid_max = val[3];
2717         sc->vres.l2t.start = val[4];
2718         sc->vres.l2t.size = val[5] - val[4] + 1;
2719         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
2720             ("%s: L2 table size (%u) larger than expected (%u)",
2721             __func__, sc->vres.l2t.size, L2T_SIZE));
2722
2723         /* get capabilites */
2724         bzero(&caps, sizeof(caps));
2725         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2726             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2727         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2728         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2729         if (rc != 0) {
2730                 device_printf(sc->dev,
2731                     "failed to get card capabilities: %d.\n", rc);
2732                 return (rc);
2733         }
2734
2735 #define READ_CAPS(x) do { \
2736         sc->x = htobe16(caps.x); \
2737 } while (0)
2738         READ_CAPS(linkcaps);
2739         READ_CAPS(niccaps);
2740         READ_CAPS(toecaps);
2741         READ_CAPS(rdmacaps);
2742         READ_CAPS(iscsicaps);
2743         READ_CAPS(fcoecaps);
2744
2745         if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
2746                 param[0] = FW_PARAM_PFVF(ETHOFLD_START);
2747                 param[1] = FW_PARAM_PFVF(ETHOFLD_END);
2748                 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2749                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
2750                 if (rc != 0) {
2751                         device_printf(sc->dev,
2752                             "failed to query NIC parameters: %d.\n", rc);
2753                         return (rc);
2754                 }
2755                 sc->tids.etid_base = val[0];
2756                 sc->params.etid_min = val[0];
2757                 sc->tids.netids = val[1] - val[0] + 1;
2758                 sc->params.netids = sc->tids.netids;
2759                 sc->params.eo_wr_cred = val[2];
2760                 sc->params.ethoffload = 1;
2761         }
2762
2763         if (sc->toecaps) {
2764                 /* query offload-related parameters */
2765                 param[0] = FW_PARAM_DEV(NTID);
2766                 param[1] = FW_PARAM_PFVF(SERVER_START);
2767                 param[2] = FW_PARAM_PFVF(SERVER_END);
2768                 param[3] = FW_PARAM_PFVF(TDDP_START);
2769                 param[4] = FW_PARAM_PFVF(TDDP_END);
2770                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2771                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2772                 if (rc != 0) {
2773                         device_printf(sc->dev,
2774                             "failed to query TOE parameters: %d.\n", rc);
2775                         return (rc);
2776                 }
2777                 sc->tids.ntids = val[0];
2778                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
2779                 sc->tids.stid_base = val[1];
2780                 sc->tids.nstids = val[2] - val[1] + 1;
2781                 sc->vres.ddp.start = val[3];
2782                 sc->vres.ddp.size = val[4] - val[3] + 1;
2783                 sc->params.ofldq_wr_cred = val[5];
2784                 sc->params.offload = 1;
2785         }
2786         if (sc->rdmacaps) {
2787                 param[0] = FW_PARAM_PFVF(STAG_START);
2788                 param[1] = FW_PARAM_PFVF(STAG_END);
2789                 param[2] = FW_PARAM_PFVF(RQ_START);
2790                 param[3] = FW_PARAM_PFVF(RQ_END);
2791                 param[4] = FW_PARAM_PFVF(PBL_START);
2792                 param[5] = FW_PARAM_PFVF(PBL_END);
2793                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2794                 if (rc != 0) {
2795                         device_printf(sc->dev,
2796                             "failed to query RDMA parameters(1): %d.\n", rc);
2797                         return (rc);
2798                 }
2799                 sc->vres.stag.start = val[0];
2800                 sc->vres.stag.size = val[1] - val[0] + 1;
2801                 sc->vres.rq.start = val[2];
2802                 sc->vres.rq.size = val[3] - val[2] + 1;
2803                 sc->vres.pbl.start = val[4];
2804                 sc->vres.pbl.size = val[5] - val[4] + 1;
2805
2806                 param[0] = FW_PARAM_PFVF(SQRQ_START);
2807                 param[1] = FW_PARAM_PFVF(SQRQ_END);
2808                 param[2] = FW_PARAM_PFVF(CQ_START);
2809                 param[3] = FW_PARAM_PFVF(CQ_END);
2810                 param[4] = FW_PARAM_PFVF(OCQ_START);
2811                 param[5] = FW_PARAM_PFVF(OCQ_END);
2812                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2813                 if (rc != 0) {
2814                         device_printf(sc->dev,
2815                             "failed to query RDMA parameters(2): %d.\n", rc);
2816                         return (rc);
2817                 }
2818                 sc->vres.qp.start = val[0];
2819                 sc->vres.qp.size = val[1] - val[0] + 1;
2820                 sc->vres.cq.start = val[2];
2821                 sc->vres.cq.size = val[3] - val[2] + 1;
2822                 sc->vres.ocq.start = val[4];
2823                 sc->vres.ocq.size = val[5] - val[4] + 1;
2824         }
2825         if (sc->iscsicaps) {
2826                 param[0] = FW_PARAM_PFVF(ISCSI_START);
2827                 param[1] = FW_PARAM_PFVF(ISCSI_END);
2828                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2829                 if (rc != 0) {
2830                         device_printf(sc->dev,
2831                             "failed to query iSCSI parameters: %d.\n", rc);
2832                         return (rc);
2833                 }
2834                 sc->vres.iscsi.start = val[0];
2835                 sc->vres.iscsi.size = val[1] - val[0] + 1;
2836         }
2837
2838         /*
2839          * We've got the params we wanted to query via the firmware.  Now grab
2840          * some others directly from the chip.
2841          */
2842         rc = t4_read_chip_settings(sc);
2843
2844         return (rc);
2845 }
2846
2847 static int
2848 set_params__post_init(struct adapter *sc)
2849 {
2850         uint32_t param, val;
2851
2852         /* ask for encapsulated CPLs */
2853         param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
2854         val = 1;
2855         (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2856
2857         return (0);
2858 }
2859
2860 #undef FW_PARAM_PFVF
2861 #undef FW_PARAM_DEV
2862
2863 static void
2864 t4_set_desc(struct adapter *sc)
2865 {
2866         char buf[128];
2867         struct adapter_params *p = &sc->params;
2868
2869         snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
2870             "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
2871             chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
2872
2873         device_set_desc_copy(sc->dev, buf);
2874 }
2875
2876 static void
2877 build_medialist(struct port_info *pi, struct ifmedia *media)
2878 {
2879         int data, m;
2880
2881         PORT_LOCK(pi);
2882
2883         ifmedia_removeall(media);
2884
2885         m = IFM_ETHER | IFM_FDX;
2886         data = (pi->port_type << 8) | pi->mod_type;
2887
2888         switch(pi->port_type) {
2889         case FW_PORT_TYPE_BT_XFI:
2890                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2891                 break;
2892
2893         case FW_PORT_TYPE_BT_XAUI:
2894                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2895                 /* fall through */
2896
2897         case FW_PORT_TYPE_BT_SGMII:
2898                 ifmedia_add(media, m | IFM_1000_T, data, NULL);
2899                 ifmedia_add(media, m | IFM_100_TX, data, NULL);
2900                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL);
2901                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2902                 break;
2903
2904         case FW_PORT_TYPE_CX4:
2905                 ifmedia_add(media, m | IFM_10G_CX4, data, NULL);
2906                 ifmedia_set(media, m | IFM_10G_CX4);
2907                 break;
2908
2909         case FW_PORT_TYPE_QSFP_10G:
2910         case FW_PORT_TYPE_SFP:
2911         case FW_PORT_TYPE_FIBER_XFI:
2912         case FW_PORT_TYPE_FIBER_XAUI:
2913                 switch (pi->mod_type) {
2914
2915                 case FW_PORT_MOD_TYPE_LR:
2916                         ifmedia_add(media, m | IFM_10G_LR, data, NULL);
2917                         ifmedia_set(media, m | IFM_10G_LR);
2918                         break;
2919
2920                 case FW_PORT_MOD_TYPE_SR:
2921                         ifmedia_add(media, m | IFM_10G_SR, data, NULL);
2922                         ifmedia_set(media, m | IFM_10G_SR);
2923                         break;
2924
2925                 case FW_PORT_MOD_TYPE_LRM:
2926                         ifmedia_add(media, m | IFM_10G_LRM, data, NULL);
2927                         ifmedia_set(media, m | IFM_10G_LRM);
2928                         break;
2929
2930                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2931                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2932                         ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL);
2933                         ifmedia_set(media, m | IFM_10G_TWINAX);
2934                         break;
2935
2936                 case FW_PORT_MOD_TYPE_NONE:
2937                         m &= ~IFM_FDX;
2938                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2939                         ifmedia_set(media, m | IFM_NONE);
2940                         break;
2941
2942                 case FW_PORT_MOD_TYPE_NA:
2943                 case FW_PORT_MOD_TYPE_ER:
2944                 default:
2945                         device_printf(pi->dev,
2946                             "unknown port_type (%d), mod_type (%d)\n",
2947                             pi->port_type, pi->mod_type);
2948                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2949                         ifmedia_set(media, m | IFM_UNKNOWN);
2950                         break;
2951                 }
2952                 break;
2953
2954         case FW_PORT_TYPE_QSFP:
2955                 switch (pi->mod_type) {
2956
2957                 case FW_PORT_MOD_TYPE_LR:
2958                         ifmedia_add(media, m | IFM_40G_LR4, data, NULL);
2959                         ifmedia_set(media, m | IFM_40G_LR4);
2960                         break;
2961
2962                 case FW_PORT_MOD_TYPE_SR:
2963                         ifmedia_add(media, m | IFM_40G_SR4, data, NULL);
2964                         ifmedia_set(media, m | IFM_40G_SR4);
2965                         break;
2966
2967                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2968                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2969                         ifmedia_add(media, m | IFM_40G_CR4, data, NULL);
2970                         ifmedia_set(media, m | IFM_40G_CR4);
2971                         break;
2972
2973                 case FW_PORT_MOD_TYPE_NONE:
2974                         m &= ~IFM_FDX;
2975                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2976                         ifmedia_set(media, m | IFM_NONE);
2977                         break;
2978
2979                 default:
2980                         device_printf(pi->dev,
2981                             "unknown port_type (%d), mod_type (%d)\n",
2982                             pi->port_type, pi->mod_type);
2983                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2984                         ifmedia_set(media, m | IFM_UNKNOWN);
2985                         break;
2986                 }
2987                 break;
2988
2989         default:
2990                 device_printf(pi->dev,
2991                     "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
2992                     pi->mod_type);
2993                 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2994                 ifmedia_set(media, m | IFM_UNKNOWN);
2995                 break;
2996         }
2997
2998         PORT_UNLOCK(pi);
2999 }
3000
3001 #define FW_MAC_EXACT_CHUNK      7
3002
3003 /*
3004  * Program the port's XGMAC based on parameters in ifnet.  The caller also
3005  * indicates which parameters should be programmed (the rest are left alone).
3006  */
3007 int
3008 update_mac_settings(struct ifnet *ifp, int flags)
3009 {
3010         int rc = 0;
3011         struct port_info *pi = ifp->if_softc;
3012         struct adapter *sc = pi->adapter;
3013         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
3014         uint16_t viid = 0xffff;
3015         int16_t *xact_addr_filt = NULL;
3016
3017         ASSERT_SYNCHRONIZED_OP(sc);
3018         KASSERT(flags, ("%s: not told what to update.", __func__));
3019
3020         if (ifp == pi->ifp) {
3021                 viid = pi->viid;
3022                 xact_addr_filt = &pi->xact_addr_filt;
3023         }
3024 #ifdef DEV_NETMAP
3025         else if (ifp == pi->nm_ifp) {
3026                 viid = pi->nm_viid;
3027                 xact_addr_filt = &pi->nm_xact_addr_filt;
3028         }
3029 #endif
3030         if (flags & XGMAC_MTU)
3031                 mtu = ifp->if_mtu;
3032
3033         if (flags & XGMAC_PROMISC)
3034                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
3035
3036         if (flags & XGMAC_ALLMULTI)
3037                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
3038
3039         if (flags & XGMAC_VLANEX)
3040                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
3041
3042         if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
3043                 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti,
3044                     1, vlanex, false);
3045                 if (rc) {
3046                         if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
3047                             rc);
3048                         return (rc);
3049                 }
3050         }
3051
3052         if (flags & XGMAC_UCADDR) {
3053                 uint8_t ucaddr[ETHER_ADDR_LEN];
3054
3055                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
3056                 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr,
3057                     true, true);
3058                 if (rc < 0) {
3059                         rc = -rc;
3060                         if_printf(ifp, "change_mac failed: %d\n", rc);
3061                         return (rc);
3062                 } else {
3063                         *xact_addr_filt = rc;
3064                         rc = 0;
3065                 }
3066         }
3067
3068         if (flags & XGMAC_MCADDRS) {
3069                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
3070                 int del = 1;
3071                 uint64_t hash = 0;
3072                 struct ifmultiaddr *ifma;
3073                 int i = 0, j;
3074
3075                 if_maddr_rlock(ifp);
3076                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
3077                         if (ifma->ifma_addr->sa_family != AF_LINK)
3078                                 continue;
3079                         mcaddr[i] =
3080                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
3081                         MPASS(ETHER_IS_MULTICAST(mcaddr[i]));
3082                         i++;
3083
3084                         if (i == FW_MAC_EXACT_CHUNK) {
3085                                 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del,
3086                                     i, mcaddr, NULL, &hash, 0);
3087                                 if (rc < 0) {
3088                                         rc = -rc;
3089                                         for (j = 0; j < i; j++) {
3090                                                 if_printf(ifp,
3091                                                     "failed to add mc address"
3092                                                     " %02x:%02x:%02x:"
3093                                                     "%02x:%02x:%02x rc=%d\n",
3094                                                     mcaddr[j][0], mcaddr[j][1],
3095                                                     mcaddr[j][2], mcaddr[j][3],
3096                                                     mcaddr[j][4], mcaddr[j][5],
3097                                                     rc);
3098                                         }
3099                                         goto mcfail;
3100                                 }
3101                                 del = 0;
3102                                 i = 0;
3103                         }
3104                 }
3105                 if (i > 0) {
3106                         rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i,
3107                             mcaddr, NULL, &hash, 0);
3108                         if (rc < 0) {
3109                                 rc = -rc;
3110                                 for (j = 0; j < i; j++) {
3111                                         if_printf(ifp,
3112                                             "failed to add mc address"
3113                                             " %02x:%02x:%02x:"
3114                                             "%02x:%02x:%02x rc=%d\n",
3115                                             mcaddr[j][0], mcaddr[j][1],
3116                                             mcaddr[j][2], mcaddr[j][3],
3117                                             mcaddr[j][4], mcaddr[j][5],
3118                                             rc);
3119                                 }
3120                                 goto mcfail;
3121                         }
3122                 }
3123
3124                 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0);
3125                 if (rc != 0)
3126                         if_printf(ifp, "failed to set mc address hash: %d", rc);
3127 mcfail:
3128                 if_maddr_runlock(ifp);
3129         }
3130
3131         return (rc);
3132 }
3133
3134 int
3135 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
3136     char *wmesg)
3137 {
3138         int rc, pri;
3139
3140 #ifdef WITNESS
3141         /* the caller thinks it's ok to sleep, but is it really? */
3142         if (flags & SLEEP_OK)
3143                 pause("t4slptst", 1);
3144 #endif
3145
3146         if (INTR_OK)
3147                 pri = PCATCH;
3148         else
3149                 pri = 0;
3150
3151         ADAPTER_LOCK(sc);
3152         for (;;) {
3153
3154                 if (pi && IS_DOOMED(pi)) {
3155                         rc = ENXIO;
3156                         goto done;
3157                 }
3158
3159                 if (!IS_BUSY(sc)) {
3160                         rc = 0;
3161                         break;
3162                 }
3163
3164                 if (!(flags & SLEEP_OK)) {
3165                         rc = EBUSY;
3166                         goto done;
3167                 }
3168
3169                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
3170                         rc = EINTR;
3171                         goto done;
3172                 }
3173         }
3174
3175         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
3176         SET_BUSY(sc);
3177 #ifdef INVARIANTS
3178         sc->last_op = wmesg;
3179         sc->last_op_thr = curthread;
3180 #endif
3181
3182 done:
3183         if (!(flags & HOLD_LOCK) || rc)
3184                 ADAPTER_UNLOCK(sc);
3185
3186         return (rc);
3187 }
3188
3189 void
3190 end_synchronized_op(struct adapter *sc, int flags)
3191 {
3192
3193         if (flags & LOCK_HELD)
3194                 ADAPTER_LOCK_ASSERT_OWNED(sc);
3195         else
3196                 ADAPTER_LOCK(sc);
3197
3198         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
3199         CLR_BUSY(sc);
3200         wakeup(&sc->flags);
3201         ADAPTER_UNLOCK(sc);
3202 }
3203
3204 static int
3205 cxgbe_init_synchronized(struct port_info *pi)
3206 {
3207         struct adapter *sc = pi->adapter;
3208         struct ifnet *ifp = pi->ifp;
3209         int rc = 0, i;
3210         struct sge_txq *txq;
3211
3212         ASSERT_SYNCHRONIZED_OP(sc);
3213
3214         if (isset(&sc->open_device_map, pi->port_id)) {
3215                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
3216                     ("mismatch between open_device_map and if_drv_flags"));
3217                 return (0);     /* already running */
3218         }
3219
3220         if (!(sc->flags & FULL_INIT_DONE) &&
3221             ((rc = adapter_full_init(sc)) != 0))
3222                 return (rc);    /* error message displayed already */
3223
3224         if (!(pi->flags & PORT_INIT_DONE) &&
3225             ((rc = port_full_init(pi)) != 0))
3226                 return (rc); /* error message displayed already */
3227
3228         rc = update_mac_settings(ifp, XGMAC_ALL);
3229         if (rc)
3230                 goto done;      /* error message displayed already */
3231
3232         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
3233         if (rc != 0) {
3234                 if_printf(ifp, "enable_vi failed: %d\n", rc);
3235                 goto done;
3236         }
3237
3238         /*
3239          * Can't fail from this point onwards.  Review cxgbe_uninit_synchronized
3240          * if this changes.
3241          */
3242
3243         for_each_txq(pi, i, txq) {
3244                 TXQ_LOCK(txq);
3245                 txq->eq.flags |= EQ_ENABLED;
3246                 TXQ_UNLOCK(txq);
3247         }
3248
3249         /*
3250          * The first iq of the first port to come up is used for tracing.
3251          */
3252         if (sc->traceq < 0) {
3253                 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
3254                 t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
3255                     A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
3256                     V_QUEUENUMBER(sc->traceq));
3257                 pi->flags |= HAS_TRACEQ;
3258         }
3259
3260         /* all ok */
3261         setbit(&sc->open_device_map, pi->port_id);
3262         PORT_LOCK(pi);
3263         ifp->if_drv_flags |= IFF_DRV_RUNNING;
3264         PORT_UNLOCK(pi);
3265
3266         callout_reset(&pi->tick, hz, cxgbe_tick, pi);
3267 done:
3268         if (rc != 0)
3269                 cxgbe_uninit_synchronized(pi);
3270
3271         return (rc);
3272 }
3273
3274 /*
3275  * Idempotent.
3276  */
3277 static int
3278 cxgbe_uninit_synchronized(struct port_info *pi)
3279 {
3280         struct adapter *sc = pi->adapter;
3281         struct ifnet *ifp = pi->ifp;
3282         int rc, i;
3283         struct sge_txq *txq;
3284
3285         ASSERT_SYNCHRONIZED_OP(sc);
3286
3287         /*
3288          * Disable the VI so that all its data in either direction is discarded
3289          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
3290          * tick) intact as the TP can deliver negative advice or data that it's
3291          * holding in its RAM (for an offloaded connection) even after the VI is
3292          * disabled.
3293          */
3294         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
3295         if (rc) {
3296                 if_printf(ifp, "disable_vi failed: %d\n", rc);
3297                 return (rc);
3298         }
3299
3300         for_each_txq(pi, i, txq) {
3301                 TXQ_LOCK(txq);
3302                 txq->eq.flags &= ~EQ_ENABLED;
3303                 TXQ_UNLOCK(txq);
3304         }
3305
3306         clrbit(&sc->open_device_map, pi->port_id);
3307         PORT_LOCK(pi);
3308         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3309         PORT_UNLOCK(pi);
3310
3311         pi->link_cfg.link_ok = 0;
3312         pi->link_cfg.speed = 0;
3313         pi->linkdnrc = -1;
3314         t4_os_link_changed(sc, pi->port_id, 0, -1);
3315
3316         return (0);
3317 }
3318
3319 /*
3320  * It is ok for this function to fail midway and return right away.  t4_detach
3321  * will walk the entire sc->irq list and clean up whatever is valid.
3322  */
3323 static int
3324 setup_intr_handlers(struct adapter *sc)
3325 {
3326         int rc, rid, p, q;
3327         char s[8];
3328         struct irq *irq;
3329         struct port_info *pi;
3330         struct sge_rxq *rxq;
3331 #ifdef TCP_OFFLOAD
3332         struct sge_ofld_rxq *ofld_rxq;
3333 #endif
3334 #ifdef DEV_NETMAP
3335         struct sge_nm_rxq *nm_rxq;
3336 #endif
3337
3338         /*
3339          * Setup interrupts.
3340          */
3341         irq = &sc->irq[0];
3342         rid = sc->intr_type == INTR_INTX ? 0 : 1;
3343         if (sc->intr_count == 1)
3344                 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
3345
3346         /* Multiple interrupts. */
3347         KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
3348             ("%s: too few intr.", __func__));
3349
3350         /* The first one is always error intr */
3351         rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
3352         if (rc != 0)
3353                 return (rc);
3354         irq++;
3355         rid++;
3356
3357         /* The second one is always the firmware event queue */
3358         rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
3359         if (rc != 0)
3360                 return (rc);
3361         irq++;
3362         rid++;
3363
3364         for_each_port(sc, p) {
3365                 pi = sc->port[p];
3366
3367                 if (pi->flags & INTR_RXQ) {
3368                         for_each_rxq(pi, q, rxq) {
3369                                 snprintf(s, sizeof(s), "%d.%d", p, q);
3370                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
3371                                     s);
3372                                 if (rc != 0)
3373                                         return (rc);
3374                                 irq++;
3375                                 rid++;
3376                         }
3377                 }
3378 #ifdef TCP_OFFLOAD
3379                 if (pi->flags & INTR_OFLD_RXQ) {
3380                         for_each_ofld_rxq(pi, q, ofld_rxq) {
3381                                 snprintf(s, sizeof(s), "%d,%d", p, q);
3382                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr,
3383                                     ofld_rxq, s);
3384                                 if (rc != 0)
3385                                         return (rc);
3386                                 irq++;
3387                                 rid++;
3388                         }
3389                 }
3390 #endif
3391 #ifdef DEV_NETMAP
3392                 if (pi->flags & INTR_NM_RXQ) {
3393                         for_each_nm_rxq(pi, q, nm_rxq) {
3394                                 snprintf(s, sizeof(s), "%d-%d", p, q);
3395                                 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr,
3396                                     nm_rxq, s);
3397                                 if (rc != 0)
3398                                         return (rc);
3399                                 irq++;
3400                                 rid++;
3401                         }
3402                 }
3403 #endif
3404         }
3405         MPASS(irq == &sc->irq[sc->intr_count]);
3406
3407         return (0);
3408 }
3409
3410 int
3411 adapter_full_init(struct adapter *sc)
3412 {
3413         int rc, i;
3414
3415         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3416         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
3417             ("%s: FULL_INIT_DONE already", __func__));
3418
3419         /*
3420          * queues that belong to the adapter (not any particular port).
3421          */
3422         rc = t4_setup_adapter_queues(sc);
3423         if (rc != 0)
3424                 goto done;
3425
3426         for (i = 0; i < nitems(sc->tq); i++) {
3427                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
3428                     taskqueue_thread_enqueue, &sc->tq[i]);
3429                 if (sc->tq[i] == NULL) {
3430                         device_printf(sc->dev,
3431                             "failed to allocate task queue %d\n", i);
3432                         rc = ENOMEM;
3433                         goto done;
3434                 }
3435                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
3436                     device_get_nameunit(sc->dev), i);
3437         }
3438
3439         t4_intr_enable(sc);
3440         sc->flags |= FULL_INIT_DONE;
3441 done:
3442         if (rc != 0)
3443                 adapter_full_uninit(sc);
3444
3445         return (rc);
3446 }
3447
3448 int
3449 adapter_full_uninit(struct adapter *sc)
3450 {
3451         int i;
3452
3453         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3454
3455         t4_teardown_adapter_queues(sc);
3456
3457         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
3458                 taskqueue_free(sc->tq[i]);
3459                 sc->tq[i] = NULL;
3460         }
3461
3462         sc->flags &= ~FULL_INIT_DONE;
3463
3464         return (0);
3465 }
3466
3467 int
3468 port_full_init(struct port_info *pi)
3469 {
3470         struct adapter *sc = pi->adapter;
3471         struct ifnet *ifp = pi->ifp;
3472         uint16_t *rss;
3473         struct sge_rxq *rxq;
3474         int rc, i, j;
3475
3476         ASSERT_SYNCHRONIZED_OP(sc);
3477         KASSERT((pi->flags & PORT_INIT_DONE) == 0,
3478             ("%s: PORT_INIT_DONE already", __func__));
3479
3480         sysctl_ctx_init(&pi->ctx);
3481         pi->flags |= PORT_SYSCTL_CTX;
3482
3483         /*
3484          * Allocate tx/rx/fl queues for this port.
3485          */
3486         rc = t4_setup_port_queues(pi);
3487         if (rc != 0)
3488                 goto done;      /* error message displayed already */
3489
3490         /*
3491          * Setup RSS for this port.  Save a copy of the RSS table for later use.
3492          */
3493         rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
3494         for (i = 0; i < pi->rss_size;) {
3495                 for_each_rxq(pi, j, rxq) {
3496                         rss[i++] = rxq->iq.abs_id;
3497                         if (i == pi->rss_size)
3498                                 break;
3499                 }
3500         }
3501
3502         rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
3503             pi->rss_size);
3504         if (rc != 0) {
3505                 if_printf(ifp, "rss_config failed: %d\n", rc);
3506                 goto done;
3507         }
3508
3509         pi->rss = rss;
3510         pi->flags |= PORT_INIT_DONE;
3511 done:
3512         if (rc != 0)
3513                 port_full_uninit(pi);
3514
3515         return (rc);
3516 }
3517
3518 /*
3519  * Idempotent.
3520  */
3521 int
3522 port_full_uninit(struct port_info *pi)
3523 {
3524         struct adapter *sc = pi->adapter;
3525         int i;
3526         struct sge_rxq *rxq;
3527         struct sge_txq *txq;
3528 #ifdef TCP_OFFLOAD
3529         struct sge_ofld_rxq *ofld_rxq;
3530         struct sge_wrq *ofld_txq;
3531 #endif
3532
3533         if (pi->flags & PORT_INIT_DONE) {
3534
3535                 /* Need to quiesce queues.  */
3536
3537                 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
3538
3539                 for_each_txq(pi, i, txq) {
3540                         quiesce_txq(sc, txq);
3541                 }
3542
3543 #ifdef TCP_OFFLOAD
3544                 for_each_ofld_txq(pi, i, ofld_txq) {
3545                         quiesce_wrq(sc, ofld_txq);
3546                 }
3547 #endif
3548
3549                 for_each_rxq(pi, i, rxq) {
3550                         quiesce_iq(sc, &rxq->iq);
3551                         quiesce_fl(sc, &rxq->fl);
3552                 }
3553
3554 #ifdef TCP_OFFLOAD
3555                 for_each_ofld_rxq(pi, i, ofld_rxq) {
3556                         quiesce_iq(sc, &ofld_rxq->iq);
3557                         quiesce_fl(sc, &ofld_rxq->fl);
3558                 }
3559 #endif
3560                 free(pi->rss, M_CXGBE);
3561         }
3562
3563         t4_teardown_port_queues(pi);
3564         pi->flags &= ~PORT_INIT_DONE;
3565
3566         return (0);
3567 }
3568
3569 static void
3570 quiesce_txq(struct adapter *sc, struct sge_txq *txq)
3571 {
3572         struct sge_eq *eq = &txq->eq;
3573         struct sge_qstat *spg = (void *)&eq->desc[eq->sidx];
3574
3575         (void) sc;      /* unused */
3576
3577 #ifdef INVARIANTS
3578         TXQ_LOCK(txq);
3579         MPASS((eq->flags & EQ_ENABLED) == 0);
3580         TXQ_UNLOCK(txq);
3581 #endif
3582
3583         /* Wait for the mp_ring to empty. */
3584         while (!mp_ring_is_idle(txq->r)) {
3585                 mp_ring_check_drainage(txq->r, 0);
3586                 pause("rquiesce", 1);
3587         }
3588
3589         /* Then wait for the hardware to finish. */
3590         while (spg->cidx != htobe16(eq->pidx))
3591                 pause("equiesce", 1);
3592
3593         /* Finally, wait for the driver to reclaim all descriptors. */
3594         while (eq->cidx != eq->pidx)
3595                 pause("dquiesce", 1);
3596 }
3597
3598 static void
3599 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq)
3600 {
3601
3602         /* XXXTX */
3603 }
3604
3605 static void
3606 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
3607 {
3608         (void) sc;      /* unused */
3609
3610         /* Synchronize with the interrupt handler */
3611         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
3612                 pause("iqfree", 1);
3613 }
3614
3615 static void
3616 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
3617 {
3618         mtx_lock(&sc->sfl_lock);
3619         FL_LOCK(fl);
3620         fl->flags |= FL_DOOMED;
3621         FL_UNLOCK(fl);
3622         mtx_unlock(&sc->sfl_lock);
3623
3624         callout_drain(&sc->sfl_callout);
3625         KASSERT((fl->flags & FL_STARVING) == 0,
3626             ("%s: still starving", __func__));
3627 }
3628
3629 static int
3630 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
3631     driver_intr_t *handler, void *arg, char *name)
3632 {
3633         int rc;
3634
3635         irq->rid = rid;
3636         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
3637             RF_SHAREABLE | RF_ACTIVE);
3638         if (irq->res == NULL) {
3639                 device_printf(sc->dev,
3640                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
3641                 return (ENOMEM);
3642         }
3643
3644         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
3645             NULL, handler, arg, &irq->tag);
3646         if (rc != 0) {
3647                 device_printf(sc->dev,
3648                     "failed to setup interrupt for rid %d, name %s: %d\n",
3649                     rid, name, rc);
3650         } else if (name)
3651                 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
3652
3653         return (rc);
3654 }
3655
3656 static int
3657 t4_free_irq(struct adapter *sc, struct irq *irq)
3658 {
3659         if (irq->tag)
3660                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
3661         if (irq->res)
3662                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
3663
3664         bzero(irq, sizeof(*irq));
3665
3666         return (0);
3667 }
3668
3669 static void
3670 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
3671     unsigned int end)
3672 {
3673         uint32_t *p = (uint32_t *)(buf + start);
3674
3675         for ( ; start <= end; start += sizeof(uint32_t))
3676                 *p++ = t4_read_reg(sc, start);
3677 }
3678
3679 static void
3680 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
3681 {
3682         int i, n;
3683         const unsigned int *reg_ranges;
3684         static const unsigned int t4_reg_ranges[] = {
3685                 0x1008, 0x1108,
3686                 0x1180, 0x11b4,
3687                 0x11fc, 0x123c,
3688                 0x1300, 0x173c,
3689                 0x1800, 0x18fc,
3690                 0x3000, 0x30d8,
3691                 0x30e0, 0x5924,
3692                 0x5960, 0x59d4,
3693                 0x5a00, 0x5af8,
3694                 0x6000, 0x6098,
3695                 0x6100, 0x6150,
3696                 0x6200, 0x6208,
3697                 0x6240, 0x6248,
3698                 0x6280, 0x6338,
3699                 0x6370, 0x638c,
3700                 0x6400, 0x643c,
3701                 0x6500, 0x6524,
3702                 0x6a00, 0x6a38,
3703                 0x6a60, 0x6a78,
3704                 0x6b00, 0x6b84,
3705                 0x6bf0, 0x6c84,
3706                 0x6cf0, 0x6d84,
3707                 0x6df0, 0x6e84,
3708                 0x6ef0, 0x6f84,
3709                 0x6ff0, 0x7084,
3710                 0x70f0, 0x7184,
3711                 0x71f0, 0x7284,
3712                 0x72f0, 0x7384,
3713                 0x73f0, 0x7450,
3714                 0x7500, 0x7530,
3715                 0x7600, 0x761c,
3716                 0x7680, 0x76cc,
3717                 0x7700, 0x7798,
3718                 0x77c0, 0x77fc,
3719                 0x7900, 0x79fc,
3720                 0x7b00, 0x7c38,
3721                 0x7d00, 0x7efc,
3722                 0x8dc0, 0x8e1c,
3723                 0x8e30, 0x8e78,
3724                 0x8ea0, 0x8f6c,
3725                 0x8fc0, 0x9074,
3726                 0x90fc, 0x90fc,
3727                 0x9400, 0x9458,
3728                 0x9600, 0x96bc,
3729                 0x9800, 0x9808,
3730                 0x9820, 0x983c,
3731                 0x9850, 0x9864,
3732                 0x9c00, 0x9c6c,
3733                 0x9c80, 0x9cec,
3734                 0x9d00, 0x9d6c,
3735                 0x9d80, 0x9dec,
3736                 0x9e00, 0x9e6c,
3737                 0x9e80, 0x9eec,
3738                 0x9f00, 0x9f6c,
3739                 0x9f80, 0x9fec,
3740                 0xd004, 0xd03c,
3741                 0xdfc0, 0xdfe0,
3742                 0xe000, 0xea7c,
3743                 0xf000, 0x11110,
3744                 0x11118, 0x11190,
3745                 0x19040, 0x1906c,
3746                 0x19078, 0x19080,
3747                 0x1908c, 0x19124,
3748                 0x19150, 0x191b0,
3749                 0x191d0, 0x191e8,
3750                 0x19238, 0x1924c,
3751                 0x193f8, 0x19474,
3752                 0x19490, 0x194f8,
3753                 0x19800, 0x19f30,
3754                 0x1a000, 0x1a06c,
3755                 0x1a0b0, 0x1a120,
3756                 0x1a128, 0x1a138,
3757                 0x1a190, 0x1a1c4,
3758                 0x1a1fc, 0x1a1fc,
3759                 0x1e040, 0x1e04c,
3760                 0x1e284, 0x1e28c,
3761                 0x1e2c0, 0x1e2c0,
3762                 0x1e2e0, 0x1e2e0,
3763                 0x1e300, 0x1e384,
3764                 0x1e3c0, 0x1e3c8,
3765                 0x1e440, 0x1e44c,
3766                 0x1e684, 0x1e68c,
3767                 0x1e6c0, 0x1e6c0,
3768                 0x1e6e0, 0x1e6e0,
3769                 0x1e700, 0x1e784,
3770                 0x1e7c0, 0x1e7c8,
3771                 0x1e840, 0x1e84c,
3772                 0x1ea84, 0x1ea8c,
3773                 0x1eac0, 0x1eac0,
3774                 0x1eae0, 0x1eae0,
3775                 0x1eb00, 0x1eb84,
3776                 0x1ebc0, 0x1ebc8,
3777                 0x1ec40, 0x1ec4c,
3778                 0x1ee84, 0x1ee8c,
3779                 0x1eec0, 0x1eec0,
3780                 0x1eee0, 0x1eee0,
3781                 0x1ef00, 0x1ef84,
3782                 0x1efc0, 0x1efc8,
3783                 0x1f040, 0x1f04c,
3784                 0x1f284, 0x1f28c,
3785                 0x1f2c0, 0x1f2c0,
3786                 0x1f2e0, 0x1f2e0,
3787                 0x1f300, 0x1f384,
3788                 0x1f3c0, 0x1f3c8,
3789                 0x1f440, 0x1f44c,
3790                 0x1f684, 0x1f68c,
3791                 0x1f6c0, 0x1f6c0,
3792                 0x1f6e0, 0x1f6e0,
3793                 0x1f700, 0x1f784,
3794                 0x1f7c0, 0x1f7c8,
3795                 0x1f840, 0x1f84c,
3796                 0x1fa84, 0x1fa8c,
3797                 0x1fac0, 0x1fac0,
3798                 0x1fae0, 0x1fae0,
3799                 0x1fb00, 0x1fb84,
3800                 0x1fbc0, 0x1fbc8,
3801                 0x1fc40, 0x1fc4c,
3802                 0x1fe84, 0x1fe8c,
3803                 0x1fec0, 0x1fec0,
3804                 0x1fee0, 0x1fee0,
3805                 0x1ff00, 0x1ff84,
3806                 0x1ffc0, 0x1ffc8,
3807                 0x20000, 0x2002c,
3808                 0x20100, 0x2013c,
3809                 0x20190, 0x201c8,
3810                 0x20200, 0x20318,
3811                 0x20400, 0x20528,
3812                 0x20540, 0x20614,
3813                 0x21000, 0x21040,
3814                 0x2104c, 0x21060,
3815                 0x210c0, 0x210ec,
3816                 0x21200, 0x21268,
3817                 0x21270, 0x21284,
3818                 0x212fc, 0x21388,
3819                 0x21400, 0x21404,
3820                 0x21500, 0x21518,
3821                 0x2152c, 0x2153c,
3822                 0x21550, 0x21554,
3823                 0x21600, 0x21600,
3824                 0x21608, 0x21628,
3825                 0x21630, 0x2163c,
3826                 0x21700, 0x2171c,
3827                 0x21780, 0x2178c,
3828                 0x21800, 0x21c38,
3829                 0x21c80, 0x21d7c,
3830                 0x21e00, 0x21e04,
3831                 0x22000, 0x2202c,
3832                 0x22100, 0x2213c,
3833                 0x22190, 0x221c8,
3834                 0x22200, 0x22318,
3835                 0x22400, 0x22528,
3836                 0x22540, 0x22614,
3837                 0x23000, 0x23040,
3838                 0x2304c, 0x23060,
3839                 0x230c0, 0x230ec,
3840                 0x23200, 0x23268,
3841                 0x23270, 0x23284,
3842                 0x232fc, 0x23388,
3843                 0x23400, 0x23404,
3844                 0x23500, 0x23518,
3845                 0x2352c, 0x2353c,
3846                 0x23550, 0x23554,
3847                 0x23600, 0x23600,
3848                 0x23608, 0x23628,
3849                 0x23630, 0x2363c,
3850                 0x23700, 0x2371c,
3851                 0x23780, 0x2378c,
3852                 0x23800, 0x23c38,
3853                 0x23c80, 0x23d7c,
3854                 0x23e00, 0x23e04,
3855                 0x24000, 0x2402c,
3856                 0x24100, 0x2413c,
3857                 0x24190, 0x241c8,
3858                 0x24200, 0x24318,
3859                 0x24400, 0x24528,
3860                 0x24540, 0x24614,
3861                 0x25000, 0x25040,
3862                 0x2504c, 0x25060,
3863                 0x250c0, 0x250ec,
3864                 0x25200, 0x25268,
3865                 0x25270, 0x25284,
3866                 0x252fc, 0x25388,
3867                 0x25400, 0x25404,
3868                 0x25500, 0x25518,
3869                 0x2552c, 0x2553c,
3870                 0x25550, 0x25554,
3871                 0x25600, 0x25600,
3872                 0x25608, 0x25628,
3873                 0x25630, 0x2563c,
3874                 0x25700, 0x2571c,
3875                 0x25780, 0x2578c,
3876                 0x25800, 0x25c38,
3877                 0x25c80, 0x25d7c,
3878                 0x25e00, 0x25e04,
3879                 0x26000, 0x2602c,
3880                 0x26100, 0x2613c,
3881                 0x26190, 0x261c8,
3882                 0x26200, 0x26318,
3883                 0x26400, 0x26528,
3884                 0x26540, 0x26614,
3885                 0x27000, 0x27040,
3886                 0x2704c, 0x27060,
3887                 0x270c0, 0x270ec,
3888                 0x27200, 0x27268,
3889                 0x27270, 0x27284,
3890                 0x272fc, 0x27388,
3891                 0x27400, 0x27404,
3892                 0x27500, 0x27518,
3893                 0x2752c, 0x2753c,
3894                 0x27550, 0x27554,
3895                 0x27600, 0x27600,
3896                 0x27608, 0x27628,
3897                 0x27630, 0x2763c,
3898                 0x27700, 0x2771c,
3899                 0x27780, 0x2778c,
3900                 0x27800, 0x27c38,
3901                 0x27c80, 0x27d7c,
3902                 0x27e00, 0x27e04
3903         };
3904         static const unsigned int t5_reg_ranges[] = {
3905                 0x1008, 0x1148,
3906                 0x1180, 0x11b4,
3907                 0x11fc, 0x123c,
3908                 0x1280, 0x173c,
3909                 0x1800, 0x18fc,
3910                 0x3000, 0x3028,
3911                 0x3060, 0x30d8,
3912                 0x30e0, 0x30fc,
3913                 0x3140, 0x357c,
3914                 0x35a8, 0x35cc,
3915                 0x35ec, 0x35ec,
3916                 0x3600, 0x5624,
3917                 0x56cc, 0x575c,
3918                 0x580c, 0x5814,
3919                 0x5890, 0x58bc,
3920                 0x5940, 0x59dc,
3921                 0x59fc, 0x5a18,
3922                 0x5a60, 0x5a9c,
3923                 0x5b94, 0x5bfc,
3924                 0x6000, 0x6040,
3925                 0x6058, 0x614c,
3926                 0x7700, 0x7798,
3927                 0x77c0, 0x78fc,
3928                 0x7b00, 0x7c54,
3929                 0x7d00, 0x7efc,
3930                 0x8dc0, 0x8de0,
3931                 0x8df8, 0x8e84,
3932                 0x8ea0, 0x8f84,
3933                 0x8fc0, 0x90f8,
3934                 0x9400, 0x9470,
3935                 0x9600, 0x96f4,
3936                 0x9800, 0x9808,
3937                 0x9820, 0x983c,
3938                 0x9850, 0x9864,
3939                 0x9c00, 0x9c6c,
3940                 0x9c80, 0x9cec,
3941                 0x9d00, 0x9d6c,
3942                 0x9d80, 0x9dec,
3943                 0x9e00, 0x9e6c,
3944                 0x9e80, 0x9eec,
3945                 0x9f00, 0x9f6c,
3946                 0x9f80, 0xa020,
3947                 0xd004, 0xd03c,
3948                 0xdfc0, 0xdfe0,
3949                 0xe000, 0x11088,
3950                 0x1109c, 0x11110,
3951                 0x11118, 0x1117c,
3952                 0x11190, 0x11204,
3953                 0x19040, 0x1906c,
3954                 0x19078, 0x19080,
3955                 0x1908c, 0x19124,
3956                 0x19150, 0x191b0,
3957                 0x191d0, 0x191e8,
3958                 0x19238, 0x19290,
3959                 0x193f8, 0x19474,
3960                 0x19490, 0x194cc,
3961                 0x194f0, 0x194f8,
3962                 0x19c00, 0x19c60,
3963                 0x19c94, 0x19e10,
3964                 0x19e50, 0x19f34,
3965                 0x19f40, 0x19f50,
3966                 0x19f90, 0x19fe4,
3967                 0x1a000, 0x1a06c,
3968                 0x1a0b0, 0x1a120,
3969                 0x1a128, 0x1a138,
3970                 0x1a190, 0x1a1c4,
3971                 0x1a1fc, 0x1a1fc,
3972                 0x1e008, 0x1e00c,
3973                 0x1e040, 0x1e04c,
3974                 0x1e284, 0x1e290,
3975                 0x1e2c0, 0x1e2c0,
3976                 0x1e2e0, 0x1e2e0,
3977                 0x1e300, 0x1e384,
3978                 0x1e3c0, 0x1e3c8,
3979                 0x1e408, 0x1e40c,
3980                 0x1e440, 0x1e44c,
3981                 0x1e684, 0x1e690,
3982                 0x1e6c0, 0x1e6c0,
3983                 0x1e6e0, 0x1e6e0,
3984                 0x1e700, 0x1e784,
3985                 0x1e7c0, 0x1e7c8,
3986                 0x1e808, 0x1e80c,
3987                 0x1e840, 0x1e84c,
3988                 0x1ea84, 0x1ea90,
3989                 0x1eac0, 0x1eac0,
3990                 0x1eae0, 0x1eae0,
3991                 0x1eb00, 0x1eb84,
3992                 0x1ebc0, 0x1ebc8,
3993                 0x1ec08, 0x1ec0c,
3994                 0x1ec40, 0x1ec4c,
3995                 0x1ee84, 0x1ee90,
3996                 0x1eec0, 0x1eec0,
3997                 0x1eee0, 0x1eee0,
3998                 0x1ef00, 0x1ef84,
3999                 0x1efc0, 0x1efc8,
4000                 0x1f008, 0x1f00c,
4001                 0x1f040, 0x1f04c,
4002                 0x1f284, 0x1f290,
4003                 0x1f2c0, 0x1f2c0,
4004                 0x1f2e0, 0x1f2e0,
4005                 0x1f300, 0x1f384,
4006                 0x1f3c0, 0x1f3c8,
4007                 0x1f408, 0x1f40c,
4008                 0x1f440, 0x1f44c,
4009                 0x1f684, 0x1f690,
4010                 0x1f6c0, 0x1f6c0,
4011                 0x1f6e0, 0x1f6e0,
4012                 0x1f700, 0x1f784,
4013                 0x1f7c0, 0x1f7c8,
4014                 0x1f808, 0x1f80c,
4015                 0x1f840, 0x1f84c,
4016                 0x1fa84, 0x1fa90,
4017                 0x1fac0, 0x1fac0,
4018                 0x1fae0, 0x1fae0,
4019                 0x1fb00, 0x1fb84,
4020                 0x1fbc0, 0x1fbc8,
4021                 0x1fc08, 0x1fc0c,
4022                 0x1fc40, 0x1fc4c,
4023                 0x1fe84, 0x1fe90,
4024                 0x1fec0, 0x1fec0,
4025                 0x1fee0, 0x1fee0,
4026                 0x1ff00, 0x1ff84,
4027                 0x1ffc0, 0x1ffc8,
4028                 0x30000, 0x30030,
4029                 0x30100, 0x30144,
4030                 0x30190, 0x301d0,
4031                 0x30200, 0x30318,
4032                 0x30400, 0x3052c,
4033                 0x30540, 0x3061c,
4034                 0x30800, 0x30834,
4035                 0x308c0, 0x30908,
4036                 0x30910, 0x309ac,
4037                 0x30a00, 0x30a2c,
4038                 0x30a44, 0x30a50,
4039                 0x30a74, 0x30c24,
4040                 0x30d00, 0x30d00,
4041                 0x30d08, 0x30d14,
4042                 0x30d1c, 0x30d20,
4043                 0x30d3c, 0x30d50,
4044                 0x31200, 0x3120c,
4045                 0x31220, 0x31220,
4046                 0x31240, 0x31240,
4047                 0x31600, 0x3160c,
4048                 0x31a00, 0x31a1c,
4049                 0x31e00, 0x31e20,
4050                 0x31e38, 0x31e3c,
4051                 0x31e80, 0x31e80,
4052                 0x31e88, 0x31ea8,
4053                 0x31eb0, 0x31eb4,
4054                 0x31ec8, 0x31ed4,
4055                 0x31fb8, 0x32004,
4056                 0x32200, 0x32200,
4057                 0x32208, 0x32240,
4058                 0x32248, 0x32280,
4059                 0x32288, 0x322c0,
4060                 0x322c8, 0x322fc,
4061                 0x32600, 0x32630,
4062                 0x32a00, 0x32abc,
4063                 0x32b00, 0x32b70,
4064                 0x33000, 0x33048,
4065                 0x33060, 0x3309c,
4066                 0x330f0, 0x33148,
4067                 0x33160, 0x3319c,
4068                 0x331f0, 0x332e4,
4069                 0x332f8, 0x333e4,
4070                 0x333f8, 0x33448,
4071                 0x33460, 0x3349c,
4072                 0x334f0, 0x33548,
4073                 0x33560, 0x3359c,
4074                 0x335f0, 0x336e4,
4075                 0x336f8, 0x337e4,
4076                 0x337f8, 0x337fc,
4077                 0x33814, 0x33814,
4078                 0x3382c, 0x3382c,
4079                 0x33880, 0x3388c,
4080                 0x338e8, 0x338ec,
4081                 0x33900, 0x33948,
4082                 0x33960, 0x3399c,
4083                 0x339f0, 0x33ae4,
4084                 0x33af8, 0x33b10,
4085                 0x33b28, 0x33b28,
4086                 0x33b3c, 0x33b50,
4087                 0x33bf0, 0x33c10,
4088                 0x33c28, 0x33c28,
4089                 0x33c3c, 0x33c50,
4090                 0x33cf0, 0x33cfc,
4091                 0x34000, 0x34030,
4092                 0x34100, 0x34144,
4093                 0x34190, 0x341d0,
4094                 0x34200, 0x34318,
4095                 0x34400, 0x3452c,
4096                 0x34540, 0x3461c,
4097                 0x34800, 0x34834,
4098                 0x348c0, 0x34908,
4099                 0x34910, 0x349ac,
4100                 0x34a00, 0x34a2c,
4101                 0x34a44, 0x34a50,
4102                 0x34a74, 0x34c24,
4103                 0x34d00, 0x34d00,
4104                 0x34d08, 0x34d14,
4105                 0x34d1c, 0x34d20,
4106                 0x34d3c, 0x34d50,
4107                 0x35200, 0x3520c,
4108                 0x35220, 0x35220,
4109                 0x35240, 0x35240,
4110                 0x35600, 0x3560c,
4111                 0x35a00, 0x35a1c,
4112                 0x35e00, 0x35e20,
4113                 0x35e38, 0x35e3c,
4114                 0x35e80, 0x35e80,
4115                 0x35e88, 0x35ea8,
4116                 0x35eb0, 0x35eb4,
4117                 0x35ec8, 0x35ed4,
4118                 0x35fb8, 0x36004,
4119                 0x36200, 0x36200,
4120                 0x36208, 0x36240,
4121                 0x36248, 0x36280,
4122                 0x36288, 0x362c0,
4123                 0x362c8, 0x362fc,
4124                 0x36600, 0x36630,
4125                 0x36a00, 0x36abc,
4126                 0x36b00, 0x36b70,
4127                 0x37000, 0x37048,
4128                 0x37060, 0x3709c,
4129                 0x370f0, 0x37148,
4130                 0x37160, 0x3719c,
4131                 0x371f0, 0x372e4,
4132                 0x372f8, 0x373e4,
4133                 0x373f8, 0x37448,
4134                 0x37460, 0x3749c,
4135                 0x374f0, 0x37548,
4136                 0x37560, 0x3759c,
4137                 0x375f0, 0x376e4,
4138                 0x376f8, 0x377e4,
4139                 0x377f8, 0x377fc,
4140                 0x37814, 0x37814,
4141                 0x3782c, 0x3782c,
4142                 0x37880, 0x3788c,
4143                 0x378e8, 0x378ec,
4144                 0x37900, 0x37948,
4145                 0x37960, 0x3799c,
4146                 0x379f0, 0x37ae4,
4147                 0x37af8, 0x37b10,
4148                 0x37b28, 0x37b28,
4149                 0x37b3c, 0x37b50,
4150                 0x37bf0, 0x37c10,
4151                 0x37c28, 0x37c28,
4152                 0x37c3c, 0x37c50,
4153                 0x37cf0, 0x37cfc,
4154                 0x38000, 0x38030,
4155                 0x38100, 0x38144,
4156                 0x38190, 0x381d0,
4157                 0x38200, 0x38318,
4158                 0x38400, 0x3852c,
4159                 0x38540, 0x3861c,
4160                 0x38800, 0x38834,
4161                 0x388c0, 0x38908,
4162                 0x38910, 0x389ac,
4163                 0x38a00, 0x38a2c,
4164                 0x38a44, 0x38a50,
4165                 0x38a74, 0x38c24,
4166                 0x38d00, 0x38d00,
4167                 0x38d08, 0x38d14,
4168                 0x38d1c, 0x38d20,
4169                 0x38d3c, 0x38d50,
4170                 0x39200, 0x3920c,
4171                 0x39220, 0x39220,
4172                 0x39240, 0x39240,
4173                 0x39600, 0x3960c,
4174                 0x39a00, 0x39a1c,
4175                 0x39e00, 0x39e20,
4176                 0x39e38, 0x39e3c,
4177                 0x39e80, 0x39e80,
4178                 0x39e88, 0x39ea8,
4179                 0x39eb0, 0x39eb4,
4180                 0x39ec8, 0x39ed4,
4181                 0x39fb8, 0x3a004,
4182                 0x3a200, 0x3a200,
4183                 0x3a208, 0x3a240,
4184                 0x3a248, 0x3a280,
4185                 0x3a288, 0x3a2c0,
4186                 0x3a2c8, 0x3a2fc,
4187                 0x3a600, 0x3a630,
4188                 0x3aa00, 0x3aabc,
4189                 0x3ab00, 0x3ab70,
4190                 0x3b000, 0x3b048,
4191                 0x3b060, 0x3b09c,
4192                 0x3b0f0, 0x3b148,
4193                 0x3b160, 0x3b19c,
4194                 0x3b1f0, 0x3b2e4,
4195                 0x3b2f8, 0x3b3e4,
4196                 0x3b3f8, 0x3b448,
4197                 0x3b460, 0x3b49c,
4198                 0x3b4f0, 0x3b548,
4199                 0x3b560, 0x3b59c,
4200                 0x3b5f0, 0x3b6e4,
4201                 0x3b6f8, 0x3b7e4,
4202                 0x3b7f8, 0x3b7fc,
4203                 0x3b814, 0x3b814,
4204                 0x3b82c, 0x3b82c,
4205                 0x3b880, 0x3b88c,
4206                 0x3b8e8, 0x3b8ec,
4207                 0x3b900, 0x3b948,
4208                 0x3b960, 0x3b99c,
4209                 0x3b9f0, 0x3bae4,
4210                 0x3baf8, 0x3bb10,
4211                 0x3bb28, 0x3bb28,
4212                 0x3bb3c, 0x3bb50,
4213                 0x3bbf0, 0x3bc10,
4214                 0x3bc28, 0x3bc28,
4215                 0x3bc3c, 0x3bc50,
4216                 0x3bcf0, 0x3bcfc,
4217                 0x3c000, 0x3c030,
4218                 0x3c100, 0x3c144,
4219                 0x3c190, 0x3c1d0,
4220                 0x3c200, 0x3c318,
4221                 0x3c400, 0x3c52c,
4222                 0x3c540, 0x3c61c,
4223                 0x3c800, 0x3c834,
4224                 0x3c8c0, 0x3c908,
4225                 0x3c910, 0x3c9ac,
4226                 0x3ca00, 0x3ca2c,
4227                 0x3ca44, 0x3ca50,
4228                 0x3ca74, 0x3cc24,
4229                 0x3cd00, 0x3cd00,
4230                 0x3cd08, 0x3cd14,
4231                 0x3cd1c, 0x3cd20,
4232                 0x3cd3c, 0x3cd50,
4233                 0x3d200, 0x3d20c,
4234                 0x3d220, 0x3d220,
4235                 0x3d240, 0x3d240,
4236                 0x3d600, 0x3d60c,
4237                 0x3da00, 0x3da1c,
4238                 0x3de00, 0x3de20,
4239                 0x3de38, 0x3de3c,
4240                 0x3de80, 0x3de80,
4241                 0x3de88, 0x3dea8,
4242                 0x3deb0, 0x3deb4,
4243                 0x3dec8, 0x3ded4,
4244                 0x3dfb8, 0x3e004,
4245                 0x3e200, 0x3e200,
4246                 0x3e208, 0x3e240,
4247                 0x3e248, 0x3e280,
4248                 0x3e288, 0x3e2c0,
4249                 0x3e2c8, 0x3e2fc,
4250                 0x3e600, 0x3e630,
4251                 0x3ea00, 0x3eabc,
4252                 0x3eb00, 0x3eb70,
4253                 0x3f000, 0x3f048,
4254                 0x3f060, 0x3f09c,
4255                 0x3f0f0, 0x3f148,
4256                 0x3f160, 0x3f19c,
4257                 0x3f1f0, 0x3f2e4,
4258                 0x3f2f8, 0x3f3e4,
4259                 0x3f3f8, 0x3f448,
4260                 0x3f460, 0x3f49c,
4261                 0x3f4f0, 0x3f548,
4262                 0x3f560, 0x3f59c,
4263                 0x3f5f0, 0x3f6e4,
4264                 0x3f6f8, 0x3f7e4,
4265                 0x3f7f8, 0x3f7fc,
4266                 0x3f814, 0x3f814,
4267                 0x3f82c, 0x3f82c,
4268                 0x3f880, 0x3f88c,
4269                 0x3f8e8, 0x3f8ec,
4270                 0x3f900, 0x3f948,
4271                 0x3f960, 0x3f99c,
4272                 0x3f9f0, 0x3fae4,
4273                 0x3faf8, 0x3fb10,
4274                 0x3fb28, 0x3fb28,
4275                 0x3fb3c, 0x3fb50,
4276                 0x3fbf0, 0x3fc10,
4277                 0x3fc28, 0x3fc28,
4278                 0x3fc3c, 0x3fc50,
4279                 0x3fcf0, 0x3fcfc,
4280                 0x40000, 0x4000c,
4281                 0x40040, 0x40068,
4282                 0x4007c, 0x40144,
4283                 0x40180, 0x4018c,
4284                 0x40200, 0x40298,
4285                 0x402ac, 0x4033c,
4286                 0x403f8, 0x403fc,
4287                 0x41304, 0x413c4,
4288                 0x41400, 0x4141c,
4289                 0x41480, 0x414d0,
4290                 0x44000, 0x44078,
4291                 0x440c0, 0x44278,
4292                 0x442c0, 0x44478,
4293                 0x444c0, 0x44678,
4294                 0x446c0, 0x44878,
4295                 0x448c0, 0x449fc,
4296                 0x45000, 0x45068,
4297                 0x45080, 0x45084,
4298                 0x450a0, 0x450b0,
4299                 0x45200, 0x45268,
4300                 0x45280, 0x45284,
4301                 0x452a0, 0x452b0,
4302                 0x460c0, 0x460e4,
4303                 0x47000, 0x4708c,
4304                 0x47200, 0x47250,
4305                 0x47400, 0x47420,
4306                 0x47600, 0x47618,
4307                 0x47800, 0x47814,
4308                 0x48000, 0x4800c,
4309                 0x48040, 0x48068,
4310                 0x4807c, 0x48144,
4311                 0x48180, 0x4818c,
4312                 0x48200, 0x48298,
4313                 0x482ac, 0x4833c,
4314                 0x483f8, 0x483fc,
4315                 0x49304, 0x493c4,
4316                 0x49400, 0x4941c,
4317                 0x49480, 0x494d0,
4318                 0x4c000, 0x4c078,
4319                 0x4c0c0, 0x4c278,
4320                 0x4c2c0, 0x4c478,
4321                 0x4c4c0, 0x4c678,
4322                 0x4c6c0, 0x4c878,
4323                 0x4c8c0, 0x4c9fc,
4324                 0x4d000, 0x4d068,
4325                 0x4d080, 0x4d084,
4326                 0x4d0a0, 0x4d0b0,
4327                 0x4d200, 0x4d268,
4328                 0x4d280, 0x4d284,
4329                 0x4d2a0, 0x4d2b0,
4330                 0x4e0c0, 0x4e0e4,
4331                 0x4f000, 0x4f08c,
4332                 0x4f200, 0x4f250,
4333                 0x4f400, 0x4f420,
4334                 0x4f600, 0x4f618,
4335                 0x4f800, 0x4f814,
4336                 0x50000, 0x500cc,
4337                 0x50400, 0x50400,
4338                 0x50800, 0x508cc,
4339                 0x50c00, 0x50c00,
4340                 0x51000, 0x5101c,
4341                 0x51300, 0x51308,
4342         };
4343
4344         if (is_t4(sc)) {
4345                 reg_ranges = &t4_reg_ranges[0];
4346                 n = nitems(t4_reg_ranges);
4347         } else {
4348                 reg_ranges = &t5_reg_ranges[0];
4349                 n = nitems(t5_reg_ranges);
4350         }
4351
4352         regs->version = chip_id(sc) | chip_rev(sc) << 10;
4353         for (i = 0; i < n; i += 2)
4354                 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
4355 }
4356
4357 static void
4358 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi)
4359 {
4360         int i;
4361         u_int v, tnl_cong_drops;
4362         struct timeval tv;
4363         const struct timeval interval = {0, 250000};    /* 250ms */
4364
4365         getmicrotime(&tv);
4366         timevalsub(&tv, &interval);
4367         if (timevalcmp(&tv, &pi->last_refreshed, <))
4368                 return;
4369
4370         tnl_cong_drops = 0;
4371         t4_get_port_stats(sc, pi->tx_chan, &pi->stats);
4372         for (i = 0; i < NCHAN; i++) {
4373                 if (pi->rx_chan_map & (1 << i)) {
4374                         mtx_lock(&sc->regwin_lock);
4375                         t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
4376                             1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4377                         mtx_unlock(&sc->regwin_lock);
4378                         tnl_cong_drops += v;
4379                 }
4380         }
4381         pi->tnl_cong_drops = tnl_cong_drops;
4382         getmicrotime(&pi->last_refreshed);
4383 }
4384
4385 static void
4386 cxgbe_tick(void *arg)
4387 {
4388         struct port_info *pi = arg;
4389         struct adapter *sc = pi->adapter;
4390         struct ifnet *ifp = pi->ifp;
4391
4392         PORT_LOCK(pi);
4393         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4394                 PORT_UNLOCK(pi);
4395                 return; /* without scheduling another callout */
4396         }
4397
4398         cxgbe_refresh_stats(sc, pi);
4399
4400         callout_schedule(&pi->tick, hz);
4401         PORT_UNLOCK(pi);
4402 }
4403
4404 static void
4405 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4406 {
4407         struct ifnet *vlan;
4408
4409         if (arg != ifp || ifp->if_type != IFT_ETHER)
4410                 return;
4411
4412         vlan = VLAN_DEVAT(ifp, vid);
4413         VLAN_SETCOOKIE(vlan, ifp);
4414 }
4415
4416 static int
4417 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4418 {
4419
4420 #ifdef INVARIANTS
4421         panic("%s: opcode 0x%02x on iq %p with payload %p",
4422             __func__, rss->opcode, iq, m);
4423 #else
4424         log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
4425             __func__, rss->opcode, iq, m);
4426         m_freem(m);
4427 #endif
4428         return (EDOOFUS);
4429 }
4430
4431 int
4432 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
4433 {
4434         uintptr_t *loc, new;
4435
4436         if (opcode >= nitems(sc->cpl_handler))
4437                 return (EINVAL);
4438
4439         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
4440         loc = (uintptr_t *) &sc->cpl_handler[opcode];
4441         atomic_store_rel_ptr(loc, new);
4442
4443         return (0);
4444 }
4445
4446 static int
4447 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
4448 {
4449
4450 #ifdef INVARIANTS
4451         panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
4452 #else
4453         log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
4454             __func__, iq, ctrl);
4455 #endif
4456         return (EDOOFUS);
4457 }
4458
4459 int
4460 t4_register_an_handler(struct adapter *sc, an_handler_t h)
4461 {
4462         uintptr_t *loc, new;
4463
4464         new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
4465         loc = (uintptr_t *) &sc->an_handler;
4466         atomic_store_rel_ptr(loc, new);
4467
4468         return (0);
4469 }
4470
4471 static int
4472 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
4473 {
4474         const struct cpl_fw6_msg *cpl =
4475             __containerof(rpl, struct cpl_fw6_msg, data[0]);
4476
4477 #ifdef INVARIANTS
4478         panic("%s: fw_msg type %d", __func__, cpl->type);
4479 #else
4480         log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
4481 #endif
4482         return (EDOOFUS);
4483 }
4484
4485 int
4486 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
4487 {
4488         uintptr_t *loc, new;
4489
4490         if (type >= nitems(sc->fw_msg_handler))
4491                 return (EINVAL);
4492
4493         /*
4494          * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
4495          * handler dispatch table.  Reject any attempt to install a handler for
4496          * this subtype.
4497          */
4498         if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
4499                 return (EINVAL);
4500
4501         new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
4502         loc = (uintptr_t *) &sc->fw_msg_handler[type];
4503         atomic_store_rel_ptr(loc, new);
4504
4505         return (0);
4506 }
4507
4508 static int
4509 t4_sysctls(struct adapter *sc)
4510 {
4511         struct sysctl_ctx_list *ctx;
4512         struct sysctl_oid *oid;
4513         struct sysctl_oid_list *children, *c0;
4514         static char *caps[] = {
4515                 "\20\1PPP\2QFC\3DCBX",                  /* caps[0] linkcaps */
4516                 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL"        /* caps[1] niccaps */
4517                     "\6HASHFILTER\7ETHOFLD",
4518                 "\20\1TOE",                             /* caps[2] toecaps */
4519                 "\20\1RDDP\2RDMAC",                     /* caps[3] rdmacaps */
4520                 "\20\1INITIATOR_PDU\2TARGET_PDU"        /* caps[4] iscsicaps */
4521                     "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
4522                     "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
4523                 "\20\1INITIATOR\2TARGET\3CTRL_OFLD"     /* caps[5] fcoecaps */
4524                     "\4PO_INITIAOR\5PO_TARGET"
4525         };
4526         static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
4527
4528         ctx = device_get_sysctl_ctx(sc->dev);
4529
4530         /*
4531          * dev.t4nex.X.
4532          */
4533         oid = device_get_sysctl_tree(sc->dev);
4534         c0 = children = SYSCTL_CHILDREN(oid);
4535
4536         sc->sc_do_rxcopy = 1;
4537         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
4538             &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
4539
4540         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
4541             sc->params.nports, "# of ports");
4542
4543         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
4544             NULL, chip_rev(sc), "chip hardware revision");
4545
4546         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
4547             CTLFLAG_RD, sc->fw_version, 0, "firmware version");
4548
4549         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
4550             CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
4551
4552         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
4553             sc->cfcsum, "config file checksum");
4554
4555         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
4556             CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
4557             sysctl_bitfield, "A", "available doorbells");
4558
4559         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
4560             CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
4561             sysctl_bitfield, "A", "available link capabilities");
4562
4563         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
4564             CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
4565             sysctl_bitfield, "A", "available NIC capabilities");
4566
4567         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
4568             CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
4569             sysctl_bitfield, "A", "available TCP offload capabilities");
4570
4571         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
4572             CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
4573             sysctl_bitfield, "A", "available RDMA capabilities");
4574
4575         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
4576             CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
4577             sysctl_bitfield, "A", "available iSCSI capabilities");
4578
4579         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
4580             CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
4581             sysctl_bitfield, "A", "available FCoE capabilities");
4582
4583         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
4584             sc->params.vpd.cclk, "core clock frequency (in KHz)");
4585
4586         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
4587             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
4588             sizeof(sc->sge.timer_val), sysctl_int_array, "A",
4589             "interrupt holdoff timer values (us)");
4590
4591         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
4592             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
4593             sizeof(sc->sge.counter_val), sysctl_int_array, "A",
4594             "interrupt holdoff packet counter values");
4595
4596         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
4597             NULL, sc->tids.nftids, "number of filters");
4598
4599         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
4600             CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
4601             "chip temperature (in Celsius)");
4602
4603         t4_sge_sysctls(sc, ctx, children);
4604
4605         sc->lro_timeout = 100;
4606         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
4607             &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
4608
4609 #ifdef SBUF_DRAIN
4610         /*
4611          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
4612          */
4613         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
4614             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
4615             "logs and miscellaneous information");
4616         children = SYSCTL_CHILDREN(oid);
4617
4618         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
4619             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4620             sysctl_cctrl, "A", "congestion control");
4621
4622         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
4623             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4624             sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
4625
4626         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
4627             CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
4628             sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
4629
4630         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
4631             CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
4632             sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
4633
4634         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
4635             CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
4636             sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
4637
4638         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
4639             CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
4640             sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
4641
4642         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
4643             CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
4644             sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
4645
4646         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
4647             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4648             sysctl_cim_la, "A", "CIM logic analyzer");
4649
4650         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
4651             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4652             sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
4653
4654         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
4655             CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
4656             sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
4657
4658         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
4659             CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
4660             sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
4661
4662         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
4663             CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
4664             sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
4665
4666         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
4667             CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
4668             sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
4669
4670         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
4671             CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
4672             sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
4673
4674         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
4675             CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
4676             sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
4677
4678         if (is_t5(sc)) {
4679                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
4680                     CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
4681                     sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
4682
4683                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
4684                     CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
4685                     sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
4686         }
4687
4688         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
4689             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4690             sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
4691
4692         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
4693             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4694             sysctl_cim_qcfg, "A", "CIM queue configuration");
4695
4696         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
4697             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4698             sysctl_cpl_stats, "A", "CPL statistics");
4699
4700         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
4701             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4702             sysctl_ddp_stats, "A", "non-TCP DDP statistics");
4703
4704         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
4705             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4706             sysctl_devlog, "A", "firmware's device log");
4707
4708         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
4709             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4710             sysctl_fcoe_stats, "A", "FCoE statistics");
4711
4712         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
4713             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4714             sysctl_hw_sched, "A", "hardware scheduler ");
4715
4716         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
4717             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4718             sysctl_l2t, "A", "hardware L2 table");
4719
4720         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
4721             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4722             sysctl_lb_stats, "A", "loopback statistics");
4723
4724         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
4725             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4726             sysctl_meminfo, "A", "memory regions");
4727
4728         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
4729             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4730             sysctl_mps_tcam, "A", "MPS TCAM entries");
4731
4732         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
4733             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4734             sysctl_path_mtus, "A", "path MTUs");
4735
4736         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
4737             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4738             sysctl_pm_stats, "A", "PM statistics");
4739
4740         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
4741             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4742             sysctl_rdma_stats, "A", "RDMA statistics");
4743
4744         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
4745             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4746             sysctl_tcp_stats, "A", "TCP statistics");
4747
4748         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
4749             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4750             sysctl_tids, "A", "TID information");
4751
4752         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
4753             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4754             sysctl_tp_err_stats, "A", "TP error statistics");
4755
4756         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
4757             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4758             sysctl_tp_la, "A", "TP logic analyzer");
4759
4760         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
4761             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4762             sysctl_tx_rate, "A", "Tx rate");
4763
4764         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
4765             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4766             sysctl_ulprx_la, "A", "ULPRX logic analyzer");
4767
4768         if (is_t5(sc)) {
4769                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
4770                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4771                     sysctl_wcwr_stats, "A", "write combined work requests");
4772         }
4773 #endif
4774
4775 #ifdef TCP_OFFLOAD
4776         if (is_offload(sc)) {
4777                 /*
4778                  * dev.t4nex.X.toe.
4779                  */
4780                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
4781                     NULL, "TOE parameters");
4782                 children = SYSCTL_CHILDREN(oid);
4783
4784                 sc->tt.sndbuf = 256 * 1024;
4785                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
4786                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
4787
4788                 sc->tt.ddp = 0;
4789                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
4790                     &sc->tt.ddp, 0, "DDP allowed");
4791
4792                 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
4793                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
4794                     &sc->tt.indsz, 0, "DDP max indicate size allowed");
4795
4796                 sc->tt.ddp_thres =
4797                     G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
4798                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
4799                     &sc->tt.ddp_thres, 0, "DDP threshold");
4800
4801                 sc->tt.rx_coalesce = 1;
4802                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
4803                     CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
4804
4805                 sc->tt.tx_align = 1;
4806                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align",
4807                     CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload");
4808         }
4809 #endif
4810
4811
4812         return (0);
4813 }
4814
4815 static int
4816 cxgbe_sysctls(struct port_info *pi)
4817 {
4818         struct sysctl_ctx_list *ctx;
4819         struct sysctl_oid *oid;
4820         struct sysctl_oid_list *children;
4821         struct adapter *sc = pi->adapter;
4822
4823         ctx = device_get_sysctl_ctx(pi->dev);
4824
4825         /*
4826          * dev.cxgbe.X.
4827          */
4828         oid = device_get_sysctl_tree(pi->dev);
4829         children = SYSCTL_CHILDREN(oid);
4830
4831         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
4832            CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
4833         if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
4834                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
4835                     CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
4836                     "PHY temperature (in Celsius)");
4837                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
4838                     CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
4839                     "PHY firmware version");
4840         }
4841         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
4842             &pi->nrxq, 0, "# of rx queues");
4843         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
4844             &pi->ntxq, 0, "# of tx queues");
4845         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
4846             &pi->first_rxq, 0, "index of first rx queue");
4847         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
4848             &pi->first_txq, 0, "index of first tx queue");
4849         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
4850             CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU",
4851             "Reserve queue 0 for non-flowid packets");
4852
4853 #ifdef TCP_OFFLOAD
4854         if (is_offload(sc)) {
4855                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
4856                     &pi->nofldrxq, 0,
4857                     "# of rx queues for offloaded TCP connections");
4858                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
4859                     &pi->nofldtxq, 0,
4860                     "# of tx queues for offloaded TCP connections");
4861                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
4862                     CTLFLAG_RD, &pi->first_ofld_rxq, 0,
4863                     "index of first TOE rx queue");
4864                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
4865                     CTLFLAG_RD, &pi->first_ofld_txq, 0,
4866                     "index of first TOE tx queue");
4867         }
4868 #endif
4869 #ifdef DEV_NETMAP
4870         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
4871             &pi->nnmrxq, 0, "# of rx queues for netmap");
4872         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
4873             &pi->nnmtxq, 0, "# of tx queues for netmap");
4874         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
4875             CTLFLAG_RD, &pi->first_nm_rxq, 0,
4876             "index of first netmap rx queue");
4877         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
4878             CTLFLAG_RD, &pi->first_nm_txq, 0,
4879             "index of first netmap tx queue");
4880 #endif
4881
4882         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
4883             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
4884             "holdoff timer index");
4885         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
4886             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
4887             "holdoff packet counter index");
4888
4889         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
4890             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
4891             "rx queue size");
4892         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
4893             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
4894             "tx queue size");
4895
4896         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
4897             CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings,
4898             "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
4899
4900         /*
4901          * dev.cxgbe.X.stats.
4902          */
4903         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
4904             NULL, "port statistics");
4905         children = SYSCTL_CHILDREN(oid);
4906         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD,
4907             &pi->tx_parse_error, 0,
4908             "# of tx packets with invalid length or # of segments");
4909
4910 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
4911         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
4912             CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
4913             sysctl_handle_t4_reg64, "QU", desc)
4914
4915         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
4916             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
4917         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
4918             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
4919         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
4920             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
4921         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
4922             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
4923         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
4924             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
4925         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
4926             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
4927         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
4928             "# of tx frames in this range",
4929             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
4930         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
4931             "# of tx frames in this range",
4932             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
4933         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
4934             "# of tx frames in this range",
4935             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
4936         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
4937             "# of tx frames in this range",
4938             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
4939         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
4940             "# of tx frames in this range",
4941             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
4942         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
4943             "# of tx frames in this range",
4944             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
4945         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
4946             "# of tx frames in this range",
4947             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
4948         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
4949             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
4950         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
4951             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
4952         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
4953             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
4954         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
4955             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
4956         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
4957             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
4958         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
4959             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
4960         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
4961             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
4962         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
4963             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
4964         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
4965             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
4966         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
4967             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
4968
4969         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
4970             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
4971         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
4972             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
4973         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
4974             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
4975         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
4976             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
4977         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
4978             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
4979         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
4980             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
4981         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
4982             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
4983         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
4984             "# of frames received with bad FCS",
4985             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
4986         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
4987             "# of frames received with length error",
4988             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
4989         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
4990             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
4991         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
4992             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
4993         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
4994             "# of rx frames in this range",
4995             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
4996         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
4997             "# of rx frames in this range",
4998             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
4999         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
5000             "# of rx frames in this range",
5001             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
5002         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
5003             "# of rx frames in this range",
5004             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
5005         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
5006             "# of rx frames in this range",
5007             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
5008         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
5009             "# of rx frames in this range",
5010             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
5011         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
5012             "# of rx frames in this range",
5013             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
5014         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
5015             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
5016         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
5017             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
5018         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
5019             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
5020         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
5021             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
5022         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
5023             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
5024         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
5025             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
5026         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
5027             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
5028         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
5029             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
5030         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
5031             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
5032
5033 #undef SYSCTL_ADD_T4_REG64
5034
5035 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
5036         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
5037             &pi->stats.name, desc)
5038
5039         /* We get these from port_stats and they may be stale by upto 1s */
5040         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
5041             "# drops due to buffer-group 0 overflows");
5042         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
5043             "# drops due to buffer-group 1 overflows");
5044         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
5045             "# drops due to buffer-group 2 overflows");
5046         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
5047             "# drops due to buffer-group 3 overflows");
5048         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
5049             "# of buffer-group 0 truncated packets");
5050         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
5051             "# of buffer-group 1 truncated packets");
5052         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
5053             "# of buffer-group 2 truncated packets");
5054         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
5055             "# of buffer-group 3 truncated packets");
5056
5057 #undef SYSCTL_ADD_T4_PORTSTAT
5058
5059         return (0);
5060 }
5061
5062 static int
5063 sysctl_int_array(SYSCTL_HANDLER_ARGS)
5064 {
5065         int rc, *i;
5066         struct sbuf sb;
5067
5068         sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
5069         for (i = arg1; arg2; arg2 -= sizeof(int), i++)
5070                 sbuf_printf(&sb, "%d ", *i);
5071         sbuf_trim(&sb);
5072         sbuf_finish(&sb);
5073         rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
5074         sbuf_delete(&sb);
5075         return (rc);
5076 }
5077
5078 static int
5079 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
5080 {
5081         int rc;
5082         struct sbuf *sb;
5083
5084         rc = sysctl_wire_old_buffer(req, 0);
5085         if (rc != 0)
5086                 return(rc);
5087
5088         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5089         if (sb == NULL)
5090                 return (ENOMEM);
5091
5092         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
5093         rc = sbuf_finish(sb);
5094         sbuf_delete(sb);
5095
5096         return (rc);
5097 }
5098
5099 static int
5100 sysctl_btphy(SYSCTL_HANDLER_ARGS)
5101 {
5102         struct port_info *pi = arg1;
5103         int op = arg2;
5104         struct adapter *sc = pi->adapter;
5105         u_int v;
5106         int rc;
5107
5108         rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt");
5109         if (rc)
5110                 return (rc);
5111         /* XXX: magic numbers */
5112         rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
5113             &v);
5114         end_synchronized_op(sc, 0);
5115         if (rc)
5116                 return (rc);
5117         if (op == 0)
5118                 v /= 256;
5119
5120         rc = sysctl_handle_int(oidp, &v, 0, req);
5121         return (rc);
5122 }
5123
5124 static int
5125 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
5126 {
5127         struct port_info *pi = arg1;
5128         int rc, val;
5129
5130         val = pi->rsrv_noflowq;
5131         rc = sysctl_handle_int(oidp, &val, 0, req);
5132         if (rc != 0 || req->newptr == NULL)
5133                 return (rc);
5134
5135         if ((val >= 1) && (pi->ntxq > 1))
5136                 pi->rsrv_noflowq = 1;
5137         else
5138                 pi->rsrv_noflowq = 0;
5139
5140         return (rc);
5141 }
5142
5143 static int
5144 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
5145 {
5146         struct port_info *pi = arg1;
5147         struct adapter *sc = pi->adapter;
5148         int idx, rc, i;
5149         struct sge_rxq *rxq;
5150 #ifdef TCP_OFFLOAD
5151         struct sge_ofld_rxq *ofld_rxq;
5152 #endif
5153         uint8_t v;
5154
5155         idx = pi->tmr_idx;
5156
5157         rc = sysctl_handle_int(oidp, &idx, 0, req);
5158         if (rc != 0 || req->newptr == NULL)
5159                 return (rc);
5160
5161         if (idx < 0 || idx >= SGE_NTIMERS)
5162                 return (EINVAL);
5163
5164         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5165             "t4tmr");
5166         if (rc)
5167                 return (rc);
5168
5169         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
5170         for_each_rxq(pi, i, rxq) {
5171 #ifdef atomic_store_rel_8
5172                 atomic_store_rel_8(&rxq->iq.intr_params, v);
5173 #else
5174                 rxq->iq.intr_params = v;
5175 #endif
5176         }
5177 #ifdef TCP_OFFLOAD
5178         for_each_ofld_rxq(pi, i, ofld_rxq) {
5179 #ifdef atomic_store_rel_8
5180                 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
5181 #else
5182                 ofld_rxq->iq.intr_params = v;
5183 #endif
5184         }
5185 #endif
5186         pi->tmr_idx = idx;
5187
5188         end_synchronized_op(sc, LOCK_HELD);
5189         return (0);
5190 }
5191
5192 static int
5193 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
5194 {
5195         struct port_info *pi = arg1;
5196         struct adapter *sc = pi->adapter;
5197         int idx, rc;
5198
5199         idx = pi->pktc_idx;
5200
5201         rc = sysctl_handle_int(oidp, &idx, 0, req);
5202         if (rc != 0 || req->newptr == NULL)
5203                 return (rc);
5204
5205         if (idx < -1 || idx >= SGE_NCOUNTERS)
5206                 return (EINVAL);
5207
5208         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5209             "t4pktc");
5210         if (rc)
5211                 return (rc);
5212
5213         if (pi->flags & PORT_INIT_DONE)
5214                 rc = EBUSY; /* cannot be changed once the queues are created */
5215         else
5216                 pi->pktc_idx = idx;
5217
5218         end_synchronized_op(sc, LOCK_HELD);
5219         return (rc);
5220 }
5221
5222 static int
5223 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
5224 {
5225         struct port_info *pi = arg1;
5226         struct adapter *sc = pi->adapter;
5227         int qsize, rc;
5228
5229         qsize = pi->qsize_rxq;
5230
5231         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5232         if (rc != 0 || req->newptr == NULL)
5233                 return (rc);
5234
5235         if (qsize < 128 || (qsize & 7))
5236                 return (EINVAL);
5237
5238         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5239             "t4rxqs");
5240         if (rc)
5241                 return (rc);
5242
5243         if (pi->flags & PORT_INIT_DONE)
5244                 rc = EBUSY; /* cannot be changed once the queues are created */
5245         else
5246                 pi->qsize_rxq = qsize;
5247
5248         end_synchronized_op(sc, LOCK_HELD);
5249         return (rc);
5250 }
5251
5252 static int
5253 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
5254 {
5255         struct port_info *pi = arg1;
5256         struct adapter *sc = pi->adapter;
5257         int qsize, rc;
5258
5259         qsize = pi->qsize_txq;
5260
5261         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5262         if (rc != 0 || req->newptr == NULL)
5263                 return (rc);
5264
5265         if (qsize < 128 || qsize > 65536)
5266                 return (EINVAL);
5267
5268         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5269             "t4txqs");
5270         if (rc)
5271                 return (rc);
5272
5273         if (pi->flags & PORT_INIT_DONE)
5274                 rc = EBUSY; /* cannot be changed once the queues are created */
5275         else
5276                 pi->qsize_txq = qsize;
5277
5278         end_synchronized_op(sc, LOCK_HELD);
5279         return (rc);
5280 }
5281
5282 static int
5283 sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
5284 {
5285         struct port_info *pi = arg1;
5286         struct adapter *sc = pi->adapter;
5287         struct link_config *lc = &pi->link_cfg;
5288         int rc;
5289
5290         if (req->newptr == NULL) {
5291                 struct sbuf *sb;
5292                 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX";
5293
5294                 rc = sysctl_wire_old_buffer(req, 0);
5295                 if (rc != 0)
5296                         return(rc);
5297
5298                 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5299                 if (sb == NULL)
5300                         return (ENOMEM);
5301
5302                 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits);
5303                 rc = sbuf_finish(sb);
5304                 sbuf_delete(sb);
5305         } else {
5306                 char s[2];
5307                 int n;
5308
5309                 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX));
5310                 s[1] = 0;
5311
5312                 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
5313                 if (rc != 0)
5314                         return(rc);
5315
5316                 if (s[1] != 0)
5317                         return (EINVAL);
5318                 if (s[0] < '0' || s[0] > '9')
5319                         return (EINVAL);        /* not a number */
5320                 n = s[0] - '0';
5321                 if (n & ~(PAUSE_TX | PAUSE_RX))
5322                         return (EINVAL);        /* some other bit is set too */
5323
5324                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE");
5325                 if (rc)
5326                         return (rc);
5327                 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) {
5328                         int link_ok = lc->link_ok;
5329
5330                         lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX);
5331                         lc->requested_fc |= n;
5332                         rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc);
5333                         lc->link_ok = link_ok;  /* restore */
5334                 }
5335                 end_synchronized_op(sc, 0);
5336         }
5337
5338         return (rc);
5339 }
5340
5341 static int
5342 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
5343 {
5344         struct adapter *sc = arg1;
5345         int reg = arg2;
5346         uint64_t val;
5347
5348         val = t4_read_reg64(sc, reg);
5349
5350         return (sysctl_handle_64(oidp, &val, 0, req));
5351 }
5352
5353 static int
5354 sysctl_temperature(SYSCTL_HANDLER_ARGS)
5355 {
5356         struct adapter *sc = arg1;
5357         int rc, t;
5358         uint32_t param, val;
5359
5360         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
5361         if (rc)
5362                 return (rc);
5363         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5364             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5365             V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
5366         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
5367         end_synchronized_op(sc, 0);
5368         if (rc)
5369                 return (rc);
5370
5371         /* unknown is returned as 0 but we display -1 in that case */
5372         t = val == 0 ? -1 : val;
5373
5374         rc = sysctl_handle_int(oidp, &t, 0, req);
5375         return (rc);
5376 }
5377
5378 #ifdef SBUF_DRAIN
5379 static int
5380 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
5381 {
5382         struct adapter *sc = arg1;
5383         struct sbuf *sb;
5384         int rc, i;
5385         uint16_t incr[NMTUS][NCCTRL_WIN];
5386         static const char *dec_fac[] = {
5387                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
5388                 "0.9375"
5389         };
5390
5391         rc = sysctl_wire_old_buffer(req, 0);
5392         if (rc != 0)
5393                 return (rc);
5394
5395         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5396         if (sb == NULL)
5397                 return (ENOMEM);
5398
5399         t4_read_cong_tbl(sc, incr);
5400
5401         for (i = 0; i < NCCTRL_WIN; ++i) {
5402                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
5403                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
5404                     incr[5][i], incr[6][i], incr[7][i]);
5405                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
5406                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
5407                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
5408                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
5409         }
5410
5411         rc = sbuf_finish(sb);
5412         sbuf_delete(sb);
5413
5414         return (rc);
5415 }
5416
5417 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
5418         "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",   /* ibq's */
5419         "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
5420         "SGE0-RX", "SGE1-RX"    /* additional obq's (T5 onwards) */
5421 };
5422
5423 static int
5424 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
5425 {
5426         struct adapter *sc = arg1;
5427         struct sbuf *sb;
5428         int rc, i, n, qid = arg2;
5429         uint32_t *buf, *p;
5430         char *qtype;
5431         u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
5432
5433         KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
5434             ("%s: bad qid %d\n", __func__, qid));
5435
5436         if (qid < CIM_NUM_IBQ) {
5437                 /* inbound queue */
5438                 qtype = "IBQ";
5439                 n = 4 * CIM_IBQ_SIZE;
5440                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5441                 rc = t4_read_cim_ibq(sc, qid, buf, n);
5442         } else {
5443                 /* outbound queue */
5444                 qtype = "OBQ";
5445                 qid -= CIM_NUM_IBQ;
5446                 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
5447                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5448                 rc = t4_read_cim_obq(sc, qid, buf, n);
5449         }
5450
5451         if (rc < 0) {
5452                 rc = -rc;
5453                 goto done;
5454         }
5455         n = rc * sizeof(uint32_t);      /* rc has # of words actually read */
5456
5457         rc = sysctl_wire_old_buffer(req, 0);
5458         if (rc != 0)
5459                 goto done;
5460
5461         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5462         if (sb == NULL) {
5463                 rc = ENOMEM;
5464                 goto done;
5465         }
5466
5467         sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
5468         for (i = 0, p = buf; i < n; i += 16, p += 4)
5469                 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
5470                     p[2], p[3]);
5471
5472         rc = sbuf_finish(sb);
5473         sbuf_delete(sb);
5474 done:
5475         free(buf, M_CXGBE);
5476         return (rc);
5477 }
5478
5479 static int
5480 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
5481 {
5482         struct adapter *sc = arg1;
5483         u_int cfg;
5484         struct sbuf *sb;
5485         uint32_t *buf, *p;
5486         int rc;
5487
5488         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
5489         if (rc != 0)
5490                 return (rc);
5491
5492         rc = sysctl_wire_old_buffer(req, 0);
5493         if (rc != 0)
5494                 return (rc);
5495
5496         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5497         if (sb == NULL)
5498                 return (ENOMEM);
5499
5500         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
5501             M_ZERO | M_WAITOK);
5502
5503         rc = -t4_cim_read_la(sc, buf, NULL);
5504         if (rc != 0)
5505                 goto done;
5506
5507         sbuf_printf(sb, "Status   Data      PC%s",
5508             cfg & F_UPDBGLACAPTPCONLY ? "" :
5509             "     LS0Stat  LS0Addr             LS0Data");
5510
5511         KASSERT((sc->params.cim_la_size & 7) == 0,
5512             ("%s: p will walk off the end of buf", __func__));
5513
5514         for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
5515                 if (cfg & F_UPDBGLACAPTPCONLY) {
5516                         sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
5517                             p[6], p[7]);
5518                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
5519                             (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
5520                             p[4] & 0xff, p[5] >> 8);
5521                         sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
5522                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5523                             p[1] & 0xf, p[2] >> 4);
5524                 } else {
5525                         sbuf_printf(sb,
5526                             "\n  %02x   %x%07x %x%07x %08x %08x "
5527                             "%08x%08x%08x%08x",
5528                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5529                             p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
5530                             p[6], p[7]);
5531                 }
5532         }
5533
5534         rc = sbuf_finish(sb);
5535         sbuf_delete(sb);
5536 done:
5537         free(buf, M_CXGBE);
5538         return (rc);
5539 }
5540
5541 static int
5542 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
5543 {
5544         struct adapter *sc = arg1;
5545         u_int i;
5546         struct sbuf *sb;
5547         uint32_t *buf, *p;
5548         int rc;
5549
5550         rc = sysctl_wire_old_buffer(req, 0);
5551         if (rc != 0)
5552                 return (rc);
5553
5554         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5555         if (sb == NULL)
5556                 return (ENOMEM);
5557
5558         buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
5559             M_ZERO | M_WAITOK);
5560
5561         t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
5562         p = buf;
5563
5564         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5565                 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
5566                     p[1], p[0]);
5567         }
5568
5569         sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
5570         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5571                 sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
5572                     (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
5573                     (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
5574                     (p[1] >> 2) | ((p[2] & 3) << 30),
5575                     (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
5576                     p[0] & 1);
5577         }
5578
5579         rc = sbuf_finish(sb);
5580         sbuf_delete(sb);
5581         free(buf, M_CXGBE);
5582         return (rc);
5583 }
5584
5585 static int
5586 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
5587 {
5588         struct adapter *sc = arg1;
5589         u_int i;
5590         struct sbuf *sb;
5591         uint32_t *buf, *p;
5592         int rc;
5593
5594         rc = sysctl_wire_old_buffer(req, 0);
5595         if (rc != 0)
5596                 return (rc);
5597
5598         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5599         if (sb == NULL)
5600                 return (ENOMEM);
5601
5602         buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
5603             M_ZERO | M_WAITOK);
5604
5605         t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
5606         p = buf;
5607
5608         sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
5609         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5610                 sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
5611                     (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
5612                     p[4], p[3], p[2], p[1], p[0]);
5613         }
5614
5615         sbuf_printf(sb, "\n\nCntl ID               Data");
5616         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5617                 sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
5618                     (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
5619         }
5620
5621         rc = sbuf_finish(sb);
5622         sbuf_delete(sb);
5623         free(buf, M_CXGBE);
5624         return (rc);
5625 }
5626
5627 static int
5628 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
5629 {
5630         struct adapter *sc = arg1;
5631         struct sbuf *sb;
5632         int rc, i;
5633         uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5634         uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5635         uint16_t thres[CIM_NUM_IBQ];
5636         uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
5637         uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
5638         u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
5639
5640         if (is_t4(sc)) {
5641                 cim_num_obq = CIM_NUM_OBQ;
5642                 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
5643                 obq_rdaddr = A_UP_OBQ_0_REALADDR;
5644         } else {
5645                 cim_num_obq = CIM_NUM_OBQ_T5;
5646                 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
5647                 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
5648         }
5649         nq = CIM_NUM_IBQ + cim_num_obq;
5650
5651         rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
5652         if (rc == 0)
5653                 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
5654         if (rc != 0)
5655                 return (rc);
5656
5657         t4_read_cimq_cfg(sc, base, size, thres);
5658
5659         rc = sysctl_wire_old_buffer(req, 0);
5660         if (rc != 0)
5661                 return (rc);
5662
5663         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5664         if (sb == NULL)
5665                 return (ENOMEM);
5666
5667         sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
5668
5669         for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
5670                 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
5671                     qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
5672                     G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5673                     G_QUEREMFLITS(p[2]) * 16);
5674         for ( ; i < nq; i++, p += 4, wr += 2)
5675                 sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
5676                     base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
5677                     wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5678                     G_QUEREMFLITS(p[2]) * 16);
5679
5680         rc = sbuf_finish(sb);
5681         sbuf_delete(sb);
5682
5683         return (rc);
5684 }
5685
5686 static int
5687 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
5688 {
5689         struct adapter *sc = arg1;
5690         struct sbuf *sb;
5691         int rc;
5692         struct tp_cpl_stats stats;
5693
5694         rc = sysctl_wire_old_buffer(req, 0);
5695         if (rc != 0)
5696                 return (rc);
5697
5698         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5699         if (sb == NULL)
5700                 return (ENOMEM);
5701
5702         t4_tp_get_cpl_stats(sc, &stats);
5703
5704         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
5705             "channel 3\n");
5706         sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
5707                    stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
5708         sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
5709                    stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
5710
5711         rc = sbuf_finish(sb);
5712         sbuf_delete(sb);
5713
5714         return (rc);
5715 }
5716
5717 static int
5718 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
5719 {
5720         struct adapter *sc = arg1;
5721         struct sbuf *sb;
5722         int rc;
5723         struct tp_usm_stats stats;
5724
5725         rc = sysctl_wire_old_buffer(req, 0);
5726         if (rc != 0)
5727                 return(rc);
5728
5729         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5730         if (sb == NULL)
5731                 return (ENOMEM);
5732
5733         t4_get_usm_stats(sc, &stats);
5734
5735         sbuf_printf(sb, "Frames: %u\n", stats.frames);
5736         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
5737         sbuf_printf(sb, "Drops:  %u", stats.drops);
5738
5739         rc = sbuf_finish(sb);
5740         sbuf_delete(sb);
5741
5742         return (rc);
5743 }
5744
5745 const char *devlog_level_strings[] = {
5746         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
5747         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
5748         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
5749         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
5750         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
5751         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
5752 };
5753
5754 const char *devlog_facility_strings[] = {
5755         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
5756         [FW_DEVLOG_FACILITY_CF]         = "CF",
5757         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
5758         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
5759         [FW_DEVLOG_FACILITY_RES]        = "RES",
5760         [FW_DEVLOG_FACILITY_HW]         = "HW",
5761         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
5762         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
5763         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
5764         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
5765         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
5766         [FW_DEVLOG_FACILITY_VI]         = "VI",
5767         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
5768         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
5769         [FW_DEVLOG_FACILITY_TM]         = "TM",
5770         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
5771         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
5772         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
5773         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
5774         [FW_DEVLOG_FACILITY_RI]         = "RI",
5775         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
5776         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
5777         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
5778         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE"
5779 };
5780
5781 static int
5782 sysctl_devlog(SYSCTL_HANDLER_ARGS)
5783 {
5784         struct adapter *sc = arg1;
5785         struct devlog_params *dparams = &sc->params.devlog;
5786         struct fw_devlog_e *buf, *e;
5787         int i, j, rc, nentries, first = 0, m;
5788         struct sbuf *sb;
5789         uint64_t ftstamp = UINT64_MAX;
5790
5791         if (dparams->start == 0) {
5792                 dparams->memtype = FW_MEMTYPE_EDC0;
5793                 dparams->start = 0x84000;
5794                 dparams->size = 32768;
5795         }
5796
5797         nentries = dparams->size / sizeof(struct fw_devlog_e);
5798
5799         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
5800         if (buf == NULL)
5801                 return (ENOMEM);
5802
5803         m = fwmtype_to_hwmtype(dparams->memtype);
5804         rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
5805         if (rc != 0)
5806                 goto done;
5807
5808         for (i = 0; i < nentries; i++) {
5809                 e = &buf[i];
5810
5811                 if (e->timestamp == 0)
5812                         break;  /* end */
5813
5814                 e->timestamp = be64toh(e->timestamp);
5815                 e->seqno = be32toh(e->seqno);
5816                 for (j = 0; j < 8; j++)
5817                         e->params[j] = be32toh(e->params[j]);
5818
5819                 if (e->timestamp < ftstamp) {
5820                         ftstamp = e->timestamp;
5821                         first = i;
5822                 }
5823         }
5824
5825         if (buf[first].timestamp == 0)
5826                 goto done;      /* nothing in the log */
5827
5828         rc = sysctl_wire_old_buffer(req, 0);
5829         if (rc != 0)
5830                 goto done;
5831
5832         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5833         if (sb == NULL) {
5834                 rc = ENOMEM;
5835                 goto done;
5836         }
5837         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
5838             "Seq#", "Tstamp", "Level", "Facility", "Message");
5839
5840         i = first;
5841         do {
5842                 e = &buf[i];
5843                 if (e->timestamp == 0)
5844                         break;  /* end */
5845
5846                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
5847                     e->seqno, e->timestamp,
5848                     (e->level < nitems(devlog_level_strings) ?
5849                         devlog_level_strings[e->level] : "UNKNOWN"),
5850                     (e->facility < nitems(devlog_facility_strings) ?
5851                         devlog_facility_strings[e->facility] : "UNKNOWN"));
5852                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
5853                     e->params[2], e->params[3], e->params[4],
5854                     e->params[5], e->params[6], e->params[7]);
5855
5856                 if (++i == nentries)
5857                         i = 0;
5858         } while (i != first);
5859
5860         rc = sbuf_finish(sb);
5861         sbuf_delete(sb);
5862 done:
5863         free(buf, M_CXGBE);
5864         return (rc);
5865 }
5866
5867 static int
5868 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
5869 {
5870         struct adapter *sc = arg1;
5871         struct sbuf *sb;
5872         int rc;
5873         struct tp_fcoe_stats stats[4];
5874
5875         rc = sysctl_wire_old_buffer(req, 0);
5876         if (rc != 0)
5877                 return (rc);
5878
5879         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5880         if (sb == NULL)
5881                 return (ENOMEM);
5882
5883         t4_get_fcoe_stats(sc, 0, &stats[0]);
5884         t4_get_fcoe_stats(sc, 1, &stats[1]);
5885         t4_get_fcoe_stats(sc, 2, &stats[2]);
5886         t4_get_fcoe_stats(sc, 3, &stats[3]);
5887
5888         sbuf_printf(sb, "                   channel 0        channel 1        "
5889             "channel 2        channel 3\n");
5890         sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
5891             stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
5892             stats[3].octetsDDP);
5893         sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
5894             stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
5895         sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
5896             stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
5897             stats[3].framesDrop);
5898
5899         rc = sbuf_finish(sb);
5900         sbuf_delete(sb);
5901
5902         return (rc);
5903 }
5904
5905 static int
5906 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
5907 {
5908         struct adapter *sc = arg1;
5909         struct sbuf *sb;
5910         int rc, i;
5911         unsigned int map, kbps, ipg, mode;
5912         unsigned int pace_tab[NTX_SCHED];
5913
5914         rc = sysctl_wire_old_buffer(req, 0);
5915         if (rc != 0)
5916                 return (rc);
5917
5918         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5919         if (sb == NULL)
5920                 return (ENOMEM);
5921
5922         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
5923         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
5924         t4_read_pace_tbl(sc, pace_tab);
5925
5926         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
5927             "Class IPG (0.1 ns)   Flow IPG (us)");
5928
5929         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
5930                 t4_get_tx_sched(sc, i, &kbps, &ipg);
5931                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
5932                     (mode & (1 << i)) ? "flow" : "class", map & 3);
5933                 if (kbps)
5934                         sbuf_printf(sb, "%9u     ", kbps);
5935                 else
5936                         sbuf_printf(sb, " disabled     ");
5937
5938                 if (ipg)
5939                         sbuf_printf(sb, "%13u        ", ipg);
5940                 else
5941                         sbuf_printf(sb, "     disabled        ");
5942
5943                 if (pace_tab[i])
5944                         sbuf_printf(sb, "%10u", pace_tab[i]);
5945                 else
5946                         sbuf_printf(sb, "  disabled");
5947         }
5948
5949         rc = sbuf_finish(sb);
5950         sbuf_delete(sb);
5951
5952         return (rc);
5953 }
5954
5955 static int
5956 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
5957 {
5958         struct adapter *sc = arg1;
5959         struct sbuf *sb;
5960         int rc, i, j;
5961         uint64_t *p0, *p1;
5962         struct lb_port_stats s[2];
5963         static const char *stat_name[] = {
5964                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
5965                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
5966                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
5967                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
5968                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
5969                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
5970                 "BG2FramesTrunc:", "BG3FramesTrunc:"
5971         };
5972
5973         rc = sysctl_wire_old_buffer(req, 0);
5974         if (rc != 0)
5975                 return (rc);
5976
5977         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5978         if (sb == NULL)
5979                 return (ENOMEM);
5980
5981         memset(s, 0, sizeof(s));
5982
5983         for (i = 0; i < 4; i += 2) {
5984                 t4_get_lb_stats(sc, i, &s[0]);
5985                 t4_get_lb_stats(sc, i + 1, &s[1]);
5986
5987                 p0 = &s[0].octets;
5988                 p1 = &s[1].octets;
5989                 sbuf_printf(sb, "%s                       Loopback %u"
5990                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
5991
5992                 for (j = 0; j < nitems(stat_name); j++)
5993                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
5994                                    *p0++, *p1++);
5995         }
5996
5997         rc = sbuf_finish(sb);
5998         sbuf_delete(sb);
5999
6000         return (rc);
6001 }
6002
6003 static int
6004 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
6005 {
6006         int rc = 0;
6007         struct port_info *pi = arg1;
6008         struct sbuf *sb;
6009         static const char *linkdnreasons[] = {
6010                 "non-specific", "remote fault", "autoneg failed", "reserved3",
6011                 "PHY overheated", "unknown", "rx los", "reserved7"
6012         };
6013
6014         rc = sysctl_wire_old_buffer(req, 0);
6015         if (rc != 0)
6016                 return(rc);
6017         sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
6018         if (sb == NULL)
6019                 return (ENOMEM);
6020
6021         if (pi->linkdnrc < 0)
6022                 sbuf_printf(sb, "n/a");
6023         else if (pi->linkdnrc < nitems(linkdnreasons))
6024                 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
6025         else
6026                 sbuf_printf(sb, "%d", pi->linkdnrc);
6027
6028         rc = sbuf_finish(sb);
6029         sbuf_delete(sb);
6030
6031         return (rc);
6032 }
6033
6034 struct mem_desc {
6035         unsigned int base;
6036         unsigned int limit;
6037         unsigned int idx;
6038 };
6039
6040 static int
6041 mem_desc_cmp(const void *a, const void *b)
6042 {
6043         return ((const struct mem_desc *)a)->base -
6044                ((const struct mem_desc *)b)->base;
6045 }
6046
6047 static void
6048 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
6049     unsigned int to)
6050 {
6051         unsigned int size;
6052
6053         size = to - from + 1;
6054         if (size == 0)
6055                 return;
6056
6057         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
6058         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
6059 }
6060
6061 static int
6062 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
6063 {
6064         struct adapter *sc = arg1;
6065         struct sbuf *sb;
6066         int rc, i, n;
6067         uint32_t lo, hi, used, alloc;
6068         static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
6069         static const char *region[] = {
6070                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
6071                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
6072                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
6073                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
6074                 "RQUDP region:", "PBL region:", "TXPBL region:",
6075                 "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
6076                 "On-chip queues:"
6077         };
6078         struct mem_desc avail[4];
6079         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
6080         struct mem_desc *md = mem;
6081
6082         rc = sysctl_wire_old_buffer(req, 0);
6083         if (rc != 0)
6084                 return (rc);
6085
6086         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6087         if (sb == NULL)
6088                 return (ENOMEM);
6089
6090         for (i = 0; i < nitems(mem); i++) {
6091                 mem[i].limit = 0;
6092                 mem[i].idx = i;
6093         }
6094
6095         /* Find and sort the populated memory ranges */
6096         i = 0;
6097         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
6098         if (lo & F_EDRAM0_ENABLE) {
6099                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
6100                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
6101                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
6102                 avail[i].idx = 0;
6103                 i++;
6104         }
6105         if (lo & F_EDRAM1_ENABLE) {
6106                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
6107                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
6108                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
6109                 avail[i].idx = 1;
6110                 i++;
6111         }
6112         if (lo & F_EXT_MEM_ENABLE) {
6113                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
6114                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
6115                 avail[i].limit = avail[i].base +
6116                     (G_EXT_MEM_SIZE(hi) << 20);
6117                 avail[i].idx = is_t4(sc) ? 2 : 3;       /* Call it MC for T4 */
6118                 i++;
6119         }
6120         if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
6121                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
6122                 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
6123                 avail[i].limit = avail[i].base +
6124                     (G_EXT_MEM1_SIZE(hi) << 20);
6125                 avail[i].idx = 4;
6126                 i++;
6127         }
6128         if (!i)                                    /* no memory available */
6129                 return 0;
6130         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
6131
6132         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
6133         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
6134         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
6135         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
6136         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
6137         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
6138         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
6139         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
6140         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
6141
6142         /* the next few have explicit upper bounds */
6143         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
6144         md->limit = md->base - 1 +
6145                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
6146                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
6147         md++;
6148
6149         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
6150         md->limit = md->base - 1 +
6151                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
6152                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
6153         md++;
6154
6155         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6156                 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
6157                 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
6158                 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
6159         } else {
6160                 md->base = 0;
6161                 md->idx = nitems(region);  /* hide it */
6162         }
6163         md++;
6164
6165 #define ulp_region(reg) \
6166         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
6167         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
6168
6169         ulp_region(RX_ISCSI);
6170         ulp_region(RX_TDDP);
6171         ulp_region(TX_TPT);
6172         ulp_region(RX_STAG);
6173         ulp_region(RX_RQ);
6174         ulp_region(RX_RQUDP);
6175         ulp_region(RX_PBL);
6176         ulp_region(TX_PBL);
6177 #undef ulp_region
6178
6179         md->base = 0;
6180         md->idx = nitems(region);
6181         if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
6182                 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
6183                 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
6184                     A_SGE_DBVFIFO_SIZE))) << 2) - 1;
6185         }
6186         md++;
6187
6188         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
6189         md->limit = md->base + sc->tids.ntids - 1;
6190         md++;
6191         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
6192         md->limit = md->base + sc->tids.ntids - 1;
6193         md++;
6194
6195         md->base = sc->vres.ocq.start;
6196         if (sc->vres.ocq.size)
6197                 md->limit = md->base + sc->vres.ocq.size - 1;
6198         else
6199                 md->idx = nitems(region);  /* hide it */
6200         md++;
6201
6202         /* add any address-space holes, there can be up to 3 */
6203         for (n = 0; n < i - 1; n++)
6204                 if (avail[n].limit < avail[n + 1].base)
6205                         (md++)->base = avail[n].limit;
6206         if (avail[n].limit)
6207                 (md++)->base = avail[n].limit;
6208
6209         n = md - mem;
6210         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
6211
6212         for (lo = 0; lo < i; lo++)
6213                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
6214                                 avail[lo].limit - 1);
6215
6216         sbuf_printf(sb, "\n");
6217         for (i = 0; i < n; i++) {
6218                 if (mem[i].idx >= nitems(region))
6219                         continue;                        /* skip holes */
6220                 if (!mem[i].limit)
6221                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
6222                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
6223                                 mem[i].limit);
6224         }
6225
6226         sbuf_printf(sb, "\n");
6227         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
6228         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
6229         mem_region_show(sb, "uP RAM:", lo, hi);
6230
6231         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
6232         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
6233         mem_region_show(sb, "uP Extmem2:", lo, hi);
6234
6235         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
6236         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
6237                    G_PMRXMAXPAGE(lo),
6238                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
6239                    (lo & F_PMRXNUMCHN) ? 2 : 1);
6240
6241         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
6242         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
6243         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
6244                    G_PMTXMAXPAGE(lo),
6245                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
6246                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
6247         sbuf_printf(sb, "%u p-structs\n",
6248                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
6249
6250         for (i = 0; i < 4; i++) {
6251                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
6252                 if (is_t4(sc)) {
6253                         used = G_USED(lo);
6254                         alloc = G_ALLOC(lo);
6255                 } else {
6256                         used = G_T5_USED(lo);
6257                         alloc = G_T5_ALLOC(lo);
6258                 }
6259                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
6260                            i, used, alloc);
6261         }
6262         for (i = 0; i < 4; i++) {
6263                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
6264                 if (is_t4(sc)) {
6265                         used = G_USED(lo);
6266                         alloc = G_ALLOC(lo);
6267                 } else {
6268                         used = G_T5_USED(lo);
6269                         alloc = G_T5_ALLOC(lo);
6270                 }
6271                 sbuf_printf(sb,
6272                            "\nLoopback %d using %u pages out of %u allocated",
6273                            i, used, alloc);
6274         }
6275
6276         rc = sbuf_finish(sb);
6277         sbuf_delete(sb);
6278
6279         return (rc);
6280 }
6281
6282 static inline void
6283 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
6284 {
6285         *mask = x | y;
6286         y = htobe64(y);
6287         memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
6288 }
6289
6290 static int
6291 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
6292 {
6293         struct adapter *sc = arg1;
6294         struct sbuf *sb;
6295         int rc, i, n;
6296
6297         rc = sysctl_wire_old_buffer(req, 0);
6298         if (rc != 0)
6299                 return (rc);
6300
6301         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6302         if (sb == NULL)
6303                 return (ENOMEM);
6304
6305         sbuf_printf(sb,
6306             "Idx  Ethernet address     Mask     Vld Ports PF"
6307             "  VF              Replication             P0 P1 P2 P3  ML");
6308         n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
6309             NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
6310         for (i = 0; i < n; i++) {
6311                 uint64_t tcamx, tcamy, mask;
6312                 uint32_t cls_lo, cls_hi;
6313                 uint8_t addr[ETHER_ADDR_LEN];
6314
6315                 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
6316                 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
6317                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
6318                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
6319
6320                 if (tcamx & tcamy)
6321                         continue;
6322
6323                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
6324                 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
6325                            "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
6326                            addr[3], addr[4], addr[5], (uintmax_t)mask,
6327                            (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
6328                            G_PORTMAP(cls_hi), G_PF(cls_lo),
6329                            (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
6330
6331                 if (cls_lo & F_REPLICATE) {
6332                         struct fw_ldst_cmd ldst_cmd;
6333
6334                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
6335                         ldst_cmd.op_to_addrspace =
6336                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
6337                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
6338                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
6339                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
6340                         ldst_cmd.u.mps.fid_ctl =
6341                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
6342                                 V_FW_LDST_CMD_CTL(i));
6343
6344                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
6345                             "t4mps");
6346                         if (rc)
6347                                 break;
6348                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
6349                             sizeof(ldst_cmd), &ldst_cmd);
6350                         end_synchronized_op(sc, 0);
6351
6352                         if (rc != 0) {
6353                                 sbuf_printf(sb,
6354                                     " ------------ error %3u ------------", rc);
6355                                 rc = 0;
6356                         } else {
6357                                 sbuf_printf(sb, " %08x %08x %08x %08x",
6358                                     be32toh(ldst_cmd.u.mps.rplc127_96),
6359                                     be32toh(ldst_cmd.u.mps.rplc95_64),
6360                                     be32toh(ldst_cmd.u.mps.rplc63_32),
6361                                     be32toh(ldst_cmd.u.mps.rplc31_0));
6362                         }
6363                 } else
6364                         sbuf_printf(sb, "%36s", "");
6365
6366                 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
6367                     G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
6368                     G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
6369         }
6370
6371         if (rc)
6372                 (void) sbuf_finish(sb);
6373         else
6374                 rc = sbuf_finish(sb);
6375         sbuf_delete(sb);
6376
6377         return (rc);
6378 }
6379
6380 static int
6381 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
6382 {
6383         struct adapter *sc = arg1;
6384         struct sbuf *sb;
6385         int rc;
6386         uint16_t mtus[NMTUS];
6387
6388         rc = sysctl_wire_old_buffer(req, 0);
6389         if (rc != 0)
6390                 return (rc);
6391
6392         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6393         if (sb == NULL)
6394                 return (ENOMEM);
6395
6396         t4_read_mtu_tbl(sc, mtus, NULL);
6397
6398         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
6399             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
6400             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
6401             mtus[14], mtus[15]);
6402
6403         rc = sbuf_finish(sb);
6404         sbuf_delete(sb);
6405
6406         return (rc);
6407 }
6408
6409 static int
6410 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
6411 {
6412         struct adapter *sc = arg1;
6413         struct sbuf *sb;
6414         int rc, i;
6415         uint32_t cnt[PM_NSTATS];
6416         uint64_t cyc[PM_NSTATS];
6417         static const char *rx_stats[] = {
6418                 "Read:", "Write bypass:", "Write mem:", "Flush:"
6419         };
6420         static const char *tx_stats[] = {
6421                 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
6422         };
6423
6424         rc = sysctl_wire_old_buffer(req, 0);
6425         if (rc != 0)
6426                 return (rc);
6427
6428         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6429         if (sb == NULL)
6430                 return (ENOMEM);
6431
6432         t4_pmtx_get_stats(sc, cnt, cyc);
6433         sbuf_printf(sb, "                Tx pcmds             Tx bytes");
6434         for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
6435                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
6436                     cyc[i]);
6437
6438         t4_pmrx_get_stats(sc, cnt, cyc);
6439         sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
6440         for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
6441                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
6442                     cyc[i]);
6443
6444         rc = sbuf_finish(sb);
6445         sbuf_delete(sb);
6446
6447         return (rc);
6448 }
6449
6450 static int
6451 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
6452 {
6453         struct adapter *sc = arg1;
6454         struct sbuf *sb;
6455         int rc;
6456         struct tp_rdma_stats stats;
6457
6458         rc = sysctl_wire_old_buffer(req, 0);
6459         if (rc != 0)
6460                 return (rc);
6461
6462         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6463         if (sb == NULL)
6464                 return (ENOMEM);
6465
6466         t4_tp_get_rdma_stats(sc, &stats);
6467         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
6468         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
6469
6470         rc = sbuf_finish(sb);
6471         sbuf_delete(sb);
6472
6473         return (rc);
6474 }
6475
6476 static int
6477 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
6478 {
6479         struct adapter *sc = arg1;
6480         struct sbuf *sb;
6481         int rc;
6482         struct tp_tcp_stats v4, v6;
6483
6484         rc = sysctl_wire_old_buffer(req, 0);
6485         if (rc != 0)
6486                 return (rc);
6487
6488         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6489         if (sb == NULL)
6490                 return (ENOMEM);
6491
6492         t4_tp_get_tcp_stats(sc, &v4, &v6);
6493         sbuf_printf(sb,
6494             "                                IP                 IPv6\n");
6495         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
6496             v4.tcpOutRsts, v6.tcpOutRsts);
6497         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
6498             v4.tcpInSegs, v6.tcpInSegs);
6499         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
6500             v4.tcpOutSegs, v6.tcpOutSegs);
6501         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
6502             v4.tcpRetransSegs, v6.tcpRetransSegs);
6503
6504         rc = sbuf_finish(sb);
6505         sbuf_delete(sb);
6506
6507         return (rc);
6508 }
6509
6510 static int
6511 sysctl_tids(SYSCTL_HANDLER_ARGS)
6512 {
6513         struct adapter *sc = arg1;
6514         struct sbuf *sb;
6515         int rc;
6516         struct tid_info *t = &sc->tids;
6517
6518         rc = sysctl_wire_old_buffer(req, 0);
6519         if (rc != 0)
6520                 return (rc);
6521
6522         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6523         if (sb == NULL)
6524                 return (ENOMEM);
6525
6526         if (t->natids) {
6527                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
6528                     t->atids_in_use);
6529         }
6530
6531         if (t->ntids) {
6532                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6533                         uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
6534
6535                         if (b) {
6536                                 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
6537                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6538                                     t->ntids - 1);
6539                         } else {
6540                                 sbuf_printf(sb, "TID range: %u-%u",
6541                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6542                                     t->ntids - 1);
6543                         }
6544                 } else
6545                         sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
6546                 sbuf_printf(sb, ", in use: %u\n",
6547                     atomic_load_acq_int(&t->tids_in_use));
6548         }
6549
6550         if (t->nstids) {
6551                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
6552                     t->stid_base + t->nstids - 1, t->stids_in_use);
6553         }
6554
6555         if (t->nftids) {
6556                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
6557                     t->ftid_base + t->nftids - 1);
6558         }
6559
6560         if (t->netids) {
6561                 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
6562                     t->etid_base + t->netids - 1);
6563         }
6564
6565         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
6566             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
6567             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
6568
6569         rc = sbuf_finish(sb);
6570         sbuf_delete(sb);
6571
6572         return (rc);
6573 }
6574
6575 static int
6576 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
6577 {
6578         struct adapter *sc = arg1;
6579         struct sbuf *sb;
6580         int rc;
6581         struct tp_err_stats stats;
6582
6583         rc = sysctl_wire_old_buffer(req, 0);
6584         if (rc != 0)
6585                 return (rc);
6586
6587         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6588         if (sb == NULL)
6589                 return (ENOMEM);
6590
6591         t4_tp_get_err_stats(sc, &stats);
6592
6593         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
6594                       "channel 3\n");
6595         sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
6596             stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
6597             stats.macInErrs[3]);
6598         sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
6599             stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
6600             stats.hdrInErrs[3]);
6601         sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
6602             stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
6603             stats.tcpInErrs[3]);
6604         sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
6605             stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
6606             stats.tcp6InErrs[3]);
6607         sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
6608             stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
6609             stats.tnlCongDrops[3]);
6610         sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
6611             stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
6612             stats.tnlTxDrops[3]);
6613         sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
6614             stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
6615             stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
6616         sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
6617             stats.ofldChanDrops[0], stats.ofldChanDrops[1],
6618             stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
6619         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
6620             stats.ofldNoNeigh, stats.ofldCongDefer);
6621
6622         rc = sbuf_finish(sb);
6623         sbuf_delete(sb);
6624
6625         return (rc);
6626 }
6627
6628 struct field_desc {
6629         const char *name;
6630         u_int start;
6631         u_int width;
6632 };
6633
6634 static void
6635 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
6636 {
6637         char buf[32];
6638         int line_size = 0;
6639
6640         while (f->name) {
6641                 uint64_t mask = (1ULL << f->width) - 1;
6642                 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
6643                     ((uintmax_t)v >> f->start) & mask);
6644
6645                 if (line_size + len >= 79) {
6646                         line_size = 8;
6647                         sbuf_printf(sb, "\n        ");
6648                 }
6649                 sbuf_printf(sb, "%s ", buf);
6650                 line_size += len + 1;
6651                 f++;
6652         }
6653         sbuf_printf(sb, "\n");
6654 }
6655
6656 static struct field_desc tp_la0[] = {
6657         { "RcfOpCodeOut", 60, 4 },
6658         { "State", 56, 4 },
6659         { "WcfState", 52, 4 },
6660         { "RcfOpcSrcOut", 50, 2 },
6661         { "CRxError", 49, 1 },
6662         { "ERxError", 48, 1 },
6663         { "SanityFailed", 47, 1 },
6664         { "SpuriousMsg", 46, 1 },
6665         { "FlushInputMsg", 45, 1 },
6666         { "FlushInputCpl", 44, 1 },
6667         { "RssUpBit", 43, 1 },
6668         { "RssFilterHit", 42, 1 },
6669         { "Tid", 32, 10 },
6670         { "InitTcb", 31, 1 },
6671         { "LineNumber", 24, 7 },
6672         { "Emsg", 23, 1 },
6673         { "EdataOut", 22, 1 },
6674         { "Cmsg", 21, 1 },
6675         { "CdataOut", 20, 1 },
6676         { "EreadPdu", 19, 1 },
6677         { "CreadPdu", 18, 1 },
6678         { "TunnelPkt", 17, 1 },
6679         { "RcfPeerFin", 16, 1 },
6680         { "RcfReasonOut", 12, 4 },
6681         { "TxCchannel", 10, 2 },
6682         { "RcfTxChannel", 8, 2 },
6683         { "RxEchannel", 6, 2 },
6684         { "RcfRxChannel", 5, 1 },
6685         { "RcfDataOutSrdy", 4, 1 },
6686         { "RxDvld", 3, 1 },
6687         { "RxOoDvld", 2, 1 },
6688         { "RxCongestion", 1, 1 },
6689         { "TxCongestion", 0, 1 },
6690         { NULL }
6691 };
6692
6693 static struct field_desc tp_la1[] = {
6694         { "CplCmdIn", 56, 8 },
6695         { "CplCmdOut", 48, 8 },
6696         { "ESynOut", 47, 1 },
6697         { "EAckOut", 46, 1 },
6698         { "EFinOut", 45, 1 },
6699         { "ERstOut", 44, 1 },
6700         { "SynIn", 43, 1 },
6701         { "AckIn", 42, 1 },
6702         { "FinIn", 41, 1 },
6703         { "RstIn", 40, 1 },
6704         { "DataIn", 39, 1 },
6705         { "DataInVld", 38, 1 },
6706         { "PadIn", 37, 1 },
6707         { "RxBufEmpty", 36, 1 },
6708         { "RxDdp", 35, 1 },
6709         { "RxFbCongestion", 34, 1 },
6710         { "TxFbCongestion", 33, 1 },
6711         { "TxPktSumSrdy", 32, 1 },
6712         { "RcfUlpType", 28, 4 },
6713         { "Eread", 27, 1 },
6714         { "Ebypass", 26, 1 },
6715         { "Esave", 25, 1 },
6716         { "Static0", 24, 1 },
6717         { "Cread", 23, 1 },
6718         { "Cbypass", 22, 1 },
6719         { "Csave", 21, 1 },
6720         { "CPktOut", 20, 1 },
6721         { "RxPagePoolFull", 18, 2 },
6722         { "RxLpbkPkt", 17, 1 },
6723         { "TxLpbkPkt", 16, 1 },
6724         { "RxVfValid", 15, 1 },
6725         { "SynLearned", 14, 1 },
6726         { "SetDelEntry", 13, 1 },
6727         { "SetInvEntry", 12, 1 },
6728         { "CpcmdDvld", 11, 1 },
6729         { "CpcmdSave", 10, 1 },
6730         { "RxPstructsFull", 8, 2 },
6731         { "EpcmdDvld", 7, 1 },
6732         { "EpcmdFlush", 6, 1 },
6733         { "EpcmdTrimPrefix", 5, 1 },
6734         { "EpcmdTrimPostfix", 4, 1 },
6735         { "ERssIp4Pkt", 3, 1 },
6736         { "ERssIp6Pkt", 2, 1 },
6737         { "ERssTcpUdpPkt", 1, 1 },
6738         { "ERssFceFipPkt", 0, 1 },
6739         { NULL }
6740 };
6741
6742 static struct field_desc tp_la2[] = {
6743         { "CplCmdIn", 56, 8 },
6744         { "MpsVfVld", 55, 1 },
6745         { "MpsPf", 52, 3 },
6746         { "MpsVf", 44, 8 },
6747         { "SynIn", 43, 1 },
6748         { "AckIn", 42, 1 },
6749         { "FinIn", 41, 1 },
6750         { "RstIn", 40, 1 },
6751         { "DataIn", 39, 1 },
6752         { "DataInVld", 38, 1 },
6753         { "PadIn", 37, 1 },
6754         { "RxBufEmpty", 36, 1 },
6755         { "RxDdp", 35, 1 },
6756         { "RxFbCongestion", 34, 1 },
6757         { "TxFbCongestion", 33, 1 },
6758         { "TxPktSumSrdy", 32, 1 },
6759         { "RcfUlpType", 28, 4 },
6760         { "Eread", 27, 1 },
6761         { "Ebypass", 26, 1 },
6762         { "Esave", 25, 1 },
6763         { "Static0", 24, 1 },
6764         { "Cread", 23, 1 },
6765         { "Cbypass", 22, 1 },
6766         { "Csave", 21, 1 },
6767         { "CPktOut", 20, 1 },
6768         { "RxPagePoolFull", 18, 2 },
6769         { "RxLpbkPkt", 17, 1 },
6770         { "TxLpbkPkt", 16, 1 },
6771         { "RxVfValid", 15, 1 },
6772         { "SynLearned", 14, 1 },
6773         { "SetDelEntry", 13, 1 },
6774         { "SetInvEntry", 12, 1 },
6775         { "CpcmdDvld", 11, 1 },
6776         { "CpcmdSave", 10, 1 },
6777         { "RxPstructsFull", 8, 2 },
6778         { "EpcmdDvld", 7, 1 },
6779         { "EpcmdFlush", 6, 1 },
6780         { "EpcmdTrimPrefix", 5, 1 },
6781         { "EpcmdTrimPostfix", 4, 1 },
6782         { "ERssIp4Pkt", 3, 1 },
6783         { "ERssIp6Pkt", 2, 1 },
6784         { "ERssTcpUdpPkt", 1, 1 },
6785         { "ERssFceFipPkt", 0, 1 },
6786         { NULL }
6787 };
6788
6789 static void
6790 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
6791 {
6792
6793         field_desc_show(sb, *p, tp_la0);
6794 }
6795
6796 static void
6797 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
6798 {
6799
6800         if (idx)
6801                 sbuf_printf(sb, "\n");
6802         field_desc_show(sb, p[0], tp_la0);
6803         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6804                 field_desc_show(sb, p[1], tp_la0);
6805 }
6806
6807 static void
6808 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
6809 {
6810
6811         if (idx)
6812                 sbuf_printf(sb, "\n");
6813         field_desc_show(sb, p[0], tp_la0);
6814         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6815                 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
6816 }
6817
6818 static int
6819 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
6820 {
6821         struct adapter *sc = arg1;
6822         struct sbuf *sb;
6823         uint64_t *buf, *p;
6824         int rc;
6825         u_int i, inc;
6826         void (*show_func)(struct sbuf *, uint64_t *, int);
6827
6828         rc = sysctl_wire_old_buffer(req, 0);
6829         if (rc != 0)
6830                 return (rc);
6831
6832         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6833         if (sb == NULL)
6834                 return (ENOMEM);
6835
6836         buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
6837
6838         t4_tp_read_la(sc, buf, NULL);
6839         p = buf;
6840
6841         switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
6842         case 2:
6843                 inc = 2;
6844                 show_func = tp_la_show2;
6845                 break;
6846         case 3:
6847                 inc = 2;
6848                 show_func = tp_la_show3;
6849                 break;
6850         default:
6851                 inc = 1;
6852                 show_func = tp_la_show;
6853         }
6854
6855         for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
6856                 (*show_func)(sb, p, i);
6857
6858         rc = sbuf_finish(sb);
6859         sbuf_delete(sb);
6860         free(buf, M_CXGBE);
6861         return (rc);
6862 }
6863
6864 static int
6865 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
6866 {
6867         struct adapter *sc = arg1;
6868         struct sbuf *sb;
6869         int rc;
6870         u64 nrate[NCHAN], orate[NCHAN];
6871
6872         rc = sysctl_wire_old_buffer(req, 0);
6873         if (rc != 0)
6874                 return (rc);
6875
6876         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6877         if (sb == NULL)
6878                 return (ENOMEM);
6879
6880         t4_get_chan_txrate(sc, nrate, orate);
6881         sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
6882                  "channel 3\n");
6883         sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
6884             nrate[0], nrate[1], nrate[2], nrate[3]);
6885         sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
6886             orate[0], orate[1], orate[2], orate[3]);
6887
6888         rc = sbuf_finish(sb);
6889         sbuf_delete(sb);
6890
6891         return (rc);
6892 }
6893
6894 static int
6895 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
6896 {
6897         struct adapter *sc = arg1;
6898         struct sbuf *sb;
6899         uint32_t *buf, *p;
6900         int rc, i;
6901
6902         rc = sysctl_wire_old_buffer(req, 0);
6903         if (rc != 0)
6904                 return (rc);
6905
6906         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6907         if (sb == NULL)
6908                 return (ENOMEM);
6909
6910         buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
6911             M_ZERO | M_WAITOK);
6912
6913         t4_ulprx_read_la(sc, buf);
6914         p = buf;
6915
6916         sbuf_printf(sb, "      Pcmd        Type   Message"
6917             "                Data");
6918         for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
6919                 sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
6920                     p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
6921         }
6922
6923         rc = sbuf_finish(sb);
6924         sbuf_delete(sb);
6925         free(buf, M_CXGBE);
6926         return (rc);
6927 }
6928
6929 static int
6930 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
6931 {
6932         struct adapter *sc = arg1;
6933         struct sbuf *sb;
6934         int rc, v;
6935
6936         rc = sysctl_wire_old_buffer(req, 0);
6937         if (rc != 0)
6938                 return (rc);
6939
6940         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6941         if (sb == NULL)
6942                 return (ENOMEM);
6943
6944         v = t4_read_reg(sc, A_SGE_STAT_CFG);
6945         if (G_STATSOURCE_T5(v) == 7) {
6946                 if (G_STATMODE(v) == 0) {
6947                         sbuf_printf(sb, "total %d, incomplete %d",
6948                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6949                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6950                 } else if (G_STATMODE(v) == 1) {
6951                         sbuf_printf(sb, "total %d, data overflow %d",
6952                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6953                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6954                 }
6955         }
6956         rc = sbuf_finish(sb);
6957         sbuf_delete(sb);
6958
6959         return (rc);
6960 }
6961 #endif
6962
6963 static uint32_t
6964 fconf_to_mode(uint32_t fconf)
6965 {
6966         uint32_t mode;
6967
6968         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
6969             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
6970
6971         if (fconf & F_FRAGMENTATION)
6972                 mode |= T4_FILTER_IP_FRAGMENT;
6973
6974         if (fconf & F_MPSHITTYPE)
6975                 mode |= T4_FILTER_MPS_HIT_TYPE;
6976
6977         if (fconf & F_MACMATCH)
6978                 mode |= T4_FILTER_MAC_IDX;
6979
6980         if (fconf & F_ETHERTYPE)
6981                 mode |= T4_FILTER_ETH_TYPE;
6982
6983         if (fconf & F_PROTOCOL)
6984                 mode |= T4_FILTER_IP_PROTO;
6985
6986         if (fconf & F_TOS)
6987                 mode |= T4_FILTER_IP_TOS;
6988
6989         if (fconf & F_VLAN)
6990                 mode |= T4_FILTER_VLAN;
6991
6992         if (fconf & F_VNIC_ID)
6993                 mode |= T4_FILTER_VNIC;
6994
6995         if (fconf & F_PORT)
6996                 mode |= T4_FILTER_PORT;
6997
6998         if (fconf & F_FCOE)
6999                 mode |= T4_FILTER_FCoE;
7000
7001         return (mode);
7002 }
7003
7004 static uint32_t
7005 mode_to_fconf(uint32_t mode)
7006 {
7007         uint32_t fconf = 0;
7008
7009         if (mode & T4_FILTER_IP_FRAGMENT)
7010                 fconf |= F_FRAGMENTATION;
7011
7012         if (mode & T4_FILTER_MPS_HIT_TYPE)
7013                 fconf |= F_MPSHITTYPE;
7014
7015         if (mode & T4_FILTER_MAC_IDX)
7016                 fconf |= F_MACMATCH;
7017
7018         if (mode & T4_FILTER_ETH_TYPE)
7019                 fconf |= F_ETHERTYPE;
7020
7021         if (mode & T4_FILTER_IP_PROTO)
7022                 fconf |= F_PROTOCOL;
7023
7024         if (mode & T4_FILTER_IP_TOS)
7025                 fconf |= F_TOS;
7026
7027         if (mode & T4_FILTER_VLAN)
7028                 fconf |= F_VLAN;
7029
7030         if (mode & T4_FILTER_VNIC)
7031                 fconf |= F_VNIC_ID;
7032
7033         if (mode & T4_FILTER_PORT)
7034                 fconf |= F_PORT;
7035
7036         if (mode & T4_FILTER_FCoE)
7037                 fconf |= F_FCOE;
7038
7039         return (fconf);
7040 }
7041
7042 static uint32_t
7043 fspec_to_fconf(struct t4_filter_specification *fs)
7044 {
7045         uint32_t fconf = 0;
7046
7047         if (fs->val.frag || fs->mask.frag)
7048                 fconf |= F_FRAGMENTATION;
7049
7050         if (fs->val.matchtype || fs->mask.matchtype)
7051                 fconf |= F_MPSHITTYPE;
7052
7053         if (fs->val.macidx || fs->mask.macidx)
7054                 fconf |= F_MACMATCH;
7055
7056         if (fs->val.ethtype || fs->mask.ethtype)
7057                 fconf |= F_ETHERTYPE;
7058
7059         if (fs->val.proto || fs->mask.proto)
7060                 fconf |= F_PROTOCOL;
7061
7062         if (fs->val.tos || fs->mask.tos)
7063                 fconf |= F_TOS;
7064
7065         if (fs->val.vlan_vld || fs->mask.vlan_vld)
7066                 fconf |= F_VLAN;
7067
7068         if (fs->val.vnic_vld || fs->mask.vnic_vld)
7069                 fconf |= F_VNIC_ID;
7070
7071         if (fs->val.iport || fs->mask.iport)
7072                 fconf |= F_PORT;
7073
7074         if (fs->val.fcoe || fs->mask.fcoe)
7075                 fconf |= F_FCOE;
7076
7077         return (fconf);
7078 }
7079
7080 static int
7081 get_filter_mode(struct adapter *sc, uint32_t *mode)
7082 {
7083         int rc;
7084         uint32_t fconf;
7085
7086         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7087             "t4getfm");
7088         if (rc)
7089                 return (rc);
7090
7091         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
7092             A_TP_VLAN_PRI_MAP);
7093
7094         if (sc->params.tp.vlan_pri_map != fconf) {
7095                 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
7096                     device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
7097                     fconf);
7098                 sc->params.tp.vlan_pri_map = fconf;
7099         }
7100
7101         *mode = fconf_to_mode(sc->params.tp.vlan_pri_map);
7102
7103         end_synchronized_op(sc, LOCK_HELD);
7104         return (0);
7105 }
7106
7107 static int
7108 set_filter_mode(struct adapter *sc, uint32_t mode)
7109 {
7110         uint32_t fconf;
7111         int rc;
7112
7113         fconf = mode_to_fconf(mode);
7114
7115         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7116             "t4setfm");
7117         if (rc)
7118                 return (rc);
7119
7120         if (sc->tids.ftids_in_use > 0) {
7121                 rc = EBUSY;
7122                 goto done;
7123         }
7124
7125 #ifdef TCP_OFFLOAD
7126         if (sc->offload_map) {
7127                 rc = EBUSY;
7128                 goto done;
7129         }
7130 #endif
7131
7132 #ifdef notyet
7133         rc = -t4_set_filter_mode(sc, fconf);
7134         if (rc == 0)
7135                 sc->filter_mode = fconf;
7136 #else
7137         rc = ENOTSUP;
7138 #endif
7139
7140 done:
7141         end_synchronized_op(sc, LOCK_HELD);
7142         return (rc);
7143 }
7144
7145 static inline uint64_t
7146 get_filter_hits(struct adapter *sc, uint32_t fid)
7147 {
7148         uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
7149         uint64_t hits;
7150
7151         memwin_info(sc, 0, &mw_base, NULL);
7152         off = position_memwin(sc, 0,
7153             tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
7154         if (is_t4(sc)) {
7155                 hits = t4_read_reg64(sc, mw_base + off + 16);
7156                 hits = be64toh(hits);
7157         } else {
7158                 hits = t4_read_reg(sc, mw_base + off + 24);
7159                 hits = be32toh(hits);
7160         }
7161
7162         return (hits);
7163 }
7164
7165 static int
7166 get_filter(struct adapter *sc, struct t4_filter *t)
7167 {
7168         int i, rc, nfilters = sc->tids.nftids;
7169         struct filter_entry *f;
7170
7171         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7172             "t4getf");
7173         if (rc)
7174                 return (rc);
7175
7176         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
7177             t->idx >= nfilters) {
7178                 t->idx = 0xffffffff;
7179                 goto done;
7180         }
7181
7182         f = &sc->tids.ftid_tab[t->idx];
7183         for (i = t->idx; i < nfilters; i++, f++) {
7184                 if (f->valid) {
7185                         t->idx = i;
7186                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
7187                         t->smtidx = f->smtidx;
7188                         if (f->fs.hitcnts)
7189                                 t->hits = get_filter_hits(sc, t->idx);
7190                         else
7191                                 t->hits = UINT64_MAX;
7192                         t->fs = f->fs;
7193
7194                         goto done;
7195                 }
7196         }
7197
7198         t->idx = 0xffffffff;
7199 done:
7200         end_synchronized_op(sc, LOCK_HELD);
7201         return (0);
7202 }
7203
7204 static int
7205 set_filter(struct adapter *sc, struct t4_filter *t)
7206 {
7207         unsigned int nfilters, nports;
7208         struct filter_entry *f;
7209         int i, rc;
7210
7211         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
7212         if (rc)
7213                 return (rc);
7214
7215         nfilters = sc->tids.nftids;
7216         nports = sc->params.nports;
7217
7218         if (nfilters == 0) {
7219                 rc = ENOTSUP;
7220                 goto done;
7221         }
7222
7223         if (!(sc->flags & FULL_INIT_DONE)) {
7224                 rc = EAGAIN;
7225                 goto done;
7226         }
7227
7228         if (t->idx >= nfilters) {
7229                 rc = EINVAL;
7230                 goto done;
7231         }
7232
7233         /* Validate against the global filter mode */
7234         if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
7235             sc->params.tp.vlan_pri_map) {
7236                 rc = E2BIG;
7237                 goto done;
7238         }
7239
7240         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
7241                 rc = EINVAL;
7242                 goto done;
7243         }
7244
7245         if (t->fs.val.iport >= nports) {
7246                 rc = EINVAL;
7247                 goto done;
7248         }
7249
7250         /* Can't specify an iq if not steering to it */
7251         if (!t->fs.dirsteer && t->fs.iq) {
7252                 rc = EINVAL;
7253                 goto done;
7254         }
7255
7256         /* IPv6 filter idx must be 4 aligned */
7257         if (t->fs.type == 1 &&
7258             ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
7259                 rc = EINVAL;
7260                 goto done;
7261         }
7262
7263         if (sc->tids.ftid_tab == NULL) {
7264                 KASSERT(sc->tids.ftids_in_use == 0,
7265                     ("%s: no memory allocated but filters_in_use > 0",
7266                     __func__));
7267
7268                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
7269                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
7270                 if (sc->tids.ftid_tab == NULL) {
7271                         rc = ENOMEM;
7272                         goto done;
7273                 }
7274                 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
7275         }
7276
7277         for (i = 0; i < 4; i++) {
7278                 f = &sc->tids.ftid_tab[t->idx + i];
7279
7280                 if (f->pending || f->valid) {
7281                         rc = EBUSY;
7282                         goto done;
7283                 }
7284                 if (f->locked) {
7285                         rc = EPERM;
7286                         goto done;
7287                 }
7288
7289                 if (t->fs.type == 0)
7290                         break;
7291         }
7292
7293         f = &sc->tids.ftid_tab[t->idx];
7294         f->fs = t->fs;
7295
7296         rc = set_filter_wr(sc, t->idx);
7297 done:
7298         end_synchronized_op(sc, 0);
7299
7300         if (rc == 0) {
7301                 mtx_lock(&sc->tids.ftid_lock);
7302                 for (;;) {
7303                         if (f->pending == 0) {
7304                                 rc = f->valid ? 0 : EIO;
7305                                 break;
7306                         }
7307
7308                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7309                             PCATCH, "t4setfw", 0)) {
7310                                 rc = EINPROGRESS;
7311                                 break;
7312                         }
7313                 }
7314                 mtx_unlock(&sc->tids.ftid_lock);
7315         }
7316         return (rc);
7317 }
7318
7319 static int
7320 del_filter(struct adapter *sc, struct t4_filter *t)
7321 {
7322         unsigned int nfilters;
7323         struct filter_entry *f;
7324         int rc;
7325
7326         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
7327         if (rc)
7328                 return (rc);
7329
7330         nfilters = sc->tids.nftids;
7331
7332         if (nfilters == 0) {
7333                 rc = ENOTSUP;
7334                 goto done;
7335         }
7336
7337         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
7338             t->idx >= nfilters) {
7339                 rc = EINVAL;
7340                 goto done;
7341         }
7342
7343         if (!(sc->flags & FULL_INIT_DONE)) {
7344                 rc = EAGAIN;
7345                 goto done;
7346         }
7347
7348         f = &sc->tids.ftid_tab[t->idx];
7349
7350         if (f->pending) {
7351                 rc = EBUSY;
7352                 goto done;
7353         }
7354         if (f->locked) {
7355                 rc = EPERM;
7356                 goto done;
7357         }
7358
7359         if (f->valid) {
7360                 t->fs = f->fs;  /* extra info for the caller */
7361                 rc = del_filter_wr(sc, t->idx);
7362         }
7363
7364 done:
7365         end_synchronized_op(sc, 0);
7366
7367         if (rc == 0) {
7368                 mtx_lock(&sc->tids.ftid_lock);
7369                 for (;;) {
7370                         if (f->pending == 0) {
7371                                 rc = f->valid ? EIO : 0;
7372                                 break;
7373                         }
7374
7375                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7376                             PCATCH, "t4delfw", 0)) {
7377                                 rc = EINPROGRESS;
7378                                 break;
7379                         }
7380                 }
7381                 mtx_unlock(&sc->tids.ftid_lock);
7382         }
7383
7384         return (rc);
7385 }
7386
7387 static void
7388 clear_filter(struct filter_entry *f)
7389 {
7390         if (f->l2t)
7391                 t4_l2t_release(f->l2t);
7392
7393         bzero(f, sizeof (*f));
7394 }
7395
7396 static int
7397 set_filter_wr(struct adapter *sc, int fidx)
7398 {
7399         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7400         struct fw_filter_wr *fwr;
7401         unsigned int ftid;
7402         struct wrq_cookie cookie;
7403
7404         ASSERT_SYNCHRONIZED_OP(sc);
7405
7406         if (f->fs.newdmac || f->fs.newvlan) {
7407                 /* This filter needs an L2T entry; allocate one. */
7408                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
7409                 if (f->l2t == NULL)
7410                         return (EAGAIN);
7411                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
7412                     f->fs.dmac)) {
7413                         t4_l2t_release(f->l2t);
7414                         f->l2t = NULL;
7415                         return (ENOMEM);
7416                 }
7417         }
7418
7419         ftid = sc->tids.ftid_base + fidx;
7420
7421         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
7422         if (fwr == NULL)
7423                 return (ENOMEM);
7424         bzero(fwr, sizeof(*fwr));
7425
7426         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
7427         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
7428         fwr->tid_to_iq =
7429             htobe32(V_FW_FILTER_WR_TID(ftid) |
7430                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
7431                 V_FW_FILTER_WR_NOREPLY(0) |
7432                 V_FW_FILTER_WR_IQ(f->fs.iq));
7433         fwr->del_filter_to_l2tix =
7434             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
7435                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
7436                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
7437                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
7438                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
7439                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
7440                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
7441                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
7442                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
7443                     f->fs.newvlan == VLAN_REWRITE) |
7444                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
7445                     f->fs.newvlan == VLAN_REWRITE) |
7446                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
7447                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
7448                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
7449                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
7450         fwr->ethtype = htobe16(f->fs.val.ethtype);
7451         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
7452         fwr->frag_to_ovlan_vldm =
7453             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
7454                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
7455                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
7456                 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
7457                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
7458                 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
7459         fwr->smac_sel = 0;
7460         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
7461             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
7462         fwr->maci_to_matchtypem =
7463             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
7464                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
7465                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
7466                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
7467                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
7468                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
7469                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
7470                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
7471         fwr->ptcl = f->fs.val.proto;
7472         fwr->ptclm = f->fs.mask.proto;
7473         fwr->ttyp = f->fs.val.tos;
7474         fwr->ttypm = f->fs.mask.tos;
7475         fwr->ivlan = htobe16(f->fs.val.vlan);
7476         fwr->ivlanm = htobe16(f->fs.mask.vlan);
7477         fwr->ovlan = htobe16(f->fs.val.vnic);
7478         fwr->ovlanm = htobe16(f->fs.mask.vnic);
7479         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
7480         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
7481         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
7482         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
7483         fwr->lp = htobe16(f->fs.val.dport);
7484         fwr->lpm = htobe16(f->fs.mask.dport);
7485         fwr->fp = htobe16(f->fs.val.sport);
7486         fwr->fpm = htobe16(f->fs.mask.sport);
7487         if (f->fs.newsmac)
7488                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
7489
7490         f->pending = 1;
7491         sc->tids.ftids_in_use++;
7492
7493         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
7494         return (0);
7495 }
7496
7497 static int
7498 del_filter_wr(struct adapter *sc, int fidx)
7499 {
7500         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7501         struct fw_filter_wr *fwr;
7502         unsigned int ftid;
7503         struct wrq_cookie cookie;
7504
7505         ftid = sc->tids.ftid_base + fidx;
7506
7507         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
7508         if (fwr == NULL)
7509                 return (ENOMEM);
7510         bzero(fwr, sizeof (*fwr));
7511
7512         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
7513
7514         f->pending = 1;
7515         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
7516         return (0);
7517 }
7518
7519 int
7520 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
7521 {
7522         struct adapter *sc = iq->adapter;
7523         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
7524         unsigned int idx = GET_TID(rpl);
7525         unsigned int rc;
7526         struct filter_entry *f;
7527
7528         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
7529             rss->opcode));
7530
7531         if (is_ftid(sc, idx)) {
7532
7533                 idx -= sc->tids.ftid_base;
7534                 f = &sc->tids.ftid_tab[idx];
7535                 rc = G_COOKIE(rpl->cookie);
7536
7537                 mtx_lock(&sc->tids.ftid_lock);
7538                 if (rc == FW_FILTER_WR_FLT_ADDED) {
7539                         KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
7540                             __func__, idx));
7541                         f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
7542                         f->pending = 0;  /* asynchronous setup completed */
7543                         f->valid = 1;
7544                 } else {
7545                         if (rc != FW_FILTER_WR_FLT_DELETED) {
7546                                 /* Add or delete failed, display an error */
7547                                 log(LOG_ERR,
7548                                     "filter %u setup failed with error %u\n",
7549                                     idx, rc);
7550                         }
7551
7552                         clear_filter(f);
7553                         sc->tids.ftids_in_use--;
7554                 }
7555                 wakeup(&sc->tids.ftid_tab);
7556                 mtx_unlock(&sc->tids.ftid_lock);
7557         }
7558
7559         return (0);
7560 }
7561
7562 static int
7563 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
7564 {
7565         int rc;
7566
7567         if (cntxt->cid > M_CTXTQID)
7568                 return (EINVAL);
7569
7570         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
7571             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
7572                 return (EINVAL);
7573
7574         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
7575         if (rc)
7576                 return (rc);
7577
7578         if (sc->flags & FW_OK) {
7579                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
7580                     &cntxt->data[0]);
7581                 if (rc == 0)
7582                         goto done;
7583         }
7584
7585         /*
7586          * Read via firmware failed or wasn't even attempted.  Read directly via
7587          * the backdoor.
7588          */
7589         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
7590 done:
7591         end_synchronized_op(sc, 0);
7592         return (rc);
7593 }
7594
7595 static int
7596 load_fw(struct adapter *sc, struct t4_data *fw)
7597 {
7598         int rc;
7599         uint8_t *fw_data;
7600
7601         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
7602         if (rc)
7603                 return (rc);
7604
7605         if (sc->flags & FULL_INIT_DONE) {
7606                 rc = EBUSY;
7607                 goto done;
7608         }
7609
7610         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
7611         if (fw_data == NULL) {
7612                 rc = ENOMEM;
7613                 goto done;
7614         }
7615
7616         rc = copyin(fw->data, fw_data, fw->len);
7617         if (rc == 0)
7618                 rc = -t4_load_fw(sc, fw_data, fw->len);
7619
7620         free(fw_data, M_CXGBE);
7621 done:
7622         end_synchronized_op(sc, 0);
7623         return (rc);
7624 }
7625
7626 static int
7627 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
7628 {
7629         uint32_t addr, off, remaining, i, n;
7630         uint32_t *buf, *b;
7631         uint32_t mw_base, mw_aperture;
7632         int rc;
7633         uint8_t *dst;
7634
7635         rc = validate_mem_range(sc, mr->addr, mr->len);
7636         if (rc != 0)
7637                 return (rc);
7638
7639         memwin_info(sc, win, &mw_base, &mw_aperture);
7640         buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
7641         addr = mr->addr;
7642         remaining = mr->len;
7643         dst = (void *)mr->data;
7644
7645         while (remaining) {
7646                 off = position_memwin(sc, win, addr);
7647
7648                 /* number of bytes that we'll copy in the inner loop */
7649                 n = min(remaining, mw_aperture - off);
7650                 for (i = 0; i < n; i += 4)
7651                         *b++ = t4_read_reg(sc, mw_base + off + i);
7652
7653                 rc = copyout(buf, dst, n);
7654                 if (rc != 0)
7655                         break;
7656
7657                 b = buf;
7658                 dst += n;
7659                 remaining -= n;
7660                 addr += n;
7661         }
7662
7663         free(buf, M_CXGBE);
7664         return (rc);
7665 }
7666
7667 static int
7668 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
7669 {
7670         int rc;
7671
7672         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
7673                 return (EINVAL);
7674
7675         if (i2cd->len > sizeof(i2cd->data))
7676                 return (EFBIG);
7677
7678         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
7679         if (rc)
7680                 return (rc);
7681         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
7682             i2cd->offset, i2cd->len, &i2cd->data[0]);
7683         end_synchronized_op(sc, 0);
7684
7685         return (rc);
7686 }
7687
7688 static int
7689 in_range(int val, int lo, int hi)
7690 {
7691
7692         return (val < 0 || (val <= hi && val >= lo));
7693 }
7694
7695 static int
7696 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
7697 {
7698         int fw_subcmd, fw_type, rc;
7699
7700         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
7701         if (rc)
7702                 return (rc);
7703
7704         if (!(sc->flags & FULL_INIT_DONE)) {
7705                 rc = EAGAIN;
7706                 goto done;
7707         }
7708
7709         /*
7710          * Translate the cxgbetool parameters into T4 firmware parameters.  (The
7711          * sub-command and type are in common locations.)
7712          */
7713         if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
7714                 fw_subcmd = FW_SCHED_SC_CONFIG;
7715         else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
7716                 fw_subcmd = FW_SCHED_SC_PARAMS;
7717         else {
7718                 rc = EINVAL;
7719                 goto done;
7720         }
7721         if (p->type == SCHED_CLASS_TYPE_PACKET)
7722                 fw_type = FW_SCHED_TYPE_PKTSCHED;
7723         else {
7724                 rc = EINVAL;
7725                 goto done;
7726         }
7727
7728         if (fw_subcmd == FW_SCHED_SC_CONFIG) {
7729                 /* Vet our parameters ..*/
7730                 if (p->u.config.minmax < 0) {
7731                         rc = EINVAL;
7732                         goto done;
7733                 }
7734
7735                 /* And pass the request to the firmware ...*/
7736                 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1);
7737                 goto done;
7738         }
7739
7740         if (fw_subcmd == FW_SCHED_SC_PARAMS) {
7741                 int fw_level;
7742                 int fw_mode;
7743                 int fw_rateunit;
7744                 int fw_ratemode;
7745
7746                 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
7747                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
7748                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
7749                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
7750                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
7751                         fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
7752                 else {
7753                         rc = EINVAL;
7754                         goto done;
7755                 }
7756
7757                 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
7758                         fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
7759                 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
7760                         fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
7761                 else {
7762                         rc = EINVAL;
7763                         goto done;
7764                 }
7765
7766                 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
7767                         fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
7768                 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
7769                         fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
7770                 else {
7771                         rc = EINVAL;
7772                         goto done;
7773                 }
7774
7775                 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
7776                         fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
7777                 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
7778                         fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
7779                 else {
7780                         rc = EINVAL;
7781                         goto done;
7782                 }
7783
7784                 /* Vet our parameters ... */
7785                 if (!in_range(p->u.params.channel, 0, 3) ||
7786                     !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
7787                     !in_range(p->u.params.minrate, 0, 10000000) ||
7788                     !in_range(p->u.params.maxrate, 0, 10000000) ||
7789                     !in_range(p->u.params.weight, 0, 100)) {
7790                         rc = ERANGE;
7791                         goto done;
7792                 }
7793
7794                 /*
7795                  * Translate any unset parameters into the firmware's
7796                  * nomenclature and/or fail the call if the parameters
7797                  * are required ...
7798                  */
7799                 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
7800                     p->u.params.channel < 0 || p->u.params.cl < 0) {
7801                         rc = EINVAL;
7802                         goto done;
7803                 }
7804                 if (p->u.params.minrate < 0)
7805                         p->u.params.minrate = 0;
7806                 if (p->u.params.maxrate < 0) {
7807                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7808                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7809                                 rc = EINVAL;
7810                                 goto done;
7811                         } else
7812                                 p->u.params.maxrate = 0;
7813                 }
7814                 if (p->u.params.weight < 0) {
7815                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
7816                                 rc = EINVAL;
7817                                 goto done;
7818                         } else
7819                                 p->u.params.weight = 0;
7820                 }
7821                 if (p->u.params.pktsize < 0) {
7822                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7823                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7824                                 rc = EINVAL;
7825                                 goto done;
7826                         } else
7827                                 p->u.params.pktsize = 0;
7828                 }
7829
7830                 /* See what the firmware thinks of the request ... */
7831                 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
7832                     fw_rateunit, fw_ratemode, p->u.params.channel,
7833                     p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
7834                     p->u.params.weight, p->u.params.pktsize, 1);
7835                 goto done;
7836         }
7837
7838         rc = EINVAL;
7839 done:
7840         end_synchronized_op(sc, 0);
7841         return (rc);
7842 }
7843
7844 static int
7845 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
7846 {
7847         struct port_info *pi = NULL;
7848         struct sge_txq *txq;
7849         uint32_t fw_mnem, fw_queue, fw_class;
7850         int i, rc;
7851
7852         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
7853         if (rc)
7854                 return (rc);
7855
7856         if (!(sc->flags & FULL_INIT_DONE)) {
7857                 rc = EAGAIN;
7858                 goto done;
7859         }
7860
7861         if (p->port >= sc->params.nports) {
7862                 rc = EINVAL;
7863                 goto done;
7864         }
7865
7866         pi = sc->port[p->port];
7867         if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
7868                 rc = EINVAL;
7869                 goto done;
7870         }
7871
7872         /*
7873          * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
7874          * Scheduling Class in this case).
7875          */
7876         fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
7877             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
7878         fw_class = p->cl < 0 ? 0xffffffff : p->cl;
7879
7880         /*
7881          * If op.queue is non-negative, then we're only changing the scheduling
7882          * on a single specified TX queue.
7883          */
7884         if (p->queue >= 0) {
7885                 txq = &sc->sge.txq[pi->first_txq + p->queue];
7886                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7887                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7888                     &fw_class);
7889                 goto done;
7890         }
7891
7892         /*
7893          * Change the scheduling on all the TX queues for the
7894          * interface.
7895          */
7896         for_each_txq(pi, i, txq) {
7897                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7898                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7899                     &fw_class);
7900                 if (rc)
7901                         goto done;
7902         }
7903
7904         rc = 0;
7905 done:
7906         end_synchronized_op(sc, 0);
7907         return (rc);
7908 }
7909
7910 int
7911 t4_os_find_pci_capability(struct adapter *sc, int cap)
7912 {
7913         int i;
7914
7915         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
7916 }
7917
7918 int
7919 t4_os_pci_save_state(struct adapter *sc)
7920 {
7921         device_t dev;
7922         struct pci_devinfo *dinfo;
7923
7924         dev = sc->dev;
7925         dinfo = device_get_ivars(dev);
7926
7927         pci_cfg_save(dev, dinfo, 0);
7928         return (0);
7929 }
7930
7931 int
7932 t4_os_pci_restore_state(struct adapter *sc)
7933 {
7934         device_t dev;
7935         struct pci_devinfo *dinfo;
7936
7937         dev = sc->dev;
7938         dinfo = device_get_ivars(dev);
7939
7940         pci_cfg_restore(dev, dinfo);
7941         return (0);
7942 }
7943
7944 void
7945 t4_os_portmod_changed(const struct adapter *sc, int idx)
7946 {
7947         struct port_info *pi = sc->port[idx];
7948         static const char *mod_str[] = {
7949                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
7950         };
7951
7952         build_medialist(pi, &pi->media);
7953 #ifdef DEV_NETMAP
7954         build_medialist(pi, &pi->nm_media);
7955 #endif
7956
7957         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
7958                 if_printf(pi->ifp, "transceiver unplugged.\n");
7959         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
7960                 if_printf(pi->ifp, "unknown transceiver inserted.\n");
7961         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
7962                 if_printf(pi->ifp, "unsupported transceiver inserted.\n");
7963         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
7964                 if_printf(pi->ifp, "%s transceiver inserted.\n",
7965                     mod_str[pi->mod_type]);
7966         } else {
7967                 if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
7968                     pi->mod_type);
7969         }
7970 }
7971
7972 void
7973 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
7974 {
7975         struct port_info *pi = sc->port[idx];
7976         struct ifnet *ifp = pi->ifp;
7977
7978         if (link_stat) {
7979                 pi->linkdnrc = -1;
7980                 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
7981                 if_link_state_change(ifp, LINK_STATE_UP);
7982         } else {
7983                 if (reason >= 0)
7984                         pi->linkdnrc = reason;
7985                 if_link_state_change(ifp, LINK_STATE_DOWN);
7986         }
7987 }
7988
7989 void
7990 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
7991 {
7992         struct adapter *sc;
7993
7994         sx_slock(&t4_list_lock);
7995         SLIST_FOREACH(sc, &t4_list, link) {
7996                 /*
7997                  * func should not make any assumptions about what state sc is
7998                  * in - the only guarantee is that sc->sc_lock is a valid lock.
7999                  */
8000                 func(sc, arg);
8001         }
8002         sx_sunlock(&t4_list_lock);
8003 }
8004
8005 static int
8006 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
8007 {
8008        return (0);
8009 }
8010
8011 static int
8012 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
8013 {
8014        return (0);
8015 }
8016
8017 static int
8018 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
8019     struct thread *td)
8020 {
8021         int rc;
8022         struct adapter *sc = dev->si_drv1;
8023
8024         rc = priv_check(td, PRIV_DRIVER);
8025         if (rc != 0)
8026                 return (rc);
8027
8028         switch (cmd) {
8029         case CHELSIO_T4_GETREG: {
8030                 struct t4_reg *edata = (struct t4_reg *)data;
8031
8032                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8033                         return (EFAULT);
8034
8035                 if (edata->size == 4)
8036                         edata->val = t4_read_reg(sc, edata->addr);
8037                 else if (edata->size == 8)
8038                         edata->val = t4_read_reg64(sc, edata->addr);
8039                 else
8040                         return (EINVAL);
8041
8042                 break;
8043         }
8044         case CHELSIO_T4_SETREG: {
8045                 struct t4_reg *edata = (struct t4_reg *)data;
8046
8047                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8048                         return (EFAULT);
8049
8050                 if (edata->size == 4) {
8051                         if (edata->val & 0xffffffff00000000)
8052                                 return (EINVAL);
8053                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
8054                 } else if (edata->size == 8)
8055                         t4_write_reg64(sc, edata->addr, edata->val);
8056                 else
8057                         return (EINVAL);
8058                 break;
8059         }
8060         case CHELSIO_T4_REGDUMP: {
8061                 struct t4_regdump *regs = (struct t4_regdump *)data;
8062                 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
8063                 uint8_t *buf;
8064
8065                 if (regs->len < reglen) {
8066                         regs->len = reglen; /* hint to the caller */
8067                         return (ENOBUFS);
8068                 }
8069
8070                 regs->len = reglen;
8071                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
8072                 t4_get_regs(sc, regs, buf);
8073                 rc = copyout(buf, regs->data, reglen);
8074                 free(buf, M_CXGBE);
8075                 break;
8076         }
8077         case CHELSIO_T4_GET_FILTER_MODE:
8078                 rc = get_filter_mode(sc, (uint32_t *)data);
8079                 break;
8080         case CHELSIO_T4_SET_FILTER_MODE:
8081                 rc = set_filter_mode(sc, *(uint32_t *)data);
8082                 break;
8083         case CHELSIO_T4_GET_FILTER:
8084                 rc = get_filter(sc, (struct t4_filter *)data);
8085                 break;
8086         case CHELSIO_T4_SET_FILTER:
8087                 rc = set_filter(sc, (struct t4_filter *)data);
8088                 break;
8089         case CHELSIO_T4_DEL_FILTER:
8090                 rc = del_filter(sc, (struct t4_filter *)data);
8091                 break;
8092         case CHELSIO_T4_GET_SGE_CONTEXT:
8093                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
8094                 break;
8095         case CHELSIO_T4_LOAD_FW:
8096                 rc = load_fw(sc, (struct t4_data *)data);
8097                 break;
8098         case CHELSIO_T4_GET_MEM:
8099                 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
8100                 break;
8101         case CHELSIO_T4_GET_I2C:
8102                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
8103                 break;
8104         case CHELSIO_T4_CLEAR_STATS: {
8105                 int i;
8106                 u_int port_id = *(uint32_t *)data;
8107                 struct port_info *pi;
8108
8109                 if (port_id >= sc->params.nports)
8110                         return (EINVAL);
8111                 pi = sc->port[port_id];
8112
8113                 /* MAC stats */
8114                 t4_clr_port_stats(sc, pi->tx_chan);
8115                 pi->tx_parse_error = 0;
8116
8117                 if (pi->flags & PORT_INIT_DONE) {
8118                         struct sge_rxq *rxq;
8119                         struct sge_txq *txq;
8120                         struct sge_wrq *wrq;
8121
8122                         for_each_rxq(pi, i, rxq) {
8123 #if defined(INET) || defined(INET6)
8124                                 rxq->lro.lro_queued = 0;
8125                                 rxq->lro.lro_flushed = 0;
8126 #endif
8127                                 rxq->rxcsum = 0;
8128                                 rxq->vlan_extraction = 0;
8129                         }
8130
8131                         for_each_txq(pi, i, txq) {
8132                                 txq->txcsum = 0;
8133                                 txq->tso_wrs = 0;
8134                                 txq->vlan_insertion = 0;
8135                                 txq->imm_wrs = 0;
8136                                 txq->sgl_wrs = 0;
8137                                 txq->txpkt_wrs = 0;
8138                                 txq->txpkts0_wrs = 0;
8139                                 txq->txpkts1_wrs = 0;
8140                                 txq->txpkts0_pkts = 0;
8141                                 txq->txpkts1_pkts = 0;
8142                                 mp_ring_reset_stats(txq->r);
8143                         }
8144
8145 #ifdef TCP_OFFLOAD
8146                         /* nothing to clear for each ofld_rxq */
8147
8148                         for_each_ofld_txq(pi, i, wrq) {
8149                                 wrq->tx_wrs_direct = 0;
8150                                 wrq->tx_wrs_copied = 0;
8151                         }
8152 #endif
8153                         wrq = &sc->sge.ctrlq[pi->port_id];
8154                         wrq->tx_wrs_direct = 0;
8155                         wrq->tx_wrs_copied = 0;
8156                 }
8157                 break;
8158         }
8159         case CHELSIO_T4_SCHED_CLASS:
8160                 rc = set_sched_class(sc, (struct t4_sched_params *)data);
8161                 break;
8162         case CHELSIO_T4_SCHED_QUEUE:
8163                 rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
8164                 break;
8165         case CHELSIO_T4_GET_TRACER:
8166                 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
8167                 break;
8168         case CHELSIO_T4_SET_TRACER:
8169                 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
8170                 break;
8171         default:
8172                 rc = EINVAL;
8173         }
8174
8175         return (rc);
8176 }
8177
8178 #ifdef TCP_OFFLOAD
8179 void
8180 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask,
8181     const unsigned int *pgsz_order)
8182 {
8183         struct port_info *pi = ifp->if_softc;
8184         struct adapter *sc = pi->adapter;
8185
8186         t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
8187         t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |
8188                 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) |
8189                 V_HPZ3(pgsz_order[3]));
8190 }
8191
8192 static int
8193 toe_capability(struct port_info *pi, int enable)
8194 {
8195         int rc;
8196         struct adapter *sc = pi->adapter;
8197
8198         ASSERT_SYNCHRONIZED_OP(sc);
8199
8200         if (!is_offload(sc))
8201                 return (ENODEV);
8202
8203         if (enable) {
8204                 if (!(sc->flags & FULL_INIT_DONE)) {
8205                         rc = cxgbe_init_synchronized(pi);
8206                         if (rc)
8207                                 return (rc);
8208                 }
8209
8210                 if (isset(&sc->offload_map, pi->port_id))
8211                         return (0);
8212
8213                 if (!(sc->flags & TOM_INIT_DONE)) {
8214                         rc = t4_activate_uld(sc, ULD_TOM);
8215                         if (rc == EAGAIN) {
8216                                 log(LOG_WARNING,
8217                                     "You must kldload t4_tom.ko before trying "
8218                                     "to enable TOE on a cxgbe interface.\n");
8219                         }
8220                         if (rc != 0)
8221                                 return (rc);
8222                         KASSERT(sc->tom_softc != NULL,
8223                             ("%s: TOM activated but softc NULL", __func__));
8224                         KASSERT(sc->flags & TOM_INIT_DONE,
8225                             ("%s: TOM activated but flag not set", __func__));
8226                 }
8227
8228                 setbit(&sc->offload_map, pi->port_id);
8229         } else {
8230                 if (!isset(&sc->offload_map, pi->port_id))
8231                         return (0);
8232
8233                 KASSERT(sc->flags & TOM_INIT_DONE,
8234                     ("%s: TOM never initialized?", __func__));
8235                 clrbit(&sc->offload_map, pi->port_id);
8236         }
8237
8238         return (0);
8239 }
8240
8241 /*
8242  * Add an upper layer driver to the global list.
8243  */
8244 int
8245 t4_register_uld(struct uld_info *ui)
8246 {
8247         int rc = 0;
8248         struct uld_info *u;
8249
8250         sx_xlock(&t4_uld_list_lock);
8251         SLIST_FOREACH(u, &t4_uld_list, link) {
8252             if (u->uld_id == ui->uld_id) {
8253                     rc = EEXIST;
8254                     goto done;
8255             }
8256         }
8257
8258         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
8259         ui->refcount = 0;
8260 done:
8261         sx_xunlock(&t4_uld_list_lock);
8262         return (rc);
8263 }
8264
8265 int
8266 t4_unregister_uld(struct uld_info *ui)
8267 {
8268         int rc = EINVAL;
8269         struct uld_info *u;
8270
8271         sx_xlock(&t4_uld_list_lock);
8272
8273         SLIST_FOREACH(u, &t4_uld_list, link) {
8274             if (u == ui) {
8275                     if (ui->refcount > 0) {
8276                             rc = EBUSY;
8277                             goto done;
8278                     }
8279
8280                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
8281                     rc = 0;
8282                     goto done;
8283             }
8284         }
8285 done:
8286         sx_xunlock(&t4_uld_list_lock);
8287         return (rc);
8288 }
8289
8290 int
8291 t4_activate_uld(struct adapter *sc, int id)
8292 {
8293         int rc = EAGAIN;
8294         struct uld_info *ui;
8295
8296         ASSERT_SYNCHRONIZED_OP(sc);
8297
8298         sx_slock(&t4_uld_list_lock);
8299
8300         SLIST_FOREACH(ui, &t4_uld_list, link) {
8301                 if (ui->uld_id == id) {
8302                         if (!(sc->flags & FULL_INIT_DONE)) {
8303                                 rc = adapter_full_init(sc);
8304                                 if (rc != 0)
8305                                         goto done;
8306                         }
8307
8308                         rc = ui->activate(sc);
8309                         if (rc == 0)
8310                                 ui->refcount++;
8311                         goto done;
8312                 }
8313         }
8314 done:
8315         sx_sunlock(&t4_uld_list_lock);
8316
8317         return (rc);
8318 }
8319
8320 int
8321 t4_deactivate_uld(struct adapter *sc, int id)
8322 {
8323         int rc = EINVAL;
8324         struct uld_info *ui;
8325
8326         ASSERT_SYNCHRONIZED_OP(sc);
8327
8328         sx_slock(&t4_uld_list_lock);
8329
8330         SLIST_FOREACH(ui, &t4_uld_list, link) {
8331                 if (ui->uld_id == id) {
8332                         rc = ui->deactivate(sc);
8333                         if (rc == 0)
8334                                 ui->refcount--;
8335                         goto done;
8336                 }
8337         }
8338 done:
8339         sx_sunlock(&t4_uld_list_lock);
8340
8341         return (rc);
8342 }
8343 #endif
8344
8345 /*
8346  * Come up with reasonable defaults for some of the tunables, provided they're
8347  * not set by the user (in which case we'll use the values as is).
8348  */
8349 static void
8350 tweak_tunables(void)
8351 {
8352         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
8353
8354         if (t4_ntxq10g < 1)
8355                 t4_ntxq10g = min(nc, NTXQ_10G);
8356
8357         if (t4_ntxq1g < 1)
8358                 t4_ntxq1g = min(nc, NTXQ_1G);
8359
8360         if (t4_nrxq10g < 1)
8361                 t4_nrxq10g = min(nc, NRXQ_10G);
8362
8363         if (t4_nrxq1g < 1)
8364                 t4_nrxq1g = min(nc, NRXQ_1G);
8365
8366 #ifdef TCP_OFFLOAD
8367         if (t4_nofldtxq10g < 1)
8368                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
8369
8370         if (t4_nofldtxq1g < 1)
8371                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
8372
8373         if (t4_nofldrxq10g < 1)
8374                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
8375
8376         if (t4_nofldrxq1g < 1)
8377                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
8378
8379         if (t4_toecaps_allowed == -1)
8380                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
8381 #else
8382         if (t4_toecaps_allowed == -1)
8383                 t4_toecaps_allowed = 0;
8384 #endif
8385
8386 #ifdef DEV_NETMAP
8387         if (t4_nnmtxq10g < 1)
8388                 t4_nnmtxq10g = min(nc, NNMTXQ_10G);
8389
8390         if (t4_nnmtxq1g < 1)
8391                 t4_nnmtxq1g = min(nc, NNMTXQ_1G);
8392
8393         if (t4_nnmrxq10g < 1)
8394                 t4_nnmrxq10g = min(nc, NNMRXQ_10G);
8395
8396         if (t4_nnmrxq1g < 1)
8397                 t4_nnmrxq1g = min(nc, NNMRXQ_1G);
8398 #endif
8399
8400         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
8401                 t4_tmr_idx_10g = TMR_IDX_10G;
8402
8403         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
8404                 t4_pktc_idx_10g = PKTC_IDX_10G;
8405
8406         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
8407                 t4_tmr_idx_1g = TMR_IDX_1G;
8408
8409         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
8410                 t4_pktc_idx_1g = PKTC_IDX_1G;
8411
8412         if (t4_qsize_txq < 128)
8413                 t4_qsize_txq = 128;
8414
8415         if (t4_qsize_rxq < 128)
8416                 t4_qsize_rxq = 128;
8417         while (t4_qsize_rxq & 7)
8418                 t4_qsize_rxq++;
8419
8420         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
8421 }
8422
8423 static struct sx mlu;   /* mod load unload */
8424 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
8425
8426 static int
8427 mod_event(module_t mod, int cmd, void *arg)
8428 {
8429         int rc = 0;
8430         static int loaded = 0;
8431
8432         switch (cmd) {
8433         case MOD_LOAD:
8434                 sx_xlock(&mlu);
8435                 if (loaded++ == 0) {
8436                         t4_sge_modload();
8437                         sx_init(&t4_list_lock, "T4/T5 adapters");
8438                         SLIST_INIT(&t4_list);
8439 #ifdef TCP_OFFLOAD
8440                         sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
8441                         SLIST_INIT(&t4_uld_list);
8442 #endif
8443                         t4_tracer_modload();
8444                         tweak_tunables();
8445                 }
8446                 sx_xunlock(&mlu);
8447                 break;
8448
8449         case MOD_UNLOAD:
8450                 sx_xlock(&mlu);
8451                 if (--loaded == 0) {
8452                         int tries;
8453
8454                         sx_slock(&t4_list_lock);
8455                         if (!SLIST_EMPTY(&t4_list)) {
8456                                 rc = EBUSY;
8457                                 sx_sunlock(&t4_list_lock);
8458                                 goto done_unload;
8459                         }
8460 #ifdef TCP_OFFLOAD
8461                         sx_slock(&t4_uld_list_lock);
8462                         if (!SLIST_EMPTY(&t4_uld_list)) {
8463                                 rc = EBUSY;
8464                                 sx_sunlock(&t4_uld_list_lock);
8465                                 sx_sunlock(&t4_list_lock);
8466                                 goto done_unload;
8467                         }
8468 #endif
8469                         tries = 0;
8470                         while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
8471                                 uprintf("%ju clusters with custom free routine "
8472                                     "still is use.\n", t4_sge_extfree_refs());
8473                                 pause("t4unload", 2 * hz);
8474                         }
8475 #ifdef TCP_OFFLOAD
8476                         sx_sunlock(&t4_uld_list_lock);
8477 #endif
8478                         sx_sunlock(&t4_list_lock);
8479
8480                         if (t4_sge_extfree_refs() == 0) {
8481                                 t4_tracer_modunload();
8482 #ifdef TCP_OFFLOAD
8483                                 sx_destroy(&t4_uld_list_lock);
8484 #endif
8485                                 sx_destroy(&t4_list_lock);
8486                                 t4_sge_modunload();
8487                                 loaded = 0;
8488                         } else {
8489                                 rc = EBUSY;
8490                                 loaded++;       /* undo earlier decrement */
8491                         }
8492                 }
8493 done_unload:
8494                 sx_xunlock(&mlu);
8495                 break;
8496         }
8497
8498         return (rc);
8499 }
8500
8501 static devclass_t t4_devclass, t5_devclass;
8502 static devclass_t cxgbe_devclass, cxl_devclass;
8503
8504 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
8505 MODULE_VERSION(t4nex, 1);
8506 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
8507
8508 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
8509 MODULE_VERSION(t5nex, 1);
8510 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
8511
8512 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
8513 MODULE_VERSION(cxgbe, 1);
8514
8515 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
8516 MODULE_VERSION(cxl, 1);