sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/conf.h>
  36 #include <sys/priv.h>
  37 #include <sys/kernel.h>
  38 #include <sys/bus.h>
  39 #include <sys/module.h>
  40 #include <sys/malloc.h>
  41 #include <sys/queue.h>
  42 #include <sys/taskqueue.h>
  43 #include <sys/pciio.h>
  44 #include <dev/pci/pcireg.h>
  45 #include <dev/pci/pcivar.h>
  46 #include <dev/pci/pci_private.h>
  47 #include <sys/firmware.h>
  48 #include <sys/sbuf.h>
  49 #include <sys/smp.h>
  50 #include <sys/socket.h>
  51 #include <sys/sockio.h>
  52 #include <sys/sysctl.h>
  53 #include <net/ethernet.h>
  54 #include <net/if.h>
  55 #include <net/if_types.h>
  56 #include <net/if_dl.h>
  57 #include <net/if_vlan_var.h>
  58 #if defined(__i386__) || defined(__amd64__)
  59 #include <vm/vm.h>
  60 #include <vm/pmap.h>
  61 #endif
  62
  63 #include "common/common.h"
  64 #include "common/t4_msg.h"
  65 #include "common/t4_regs.h"
  66 #include "common/t4_regs_values.h"
  67 #include "t4_ioctl.h"
  68 #include "t4_l2t.h"
  69
  70 /* T4 bus driver interface */
  71 static int t4_probe(device_t);
  72 static int t4_attach(device_t);
  73 static int t4_detach(device_t);
  74 static device_method_t t4_methods[] = {
  75         DEVMETHOD(device_probe,         t4_probe),
  76         DEVMETHOD(device_attach,        t4_attach),
  77         DEVMETHOD(device_detach,        t4_detach),
  78
  79         DEVMETHOD_END
  80 };
  81 static driver_t t4_driver = {
  82         "t4nex",
  83         t4_methods,
  84         sizeof(struct adapter)
  85 };
  86
  87
  88 /* T4 port (cxgbe) interface */
  89 static int cxgbe_probe(device_t);
  90 static int cxgbe_attach(device_t);
  91 static int cxgbe_detach(device_t);
  92 static device_method_t cxgbe_methods[] = {
  93         DEVMETHOD(device_probe,         cxgbe_probe),
  94         DEVMETHOD(device_attach,        cxgbe_attach),
  95         DEVMETHOD(device_detach,        cxgbe_detach),
  96         { 0, 0 }
  97 };
  98 static driver_t cxgbe_driver = {
  99         "cxgbe",
 100         cxgbe_methods,
 101         sizeof(struct port_info)
 102 };
 103
 104 static d_ioctl_t t4_ioctl;
 105 static d_open_t t4_open;
 106 static d_close_t t4_close;
 107
 108 static struct cdevsw t4_cdevsw = {
 109        .d_version = D_VERSION,
 110        .d_flags = 0,
 111        .d_open = t4_open,
 112        .d_close = t4_close,
 113        .d_ioctl = t4_ioctl,
 114        .d_name = "t4nex",
 115 };
 116
 117 /* T5 bus driver interface */
 118 static int t5_probe(device_t);
 119 static device_method_t t5_methods[] = {
 120         DEVMETHOD(device_probe,         t5_probe),
 121         DEVMETHOD(device_attach,        t4_attach),
 122         DEVMETHOD(device_detach,        t4_detach),
 123
 124         DEVMETHOD_END
 125 };
 126 static driver_t t5_driver = {
 127         "t5nex",
 128         t5_methods,
 129         sizeof(struct adapter)
 130 };
 131
 132
 133 /* T5 port (cxl) interface */
 134 static driver_t cxl_driver = {
 135         "cxl",
 136         cxgbe_methods,
 137         sizeof(struct port_info)
 138 };
 139
 140 static struct cdevsw t5_cdevsw = {
 141        .d_version = D_VERSION,
 142        .d_flags = 0,
 143        .d_open = t4_open,
 144        .d_close = t4_close,
 145        .d_ioctl = t4_ioctl,
 146        .d_name = "t5nex",
 147 };
 148
 149 /* ifnet + media interface */
 150 static void cxgbe_init(void *);
 151 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 152 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 153 static void cxgbe_qflush(struct ifnet *);
 154 static int cxgbe_media_change(struct ifnet *);
 155 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 156
 157 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
 158
 159 /*
 160  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 161  * then ADAPTER_LOCK, then t4_uld_list_lock.
 162  */
 163 static struct sx t4_list_lock;
 164 SLIST_HEAD(, adapter) t4_list;
 165 #ifdef TCP_OFFLOAD
 166 static struct sx t4_uld_list_lock;
 167 SLIST_HEAD(, uld_info) t4_uld_list;
 168 #endif
 169
 170 /*
 171  * Tunables.  See tweak_tunables() too.
 172  *
 173  * Each tunable is set to a default value here if it's known at compile-time.
 174  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
 175  * provide a reasonable default when the driver is loaded.
 176  *
 177  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
 178  * T5 are under hw.cxl.
 179  */
 180
 181 /*
 182  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 183  */
 184 #define NTXQ_10G 16
 185 static int t4_ntxq10g = -1;
 186 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 187
 188 #define NRXQ_10G 8
 189 static int t4_nrxq10g = -1;
 190 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 191
 192 #define NTXQ_1G 4
 193 static int t4_ntxq1g = -1;
 194 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 195
 196 #define NRXQ_1G 2
 197 static int t4_nrxq1g = -1;
 198 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 199
 200 static int t4_rsrv_noflowq = 0;
 201 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
 202
 203 #ifdef TCP_OFFLOAD
 204 #define NOFLDTXQ_10G 8
 205 static int t4_nofldtxq10g = -1;
 206 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 207
 208 #define NOFLDRXQ_10G 2
 209 static int t4_nofldrxq10g = -1;
 210 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 211
 212 #define NOFLDTXQ_1G 2
 213 static int t4_nofldtxq1g = -1;
 214 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 215
 216 #define NOFLDRXQ_1G 1
 217 static int t4_nofldrxq1g = -1;
 218 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 219 #endif
 220
 221 #ifdef DEV_NETMAP
 222 #define NNMTXQ_10G 2
 223 static int t4_nnmtxq10g = -1;
 224 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g);
 225
 226 #define NNMRXQ_10G 2
 227 static int t4_nnmrxq10g = -1;
 228 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g);
 229
 230 #define NNMTXQ_1G 1
 231 static int t4_nnmtxq1g = -1;
 232 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g);
 233
 234 #define NNMRXQ_1G 1
 235 static int t4_nnmrxq1g = -1;
 236 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g);
 237 #endif
 238
 239 /*
 240  * Holdoff parameters for 10G and 1G ports.
 241  */
 242 #define TMR_IDX_10G 1
 243 static int t4_tmr_idx_10g = TMR_IDX_10G;
 244 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 245
 246 #define PKTC_IDX_10G (-1)
 247 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 248 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 249
 250 #define TMR_IDX_1G 1
 251 static int t4_tmr_idx_1g = TMR_IDX_1G;
 252 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 253
 254 #define PKTC_IDX_1G (-1)
 255 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 256 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 257
 258 /*
 259  * Size (# of entries) of each tx and rx queue.
 260  */
 261 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 262 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 263
 264 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 265 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 266
 267 /*
 268  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 269  */
 270 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 271 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 272
 273 /*
 274  * Configuration file.
 275  */
 276 #define DEFAULT_CF      "default"
 277 #define FLASH_CF        "flash"
 278 #define UWIRE_CF        "uwire"
 279 #define FPGA_CF         "fpga"
 280 static char t4_cfg_file[32] = DEFAULT_CF;
 281 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 282
 283 /*
 284  * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively).
 285  * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
 286  * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
 287  *            mark or when signalled to do so, 0 to never emit PAUSE.
 288  */
 289 static int t4_pause_settings = PAUSE_TX | PAUSE_RX;
 290 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings);
 291
 292 /*
 293  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
 294  * encouraged respectively).
 295  */
 296 static unsigned int t4_fw_install = 1;
 297 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 298
 299 /*
 300  * ASIC features that will be used.  Disable the ones you don't want so that the
 301  * chip resources aren't wasted on features that will not be used.
 302  */
 303 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 304 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 305
 306 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 307 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 308
 309 static int t4_toecaps_allowed = -1;
 310 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 311
 312 static int t4_rdmacaps_allowed = 0;
 313 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 314
 315 static int t4_iscsicaps_allowed = 0;
 316 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 317
 318 static int t4_fcoecaps_allowed = 0;
 319 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 320
 321 static int t5_write_combine = 0;
 322 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 323
 324 struct intrs_and_queues {
 325         uint16_t intr_type;     /* INTx, MSI, or MSI-X */
 326         uint16_t nirq;          /* Total # of vectors */
 327         uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
 328         uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */
 329         uint16_t ntxq10g;       /* # of NIC txq's for each 10G port */
 330         uint16_t nrxq10g;       /* # of NIC rxq's for each 10G port */
 331         uint16_t ntxq1g;        /* # of NIC txq's for each 1G port */
 332         uint16_t nrxq1g;        /* # of NIC rxq's for each 1G port */
 333         uint16_t rsrv_noflowq;  /* Flag whether to reserve queue 0 */
 334 #ifdef TCP_OFFLOAD
 335         uint16_t nofldtxq10g;   /* # of TOE txq's for each 10G port */
 336         uint16_t nofldrxq10g;   /* # of TOE rxq's for each 10G port */
 337         uint16_t nofldtxq1g;    /* # of TOE txq's for each 1G port */
 338         uint16_t nofldrxq1g;    /* # of TOE rxq's for each 1G port */
 339 #endif
 340 #ifdef DEV_NETMAP
 341         uint16_t nnmtxq10g;     /* # of netmap txq's for each 10G port */
 342         uint16_t nnmrxq10g;     /* # of netmap rxq's for each 10G port */
 343         uint16_t nnmtxq1g;      /* # of netmap txq's for each 1G port */
 344         uint16_t nnmrxq1g;      /* # of netmap rxq's for each 1G port */
 345 #endif
 346 };
 347
 348 struct filter_entry {
 349         uint32_t valid:1;       /* filter allocated and valid */
 350         uint32_t locked:1;      /* filter is administratively locked */
 351         uint32_t pending:1;     /* filter action is pending firmware reply */
 352         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 353         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 354
 355         struct t4_filter_specification fs;
 356 };
 357
 358 static int map_bars_0_and_4(struct adapter *);
 359 static int map_bar_2(struct adapter *);
 360 static void setup_memwin(struct adapter *);
 361 static int validate_mem_range(struct adapter *, uint32_t, int);
 362 static int fwmtype_to_hwmtype(int);
 363 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
 364     uint32_t *);
 365 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
 366 static uint32_t position_memwin(struct adapter *, int, uint32_t);
 367 static int cfg_itype_and_nqueues(struct adapter *, int, int,
 368     struct intrs_and_queues *);
 369 static int prep_firmware(struct adapter *);
 370 static int partition_resources(struct adapter *, const struct firmware *,
 371     const char *);
 372 static int get_params__pre_init(struct adapter *);
 373 static int get_params__post_init(struct adapter *);
 374 static int set_params__post_init(struct adapter *);
 375 static void t4_set_desc(struct adapter *);
 376 static void build_medialist(struct port_info *, struct ifmedia *);
 377 static int cxgbe_init_synchronized(struct port_info *);
 378 static int cxgbe_uninit_synchronized(struct port_info *);
 379 static int setup_intr_handlers(struct adapter *);
 380 static void quiesce_eq(struct adapter *, struct sge_eq *);
 381 static void quiesce_iq(struct adapter *, struct sge_iq *);
 382 static void quiesce_fl(struct adapter *, struct sge_fl *);
 383 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 384     driver_intr_t *, void *, char *);
 385 static int t4_free_irq(struct adapter *, struct irq *);
 386 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
 387     unsigned int);
 388 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 389 static void cxgbe_tick(void *);
 390 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 391 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
 392     struct mbuf *);
 393 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 394 static int fw_msg_not_handled(struct adapter *, const __be64 *);
 395 static int t4_sysctls(struct adapter *);
 396 static int cxgbe_sysctls(struct port_info *);
 397 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 398 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 399 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
 400 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 401 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 402 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 403 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 404 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 405 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
 406 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 407 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 408 #ifdef SBUF_DRAIN
 409 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 410 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 411 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 412 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
 413 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
 414 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 415 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 416 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 417 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 418 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 419 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 420 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 421 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
 422 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 423 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
 424 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 425 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 426 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 427 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 428 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 429 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 430 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
 431 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 432 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 433 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 434 #endif
 435 static inline void txq_start(struct ifnet *, struct sge_txq *);
 436 static uint32_t fconf_to_mode(uint32_t);
 437 static uint32_t mode_to_fconf(uint32_t);
 438 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 439 static int get_filter_mode(struct adapter *, uint32_t *);
 440 static int set_filter_mode(struct adapter *, uint32_t);
 441 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 442 static int get_filter(struct adapter *, struct t4_filter *);
 443 static int set_filter(struct adapter *, struct t4_filter *);
 444 static int del_filter(struct adapter *, struct t4_filter *);
 445 static void clear_filter(struct filter_entry *);
 446 static int set_filter_wr(struct adapter *, int);
 447 static int del_filter_wr(struct adapter *, int);
 448 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 449 static int load_fw(struct adapter *, struct t4_data *);
 450 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 451 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 452 static int set_sched_class(struct adapter *, struct t4_sched_params *);
 453 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
 454 #ifdef TCP_OFFLOAD
 455 static int toe_capability(struct port_info *, int);
 456 #endif
 457 static int mod_event(module_t, int, void *);
 458
 459 struct {
 460         uint16_t device;
 461         char *desc;
 462 } t4_pciids[] = {
 463         {0xa000, "Chelsio Terminator 4 FPGA"},
 464         {0x4400, "Chelsio T440-dbg"},
 465         {0x4401, "Chelsio T420-CR"},
 466         {0x4402, "Chelsio T422-CR"},
 467         {0x4403, "Chelsio T440-CR"},
 468         {0x4404, "Chelsio T420-BCH"},
 469         {0x4405, "Chelsio T440-BCH"},
 470         {0x4406, "Chelsio T440-CH"},
 471         {0x4407, "Chelsio T420-SO"},
 472         {0x4408, "Chelsio T420-CX"},
 473         {0x4409, "Chelsio T420-BT"},
 474         {0x440a, "Chelsio T404-BT"},
 475         {0x440e, "Chelsio T440-LP-CR"},
 476 }, t5_pciids[] = {
 477         {0xb000, "Chelsio Terminator 5 FPGA"},
 478         {0x5400, "Chelsio T580-dbg"},
 479         {0x5401,  "Chelsio T520-CR"},           /* 2 x 10G */
 480         {0x5402,  "Chelsio T522-CR"},           /* 2 x 10G, 2 X 1G */
 481         {0x5403,  "Chelsio T540-CR"},           /* 4 x 10G */
 482         {0x5407,  "Chelsio T520-SO"},           /* 2 x 10G, nomem */
 483         {0x5409,  "Chelsio T520-BT"},           /* 2 x 10GBaseT */
 484         {0x540a,  "Chelsio T504-BT"},           /* 4 x 1G */
 485         {0x540d,  "Chelsio T580-CR"},           /* 2 x 40G */
 486         {0x540e,  "Chelsio T540-LP-CR"},        /* 4 x 10G */
 487         {0x5410,  "Chelsio T580-LP-CR"},        /* 2 x 40G */
 488         {0x5411,  "Chelsio T520-LL-CR"},        /* 2 x 10G */
 489         {0x5412,  "Chelsio T560-CR"},           /* 1 x 40G, 2 x 10G */
 490         {0x5414,  "Chelsio T580-LP-SO-CR"},     /* 2 x 40G, nomem */
 491         {0x5415,  "Chelsio T502-BT"},           /* 2 x 1G */
 492 #ifdef notyet
 493         {0x5404,  "Chelsio T520-BCH"},
 494         {0x5405,  "Chelsio T540-BCH"},
 495         {0x5406,  "Chelsio T540-CH"},
 496         {0x5408,  "Chelsio T520-CX"},
 497         {0x540b,  "Chelsio B520-SR"},
 498         {0x540c,  "Chelsio B504-BT"},
 499         {0x540f,  "Chelsio Amsterdam"},
 500         {0x5413,  "Chelsio T580-CHR"},
 501 #endif
 502 };
 503
 504 #ifdef TCP_OFFLOAD
 505 /*
 506  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 507  * exactly the same for both rxq and ofld_rxq.
 508  */
 509 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 510 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 511 #endif
 512
 513 /* No easy way to include t4_msg.h before adapter.h so we check this way */
 514 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
 515 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
 516
 517 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
 518
 519 static int
 520 t4_probe(device_t dev)
 521 {
 522         int i;
 523         uint16_t v = pci_get_vendor(dev);
 524         uint16_t d = pci_get_device(dev);
 525         uint8_t f = pci_get_function(dev);
 526
 527         if (v != PCI_VENDOR_ID_CHELSIO)
 528                 return (ENXIO);
 529
 530         /* Attach only to PF0 of the FPGA */
 531         if (d == 0xa000 && f != 0)
 532                 return (ENXIO);
 533
 534         for (i = 0; i < nitems(t4_pciids); i++) {
 535                 if (d == t4_pciids[i].device) {
 536                         device_set_desc(dev, t4_pciids[i].desc);
 537                         return (BUS_PROBE_DEFAULT);
 538                 }
 539         }
 540
 541         return (ENXIO);
 542 }
 543
 544 static int
 545 t5_probe(device_t dev)
 546 {
 547         int i;
 548         uint16_t v = pci_get_vendor(dev);
 549         uint16_t d = pci_get_device(dev);
 550         uint8_t f = pci_get_function(dev);
 551
 552         if (v != PCI_VENDOR_ID_CHELSIO)
 553                 return (ENXIO);
 554
 555         /* Attach only to PF0 of the FPGA */
 556         if (d == 0xb000 && f != 0)
 557                 return (ENXIO);
 558
 559         for (i = 0; i < nitems(t5_pciids); i++) {
 560                 if (d == t5_pciids[i].device) {
 561                         device_set_desc(dev, t5_pciids[i].desc);
 562                         return (BUS_PROBE_DEFAULT);
 563                 }
 564         }
 565
 566         return (ENXIO);
 567 }
 568
 569 static int
 570 t4_attach(device_t dev)
 571 {
 572         struct adapter *sc;
 573         int rc = 0, i, n10g, n1g, rqidx, tqidx;
 574         struct intrs_and_queues iaq;
 575         struct sge *s;
 576 #ifdef TCP_OFFLOAD
 577         int ofld_rqidx, ofld_tqidx;
 578 #endif
 579 #ifdef DEV_NETMAP
 580         int nm_rqidx, nm_tqidx;
 581 #endif
 582
 583         sc = device_get_softc(dev);
 584         sc->dev = dev;
 585
 586         pci_enable_busmaster(dev);
 587         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 588                 uint32_t v;
 589
 590                 pci_set_max_read_req(dev, 4096);
 591                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 592                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 593                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 594
 595                 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
 596         }
 597
 598         sc->traceq = -1;
 599         mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
 600         snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
 601             device_get_nameunit(dev));
 602
 603         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 604             device_get_nameunit(dev));
 605         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 606         sx_xlock(&t4_list_lock);
 607         SLIST_INSERT_HEAD(&t4_list, sc, link);
 608         sx_xunlock(&t4_list_lock);
 609
 610         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 611         TAILQ_INIT(&sc->sfl);
 612         callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
 613
 614         rc = map_bars_0_and_4(sc);
 615         if (rc != 0)
 616                 goto done; /* error message displayed already */
 617
 618         /*
 619          * This is the real PF# to which we're attaching.  Works from within PCI
 620          * passthrough environments too, where pci_get_function() could return a
 621          * different PF# depending on the passthrough configuration.  We need to
 622          * use the real PF# in all our communication with the firmware.
 623          */
 624         sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 625         sc->mbox = sc->pf;
 626
 627         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 628         sc->an_handler = an_not_handled;
 629         for (i = 0; i < nitems(sc->cpl_handler); i++)
 630                 sc->cpl_handler[i] = cpl_not_handled;
 631         for (i = 0; i < nitems(sc->fw_msg_handler); i++)
 632                 sc->fw_msg_handler[i] = fw_msg_not_handled;
 633         t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 634         t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
 635         t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt);
 636         t4_init_sge_cpl_handlers(sc);
 637
 638         /* Prepare the adapter for operation */
 639         rc = -t4_prep_adapter(sc);
 640         if (rc != 0) {
 641                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 642                 goto done;
 643         }
 644
 645         /*
 646          * Do this really early, with the memory windows set up even before the
 647          * character device.  The userland tool's register i/o and mem read
 648          * will work even in "recovery mode".
 649          */
 650         setup_memwin(sc);
 651         sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
 652             device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
 653             device_get_nameunit(dev));
 654         if (sc->cdev == NULL)
 655                 device_printf(dev, "failed to create nexus char device.\n");
 656         else
 657                 sc->cdev->si_drv1 = sc;
 658
 659         /* Go no further if recovery mode has been requested. */
 660         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 661                 device_printf(dev, "recovery mode.\n");
 662                 goto done;
 663         }
 664
 665         /* Prepare the firmware for operation */
 666         rc = prep_firmware(sc);
 667         if (rc != 0)
 668                 goto done; /* error message displayed already */
 669
 670         rc = get_params__post_init(sc);
 671         if (rc != 0)
 672                 goto done; /* error message displayed already */
 673
 674         rc = set_params__post_init(sc);
 675         if (rc != 0)
 676                 goto done; /* error message displayed already */
 677
 678         rc = map_bar_2(sc);
 679         if (rc != 0)
 680                 goto done; /* error message displayed already */
 681
 682         rc = t4_create_dma_tag(sc);
 683         if (rc != 0)
 684                 goto done; /* error message displayed already */
 685
 686         /*
 687          * First pass over all the ports - allocate VIs and initialize some
 688          * basic parameters like mac address, port type, etc.  We also figure
 689          * out whether a port is 10G or 1G and use that information when
 690          * calculating how many interrupts to attempt to allocate.
 691          */
 692         n10g = n1g = 0;
 693         for_each_port(sc, i) {
 694                 struct port_info *pi;
 695
 696                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 697                 sc->port[i] = pi;
 698
 699                 /* These must be set before t4_port_init */
 700                 pi->adapter = sc;
 701                 pi->port_id = i;
 702
 703                 /* Allocate the vi and initialize parameters like mac addr */
 704                 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 705                 if (rc != 0) {
 706                         device_printf(dev, "unable to initialize port %d: %d\n",
 707                             i, rc);
 708                         free(pi, M_CXGBE);
 709                         sc->port[i] = NULL;
 710                         goto done;
 711                 }
 712
 713                 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX);
 714                 pi->link_cfg.requested_fc |= t4_pause_settings;
 715                 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX);
 716                 pi->link_cfg.fc |= t4_pause_settings;
 717
 718                 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
 719                 if (rc != 0) {
 720                         device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
 721                         free(pi, M_CXGBE);
 722                         sc->port[i] = NULL;
 723                         goto done;
 724                 }
 725
 726                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 727                     device_get_nameunit(dev), i);
 728                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 729                 sc->chan_map[pi->tx_chan] = i;
 730
 731                 if (is_10G_port(pi) || is_40G_port(pi)) {
 732                         n10g++;
 733                         pi->tmr_idx = t4_tmr_idx_10g;
 734                         pi->pktc_idx = t4_pktc_idx_10g;
 735                 } else {
 736                         n1g++;
 737                         pi->tmr_idx = t4_tmr_idx_1g;
 738                         pi->pktc_idx = t4_pktc_idx_1g;
 739                 }
 740
 741                 pi->xact_addr_filt = -1;
 742                 pi->linkdnrc = -1;
 743
 744                 pi->qsize_rxq = t4_qsize_rxq;
 745                 pi->qsize_txq = t4_qsize_txq;
 746
 747                 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
 748                 if (pi->dev == NULL) {
 749                         device_printf(dev,
 750                             "failed to add device for port %d.\n", i);
 751                         rc = ENXIO;
 752                         goto done;
 753                 }
 754                 device_set_softc(pi->dev, pi);
 755         }
 756
 757         /*
 758          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 759          */
 760         rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
 761         if (rc != 0)
 762                 goto done; /* error message displayed already */
 763
 764         sc->intr_type = iaq.intr_type;
 765         sc->intr_count = iaq.nirq;
 766
 767         s = &sc->sge;
 768         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 769         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 770         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
 771         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 772         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
 773 #ifdef TCP_OFFLOAD
 774         if (is_offload(sc)) {
 775                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 776                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 777                 s->neq += s->nofldtxq + s->nofldrxq;
 778                 s->niq += s->nofldrxq;
 779
 780                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 781                     M_CXGBE, M_ZERO | M_WAITOK);
 782                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 783                     M_CXGBE, M_ZERO | M_WAITOK);
 784         }
 785 #endif
 786 #ifdef DEV_NETMAP
 787         s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g;
 788         s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g;
 789         s->neq += s->nnmtxq + s->nnmrxq;
 790         s->niq += s->nnmrxq;
 791
 792         s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
 793             M_CXGBE, M_ZERO | M_WAITOK);
 794         s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
 795             M_CXGBE, M_ZERO | M_WAITOK);
 796 #endif
 797
 798         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 799             M_ZERO | M_WAITOK);
 800         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 801             M_ZERO | M_WAITOK);
 802         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 803             M_ZERO | M_WAITOK);
 804         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 805             M_ZERO | M_WAITOK);
 806         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 807             M_ZERO | M_WAITOK);
 808
 809         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 810             M_ZERO | M_WAITOK);
 811
 812         t4_init_l2t(sc, M_WAITOK);
 813
 814         /*
 815          * Second pass over the ports.  This time we know the number of rx and
 816          * tx queues that each port should get.
 817          */
 818         rqidx = tqidx = 0;
 819 #ifdef TCP_OFFLOAD
 820         ofld_rqidx = ofld_tqidx = 0;
 821 #endif
 822 #ifdef DEV_NETMAP
 823         nm_rqidx = nm_tqidx = 0;
 824 #endif
 825         for_each_port(sc, i) {
 826                 struct port_info *pi = sc->port[i];
 827
 828                 if (pi == NULL)
 829                         continue;
 830
 831                 pi->first_rxq = rqidx;
 832                 pi->first_txq = tqidx;
 833                 if (is_10G_port(pi) || is_40G_port(pi)) {
 834                         pi->flags |= iaq.intr_flags_10g;
 835                         pi->nrxq = iaq.nrxq10g;
 836                         pi->ntxq = iaq.ntxq10g;
 837                 } else {
 838                         pi->flags |= iaq.intr_flags_1g;
 839                         pi->nrxq = iaq.nrxq1g;
 840                         pi->ntxq = iaq.ntxq1g;
 841                 }
 842
 843                 if (pi->ntxq > 1)
 844                         pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
 845                 else
 846                         pi->rsrv_noflowq = 0;
 847
 848                 rqidx += pi->nrxq;
 849                 tqidx += pi->ntxq;
 850 #ifdef TCP_OFFLOAD
 851                 if (is_offload(sc)) {
 852                         pi->first_ofld_rxq = ofld_rqidx;
 853                         pi->first_ofld_txq = ofld_tqidx;
 854                         if (is_10G_port(pi) || is_40G_port(pi)) {
 855                                 pi->nofldrxq = iaq.nofldrxq10g;
 856                                 pi->nofldtxq = iaq.nofldtxq10g;
 857                         } else {
 858                                 pi->nofldrxq = iaq.nofldrxq1g;
 859                                 pi->nofldtxq = iaq.nofldtxq1g;
 860                         }
 861                         ofld_rqidx += pi->nofldrxq;
 862                         ofld_tqidx += pi->nofldtxq;
 863                 }
 864 #endif
 865 #ifdef DEV_NETMAP
 866                 pi->first_nm_rxq = nm_rqidx;
 867                 pi->first_nm_txq = nm_tqidx;
 868                 if (is_10G_port(pi) || is_40G_port(pi)) {
 869                         pi->nnmrxq = iaq.nnmrxq10g;
 870                         pi->nnmtxq = iaq.nnmtxq10g;
 871                 } else {
 872                         pi->nnmrxq = iaq.nnmrxq1g;
 873                         pi->nnmtxq = iaq.nnmtxq1g;
 874                 }
 875                 nm_rqidx += pi->nnmrxq;
 876                 nm_tqidx += pi->nnmtxq;
 877 #endif
 878         }
 879
 880         rc = setup_intr_handlers(sc);
 881         if (rc != 0) {
 882                 device_printf(dev,
 883                     "failed to setup interrupt handlers: %d\n", rc);
 884                 goto done;
 885         }
 886
 887         rc = bus_generic_attach(dev);
 888         if (rc != 0) {
 889                 device_printf(dev,
 890                     "failed to attach all child ports: %d\n", rc);
 891                 goto done;
 892         }
 893
 894         device_printf(dev,
 895             "PCIe x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
 896             sc->params.pci.width, sc->params.nports, sc->intr_count,
 897             sc->intr_type == INTR_MSIX ? "MSI-X" :
 898             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
 899             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
 900
 901         t4_set_desc(sc);
 902
 903 done:
 904         if (rc != 0 && sc->cdev) {
 905                 /* cdev was created and so cxgbetool works; recover that way. */
 906                 device_printf(dev,
 907                     "error during attach, adapter is now in recovery mode.\n");
 908                 rc = 0;
 909         }
 910
 911         if (rc != 0)
 912                 t4_detach(dev);
 913         else
 914                 t4_sysctls(sc);
 915
 916         return (rc);
 917 }
 918
 919 /*
 920  * Idempotent
 921  */
 922 static int
 923 t4_detach(device_t dev)
 924 {
 925         struct adapter *sc;
 926         struct port_info *pi;
 927         int i, rc;
 928
 929         sc = device_get_softc(dev);
 930
 931         if (sc->flags & FULL_INIT_DONE)
 932                 t4_intr_disable(sc);
 933
 934         if (sc->cdev) {
 935                 destroy_dev(sc->cdev);
 936                 sc->cdev = NULL;
 937         }
 938
 939         rc = bus_generic_detach(dev);
 940         if (rc) {
 941                 device_printf(dev,
 942                     "failed to detach child devices: %d\n", rc);
 943                 return (rc);
 944         }
 945
 946         for (i = 0; i < sc->intr_count; i++)
 947                 t4_free_irq(sc, &sc->irq[i]);
 948
 949         for (i = 0; i < MAX_NPORTS; i++) {
 950                 pi = sc->port[i];
 951                 if (pi) {
 952                         t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid);
 953                         if (pi->dev)
 954                                 device_delete_child(dev, pi->dev);
 955
 956                         mtx_destroy(&pi->pi_lock);
 957                         free(pi, M_CXGBE);
 958                 }
 959         }
 960
 961         if (sc->flags & FULL_INIT_DONE)
 962                 adapter_full_uninit(sc);
 963
 964         if (sc->flags & FW_OK)
 965                 t4_fw_bye(sc, sc->mbox);
 966
 967         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
 968                 pci_release_msi(dev);
 969
 970         if (sc->regs_res)
 971                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
 972                     sc->regs_res);
 973
 974         if (sc->udbs_res)
 975                 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
 976                     sc->udbs_res);
 977
 978         if (sc->msix_res)
 979                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
 980                     sc->msix_res);
 981
 982         if (sc->l2t)
 983                 t4_free_l2t(sc->l2t);
 984
 985 #ifdef TCP_OFFLOAD
 986         free(sc->sge.ofld_rxq, M_CXGBE);
 987         free(sc->sge.ofld_txq, M_CXGBE);
 988 #endif
 989 #ifdef DEV_NETMAP
 990         free(sc->sge.nm_rxq, M_CXGBE);
 991         free(sc->sge.nm_txq, M_CXGBE);
 992 #endif
 993         free(sc->irq, M_CXGBE);
 994         free(sc->sge.rxq, M_CXGBE);
 995         free(sc->sge.txq, M_CXGBE);
 996         free(sc->sge.ctrlq, M_CXGBE);
 997         free(sc->sge.iqmap, M_CXGBE);
 998         free(sc->sge.eqmap, M_CXGBE);
 999         free(sc->tids.ftid_tab, M_CXGBE);
1000         t4_destroy_dma_tag(sc);
1001         if (mtx_initialized(&sc->sc_lock)) {
1002                 sx_xlock(&t4_list_lock);
1003                 SLIST_REMOVE(&t4_list, sc, adapter, link);
1004                 sx_xunlock(&t4_list_lock);
1005                 mtx_destroy(&sc->sc_lock);
1006         }
1007
1008         if (mtx_initialized(&sc->tids.ftid_lock))
1009                 mtx_destroy(&sc->tids.ftid_lock);
1010         if (mtx_initialized(&sc->sfl_lock))
1011                 mtx_destroy(&sc->sfl_lock);
1012         if (mtx_initialized(&sc->ifp_lock))
1013                 mtx_destroy(&sc->ifp_lock);
1014
1015         bzero(sc, sizeof(*sc));
1016
1017         return (0);
1018 }
1019
1020 static int
1021 cxgbe_probe(device_t dev)
1022 {
1023         char buf[128];
1024         struct port_info *pi = device_get_softc(dev);
1025
1026         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
1027         device_set_desc_copy(dev, buf);
1028
1029         return (BUS_PROBE_DEFAULT);
1030 }
1031
1032 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
1033     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1034     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
1035 #define T4_CAP_ENABLE (T4_CAP)
1036
1037 static int
1038 cxgbe_attach(device_t dev)
1039 {
1040         struct port_info *pi = device_get_softc(dev);
1041         struct ifnet *ifp;
1042         char *s;
1043         int n, o;
1044
1045         /* Allocate an ifnet and set it up */
1046         ifp = if_alloc(IFT_ETHER);
1047         if (ifp == NULL) {
1048                 device_printf(dev, "Cannot allocate ifnet\n");
1049                 return (ENOMEM);
1050         }
1051         pi->ifp = ifp;
1052         ifp->if_softc = pi;
1053
1054         callout_init(&pi->tick, CALLOUT_MPSAFE);
1055
1056         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1057         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1058
1059         ifp->if_init = cxgbe_init;
1060         ifp->if_ioctl = cxgbe_ioctl;
1061         ifp->if_transmit = cxgbe_transmit;
1062         ifp->if_qflush = cxgbe_qflush;
1063
1064         ifp->if_capabilities = T4_CAP;
1065 #ifdef TCP_OFFLOAD
1066         if (is_offload(pi->adapter))
1067                 ifp->if_capabilities |= IFCAP_TOE;
1068 #endif
1069         ifp->if_capenable = T4_CAP_ENABLE;
1070         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1071             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1072
1073         ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1074         ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS;
1075         ifp->if_hw_tsomaxsegsize = 65536;
1076
1077         /* Initialize ifmedia for this port */
1078         ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1079             cxgbe_media_status);
1080         build_medialist(pi, &pi->media);
1081
1082         pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1083             EVENTHANDLER_PRI_ANY);
1084
1085         ether_ifattach(ifp, pi->hw_addr);
1086
1087         n = 128;
1088         s = malloc(n, M_CXGBE, M_WAITOK);
1089         o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq);
1090         MPASS(n > o);
1091 #ifdef TCP_OFFLOAD
1092         if (is_offload(pi->adapter)) {
1093                 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)",
1094                     pi->nofldtxq, pi->nofldrxq);
1095                 MPASS(n > o);
1096         }
1097 #endif
1098 #ifdef DEV_NETMAP
1099         o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq,
1100             pi->nnmrxq);
1101         MPASS(n > o);
1102 #endif
1103         device_printf(dev, "%s\n", s);
1104         free(s, M_CXGBE);
1105
1106 #ifdef DEV_NETMAP
1107         /* nm_media handled here to keep implementation private to this file */
1108         ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change,
1109             cxgbe_media_status);
1110         build_medialist(pi, &pi->nm_media);
1111         create_netmap_ifnet(pi);        /* logs errors it something fails */
1112 #endif
1113         cxgbe_sysctls(pi);
1114
1115         return (0);
1116 }
1117
1118 static int
1119 cxgbe_detach(device_t dev)
1120 {
1121         struct port_info *pi = device_get_softc(dev);
1122         struct adapter *sc = pi->adapter;
1123         struct ifnet *ifp = pi->ifp;
1124
1125         /* Tell if_ioctl and if_init that the port is going away */
1126         ADAPTER_LOCK(sc);
1127         SET_DOOMED(pi);
1128         wakeup(&sc->flags);
1129         while (IS_BUSY(sc))
1130                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
1131         SET_BUSY(sc);
1132 #ifdef INVARIANTS
1133         sc->last_op = "t4detach";
1134         sc->last_op_thr = curthread;
1135 #endif
1136         ADAPTER_UNLOCK(sc);
1137
1138         if (pi->flags & HAS_TRACEQ) {
1139                 sc->traceq = -1;        /* cloner should not create ifnet */
1140                 t4_tracer_port_detach(sc);
1141         }
1142
1143         if (pi->vlan_c)
1144                 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
1145
1146         PORT_LOCK(pi);
1147         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1148         callout_stop(&pi->tick);
1149         PORT_UNLOCK(pi);
1150         callout_drain(&pi->tick);
1151
1152         /* Let detach proceed even if these fail. */
1153         cxgbe_uninit_synchronized(pi);
1154         port_full_uninit(pi);
1155
1156         ifmedia_removeall(&pi->media);
1157         ether_ifdetach(pi->ifp);
1158         if_free(pi->ifp);
1159
1160 #ifdef DEV_NETMAP
1161         /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */
1162         destroy_netmap_ifnet(pi);
1163 #endif
1164
1165         ADAPTER_LOCK(sc);
1166         CLR_BUSY(sc);
1167         wakeup(&sc->flags);
1168         ADAPTER_UNLOCK(sc);
1169
1170         return (0);
1171 }
1172
1173 static void
1174 cxgbe_init(void *arg)
1175 {
1176         struct port_info *pi = arg;
1177         struct adapter *sc = pi->adapter;
1178
1179         if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
1180                 return;
1181         cxgbe_init_synchronized(pi);
1182         end_synchronized_op(sc, 0);
1183 }
1184
1185 static int
1186 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1187 {
1188         int rc = 0, mtu, flags, can_sleep;
1189         struct port_info *pi = ifp->if_softc;
1190         struct adapter *sc = pi->adapter;
1191         struct ifreq *ifr = (struct ifreq *)data;
1192         uint32_t mask;
1193
1194         switch (cmd) {
1195         case SIOCSIFMTU:
1196                 mtu = ifr->ifr_mtu;
1197                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1198                         return (EINVAL);
1199
1200                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
1201                 if (rc)
1202                         return (rc);
1203                 ifp->if_mtu = mtu;
1204                 if (pi->flags & PORT_INIT_DONE) {
1205                         t4_update_fl_bufsize(ifp);
1206                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1207                                 rc = update_mac_settings(ifp, XGMAC_MTU);
1208                 }
1209                 end_synchronized_op(sc, 0);
1210                 break;
1211
1212         case SIOCSIFFLAGS:
1213                 can_sleep = 0;
1214 redo_sifflags:
1215                 rc = begin_synchronized_op(sc, pi,
1216                     can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg");
1217                 if (rc)
1218                         return (rc);
1219
1220                 if (ifp->if_flags & IFF_UP) {
1221                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1222                                 flags = pi->if_flags;
1223                                 if ((ifp->if_flags ^ flags) &
1224                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1225                                         if (can_sleep == 1) {
1226                                                 end_synchronized_op(sc, 0);
1227                                                 can_sleep = 0;
1228                                                 goto redo_sifflags;
1229                                         }
1230                                         rc = update_mac_settings(ifp,
1231                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1232                                 }
1233                         } else {
1234                                 if (can_sleep == 0) {
1235                                         end_synchronized_op(sc, LOCK_HELD);
1236                                         can_sleep = 1;
1237                                         goto redo_sifflags;
1238                                 }
1239                                 rc = cxgbe_init_synchronized(pi);
1240                         }
1241                         pi->if_flags = ifp->if_flags;
1242                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1243                         if (can_sleep == 0) {
1244                                 end_synchronized_op(sc, LOCK_HELD);
1245                                 can_sleep = 1;
1246                                 goto redo_sifflags;
1247                         }
1248                         rc = cxgbe_uninit_synchronized(pi);
1249                 }
1250                 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD);
1251                 break;
1252
1253         case SIOCADDMULTI:
1254         case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1255                 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
1256                 if (rc)
1257                         return (rc);
1258                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1259                         rc = update_mac_settings(ifp, XGMAC_MCADDRS);
1260                 end_synchronized_op(sc, LOCK_HELD);
1261                 break;
1262
1263         case SIOCSIFCAP:
1264                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
1265                 if (rc)
1266                         return (rc);
1267
1268                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1269                 if (mask & IFCAP_TXCSUM) {
1270                         ifp->if_capenable ^= IFCAP_TXCSUM;
1271                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1272
1273                         if (IFCAP_TSO4 & ifp->if_capenable &&
1274                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1275                                 ifp->if_capenable &= ~IFCAP_TSO4;
1276                                 if_printf(ifp,
1277                                     "tso4 disabled due to -txcsum.\n");
1278                         }
1279                 }
1280                 if (mask & IFCAP_TXCSUM_IPV6) {
1281                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1282                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1283
1284                         if (IFCAP_TSO6 & ifp->if_capenable &&
1285                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1286                                 ifp->if_capenable &= ~IFCAP_TSO6;
1287                                 if_printf(ifp,
1288                                     "tso6 disabled due to -txcsum6.\n");
1289                         }
1290                 }
1291                 if (mask & IFCAP_RXCSUM)
1292                         ifp->if_capenable ^= IFCAP_RXCSUM;
1293                 if (mask & IFCAP_RXCSUM_IPV6)
1294                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1295
1296                 /*
1297                  * Note that we leave CSUM_TSO alone (it is always set).  The
1298                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1299                  * sending a TSO request our way, so it's sufficient to toggle
1300                  * IFCAP_TSOx only.
1301                  */
1302                 if (mask & IFCAP_TSO4) {
1303                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1304                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1305                                 if_printf(ifp, "enable txcsum first.\n");
1306                                 rc = EAGAIN;
1307                                 goto fail;
1308                         }
1309                         ifp->if_capenable ^= IFCAP_TSO4;
1310                 }
1311                 if (mask & IFCAP_TSO6) {
1312                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1313                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1314                                 if_printf(ifp, "enable txcsum6 first.\n");
1315                                 rc = EAGAIN;
1316                                 goto fail;
1317                         }
1318                         ifp->if_capenable ^= IFCAP_TSO6;
1319                 }
1320                 if (mask & IFCAP_LRO) {
1321 #if defined(INET) || defined(INET6)
1322                         int i;
1323                         struct sge_rxq *rxq;
1324
1325                         ifp->if_capenable ^= IFCAP_LRO;
1326                         for_each_rxq(pi, i, rxq) {
1327                                 if (ifp->if_capenable & IFCAP_LRO)
1328                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1329                                 else
1330                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1331                         }
1332 #endif
1333                 }
1334 #ifdef TCP_OFFLOAD
1335                 if (mask & IFCAP_TOE) {
1336                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1337
1338                         rc = toe_capability(pi, enable);
1339                         if (rc != 0)
1340                                 goto fail;
1341
1342                         ifp->if_capenable ^= mask;
1343                 }
1344 #endif
1345                 if (mask & IFCAP_VLAN_HWTAGGING) {
1346                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1347                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1348                                 rc = update_mac_settings(ifp, XGMAC_VLANEX);
1349                 }
1350                 if (mask & IFCAP_VLAN_MTU) {
1351                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1352
1353                         /* Need to find out how to disable auto-mtu-inflation */
1354                 }
1355                 if (mask & IFCAP_VLAN_HWTSO)
1356                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1357                 if (mask & IFCAP_VLAN_HWCSUM)
1358                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1359
1360 #ifdef VLAN_CAPABILITIES
1361                 VLAN_CAPABILITIES(ifp);
1362 #endif
1363 fail:
1364                 end_synchronized_op(sc, 0);
1365                 break;
1366
1367         case SIOCSIFMEDIA:
1368         case SIOCGIFMEDIA:
1369                 ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1370                 break;
1371
1372         default:
1373                 rc = ether_ioctl(ifp, cmd, data);
1374         }
1375
1376         return (rc);
1377 }
1378
1379 static int
1380 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1381 {
1382         struct port_info *pi = ifp->if_softc;
1383         struct adapter *sc = pi->adapter;
1384         struct sge_txq *txq = &sc->sge.txq[pi->first_txq];
1385         struct buf_ring *br;
1386         int rc;
1387
1388         M_ASSERTPKTHDR(m);
1389
1390         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1391                 m_freem(m);
1392                 return (ENETDOWN);
1393         }
1394
1395         if (m->m_flags & M_FLOWID)
1396                 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq))
1397                     + pi->rsrv_noflowq);
1398         br = txq->br;
1399
1400         if (TXQ_TRYLOCK(txq) == 0) {
1401                 struct sge_eq *eq = &txq->eq;
1402
1403                 /*
1404                  * It is possible that t4_eth_tx finishes up and releases the
1405                  * lock between the TRYLOCK above and the drbr_enqueue here.  We
1406                  * need to make sure that this mbuf doesn't just sit there in
1407                  * the drbr.
1408                  */
1409
1410                 rc = drbr_enqueue(ifp, br, m);
1411                 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 &&
1412                     !(eq->flags & EQ_DOOMED))
1413                         callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq);
1414                 return (rc);
1415         }
1416
1417         /*
1418          * txq->m is the mbuf that is held up due to a temporary shortage of
1419          * resources and it should be put on the wire first.  Then what's in
1420          * drbr and finally the mbuf that was just passed in to us.
1421          *
1422          * Return code should indicate the fate of the mbuf that was passed in
1423          * this time.
1424          */
1425
1426         TXQ_LOCK_ASSERT_OWNED(txq);
1427         if (drbr_needs_enqueue(ifp, br) || txq->m) {
1428
1429                 /* Queued for transmission. */
1430
1431                 rc = drbr_enqueue(ifp, br, m);
1432                 m = txq->m ? txq->m : drbr_dequeue(ifp, br);
1433                 (void) t4_eth_tx(ifp, txq, m);
1434                 TXQ_UNLOCK(txq);
1435                 return (rc);
1436         }
1437
1438         /* Direct transmission. */
1439         rc = t4_eth_tx(ifp, txq, m);
1440         if (rc != 0 && txq->m)
1441                 rc = 0; /* held, will be transmitted soon (hopefully) */
1442
1443         TXQ_UNLOCK(txq);
1444         return (rc);
1445 }
1446
1447 static void
1448 cxgbe_qflush(struct ifnet *ifp)
1449 {
1450         struct port_info *pi = ifp->if_softc;
1451         struct sge_txq *txq;
1452         int i;
1453         struct mbuf *m;
1454
1455         /* queues do not exist if !PORT_INIT_DONE. */
1456         if (pi->flags & PORT_INIT_DONE) {
1457                 for_each_txq(pi, i, txq) {
1458                         TXQ_LOCK(txq);
1459                         m_freem(txq->m);
1460                         txq->m = NULL;
1461                         while ((m = buf_ring_dequeue_sc(txq->br)) != NULL)
1462                                 m_freem(m);
1463                         TXQ_UNLOCK(txq);
1464                 }
1465         }
1466         if_qflush(ifp);
1467 }
1468
1469 static int
1470 cxgbe_media_change(struct ifnet *ifp)
1471 {
1472         struct port_info *pi = ifp->if_softc;
1473
1474         device_printf(pi->dev, "%s unimplemented.\n", __func__);
1475
1476         return (EOPNOTSUPP);
1477 }
1478
1479 static void
1480 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1481 {
1482         struct port_info *pi = ifp->if_softc;
1483         struct ifmedia *media = NULL;
1484         struct ifmedia_entry *cur;
1485         int speed = pi->link_cfg.speed;
1486 #ifdef INVARIANTS
1487         int data = (pi->port_type << 8) | pi->mod_type;
1488 #endif
1489
1490         if (ifp == pi->ifp)
1491                 media = &pi->media;
1492 #ifdef DEV_NETMAP
1493         else if (ifp == pi->nm_ifp)
1494                 media = &pi->nm_media;
1495 #endif
1496         MPASS(media != NULL);
1497
1498         cur = media->ifm_cur;
1499         MPASS(cur->ifm_data == data);
1500
1501         ifmr->ifm_status = IFM_AVALID;
1502         if (!pi->link_cfg.link_ok)
1503                 return;
1504
1505         ifmr->ifm_status |= IFM_ACTIVE;
1506
1507         /* active and current will differ iff current media is autoselect. */
1508         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1509                 return;
1510
1511         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1512         if (speed == SPEED_10000)
1513                 ifmr->ifm_active |= IFM_10G_T;
1514         else if (speed == SPEED_1000)
1515                 ifmr->ifm_active |= IFM_1000_T;
1516         else if (speed == SPEED_100)
1517                 ifmr->ifm_active |= IFM_100_TX;
1518         else if (speed == SPEED_10)
1519                 ifmr->ifm_active |= IFM_10_T;
1520         else
1521                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1522                             speed));
1523 }
1524
1525 void
1526 t4_fatal_err(struct adapter *sc)
1527 {
1528         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1529         t4_intr_disable(sc);
1530         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1531             device_get_nameunit(sc->dev));
1532 }
1533
1534 static int
1535 map_bars_0_and_4(struct adapter *sc)
1536 {
1537         sc->regs_rid = PCIR_BAR(0);
1538         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1539             &sc->regs_rid, RF_ACTIVE);
1540         if (sc->regs_res == NULL) {
1541                 device_printf(sc->dev, "cannot map registers.\n");
1542                 return (ENXIO);
1543         }
1544         sc->bt = rman_get_bustag(sc->regs_res);
1545         sc->bh = rman_get_bushandle(sc->regs_res);
1546         sc->mmio_len = rman_get_size(sc->regs_res);
1547         setbit(&sc->doorbells, DOORBELL_KDB);
1548
1549         sc->msix_rid = PCIR_BAR(4);
1550         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1551             &sc->msix_rid, RF_ACTIVE);
1552         if (sc->msix_res == NULL) {
1553                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1554                 return (ENXIO);
1555         }
1556
1557         return (0);
1558 }
1559
1560 static int
1561 map_bar_2(struct adapter *sc)
1562 {
1563
1564         /*
1565          * T4: only iWARP driver uses the userspace doorbells.  There is no need
1566          * to map it if RDMA is disabled.
1567          */
1568         if (is_t4(sc) && sc->rdmacaps == 0)
1569                 return (0);
1570
1571         sc->udbs_rid = PCIR_BAR(2);
1572         sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1573             &sc->udbs_rid, RF_ACTIVE);
1574         if (sc->udbs_res == NULL) {
1575                 device_printf(sc->dev, "cannot map doorbell BAR.\n");
1576                 return (ENXIO);
1577         }
1578         sc->udbs_base = rman_get_virtual(sc->udbs_res);
1579
1580         if (is_t5(sc)) {
1581                 setbit(&sc->doorbells, DOORBELL_UDB);
1582 #if defined(__i386__) || defined(__amd64__)
1583                 if (t5_write_combine) {
1584                         int rc;
1585
1586                         /*
1587                          * Enable write combining on BAR2.  This is the
1588                          * userspace doorbell BAR and is split into 128B
1589                          * (UDBS_SEG_SIZE) doorbell regions, each associated
1590                          * with an egress queue.  The first 64B has the doorbell
1591                          * and the second 64B can be used to submit a tx work
1592                          * request with an implicit doorbell.
1593                          */
1594
1595                         rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
1596                             rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
1597                         if (rc == 0) {
1598                                 clrbit(&sc->doorbells, DOORBELL_UDB);
1599                                 setbit(&sc->doorbells, DOORBELL_WCWR);
1600                                 setbit(&sc->doorbells, DOORBELL_UDBWC);
1601                         } else {
1602                                 device_printf(sc->dev,
1603                                     "couldn't enable write combining: %d\n",
1604                                     rc);
1605                         }
1606
1607                         t4_write_reg(sc, A_SGE_STAT_CFG,
1608                             V_STATSOURCE_T5(7) | V_STATMODE(0));
1609                 }
1610 #endif
1611         }
1612
1613         return (0);
1614 }
1615
1616 static const struct memwin t4_memwin[] = {
1617         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1618         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1619         { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
1620 };
1621
1622 static const struct memwin t5_memwin[] = {
1623         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1624         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1625         { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1626 };
1627
1628 static void
1629 setup_memwin(struct adapter *sc)
1630 {
1631         const struct memwin *mw;
1632         int i, n;
1633         uint32_t bar0;
1634
1635         if (is_t4(sc)) {
1636                 /*
1637                  * Read low 32b of bar0 indirectly via the hardware backdoor
1638                  * mechanism.  Works from within PCI passthrough environments
1639                  * too, where rman_get_start() can return a different value.  We
1640                  * need to program the T4 memory window decoders with the actual
1641                  * addresses that will be coming across the PCIe link.
1642                  */
1643                 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1644                 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1645
1646                 mw = &t4_memwin[0];
1647                 n = nitems(t4_memwin);
1648         } else {
1649                 /* T5 uses the relative offset inside the PCIe BAR */
1650                 bar0 = 0;
1651
1652                 mw = &t5_memwin[0];
1653                 n = nitems(t5_memwin);
1654         }
1655
1656         for (i = 0; i < n; i++, mw++) {
1657                 t4_write_reg(sc,
1658                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
1659                     (mw->base + bar0) | V_BIR(0) |
1660                     V_WINDOW(ilog2(mw->aperture) - 10));
1661         }
1662
1663         /* flush */
1664         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1665 }
1666
1667 /*
1668  * Verify that the memory range specified by the addr/len pair is valid and lies
1669  * entirely within a single region (EDCx or MCx).
1670  */
1671 static int
1672 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
1673 {
1674         uint32_t em, addr_len, maddr, mlen;
1675
1676         /* Memory can only be accessed in naturally aligned 4 byte units */
1677         if (addr & 3 || len & 3 || len == 0)
1678                 return (EINVAL);
1679
1680         /* Enabled memories */
1681         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1682         if (em & F_EDRAM0_ENABLE) {
1683                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1684                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1685                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1686                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1687                     addr + len <= maddr + mlen)
1688                         return (0);
1689         }
1690         if (em & F_EDRAM1_ENABLE) {
1691                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1692                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1693                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1694                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1695                     addr + len <= maddr + mlen)
1696                         return (0);
1697         }
1698         if (em & F_EXT_MEM_ENABLE) {
1699                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1700                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1701                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1702                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1703                     addr + len <= maddr + mlen)
1704                         return (0);
1705         }
1706         if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
1707                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1708                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1709                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1710                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1711                     addr + len <= maddr + mlen)
1712                         return (0);
1713         }
1714
1715         return (EFAULT);
1716 }
1717
1718 static int
1719 fwmtype_to_hwmtype(int mtype)
1720 {
1721
1722         switch (mtype) {
1723         case FW_MEMTYPE_EDC0:
1724                 return (MEM_EDC0);
1725         case FW_MEMTYPE_EDC1:
1726                 return (MEM_EDC1);
1727         case FW_MEMTYPE_EXTMEM:
1728                 return (MEM_MC0);
1729         case FW_MEMTYPE_EXTMEM1:
1730                 return (MEM_MC1);
1731         default:
1732                 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
1733         }
1734 }
1735
1736 /*
1737  * Verify that the memory range specified by the memtype/offset/len pair is
1738  * valid and lies entirely within the memtype specified.  The global address of
1739  * the start of the range is returned in addr.
1740  */
1741 static int
1742 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1743     uint32_t *addr)
1744 {
1745         uint32_t em, addr_len, maddr, mlen;
1746
1747         /* Memory can only be accessed in naturally aligned 4 byte units */
1748         if (off & 3 || len & 3 || len == 0)
1749                 return (EINVAL);
1750
1751         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1752         switch (fwmtype_to_hwmtype(mtype)) {
1753         case MEM_EDC0:
1754                 if (!(em & F_EDRAM0_ENABLE))
1755                         return (EINVAL);
1756                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1757                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1758                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1759                 break;
1760         case MEM_EDC1:
1761                 if (!(em & F_EDRAM1_ENABLE))
1762                         return (EINVAL);
1763                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1764                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1765                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1766                 break;
1767         case MEM_MC:
1768                 if (!(em & F_EXT_MEM_ENABLE))
1769                         return (EINVAL);
1770                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1771                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1772                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1773                 break;
1774         case MEM_MC1:
1775                 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
1776                         return (EINVAL);
1777                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1778                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1779                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1780                 break;
1781         default:
1782                 return (EINVAL);
1783         }
1784
1785         if (mlen > 0 && off < mlen && off + len <= mlen) {
1786                 *addr = maddr + off;    /* global address */
1787                 return (0);
1788         }
1789
1790         return (EFAULT);
1791 }
1792
1793 static void
1794 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1795 {
1796         const struct memwin *mw;
1797
1798         if (is_t4(sc)) {
1799                 KASSERT(win >= 0 && win < nitems(t4_memwin),
1800                     ("%s: incorrect memwin# (%d)", __func__, win));
1801                 mw = &t4_memwin[win];
1802         } else {
1803                 KASSERT(win >= 0 && win < nitems(t5_memwin),
1804                     ("%s: incorrect memwin# (%d)", __func__, win));
1805                 mw = &t5_memwin[win];
1806         }
1807
1808         if (base != NULL)
1809                 *base = mw->base;
1810         if (aperture != NULL)
1811                 *aperture = mw->aperture;
1812 }
1813
1814 /*
1815  * Positions the memory window such that it can be used to access the specified
1816  * address in the chip's address space.  The return value is the offset of addr
1817  * from the start of the window.
1818  */
1819 static uint32_t
1820 position_memwin(struct adapter *sc, int n, uint32_t addr)
1821 {
1822         uint32_t start, pf;
1823         uint32_t reg;
1824
1825         KASSERT(n >= 0 && n <= 3,
1826             ("%s: invalid window %d.", __func__, n));
1827         KASSERT((addr & 3) == 0,
1828             ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
1829
1830         if (is_t4(sc)) {
1831                 pf = 0;
1832                 start = addr & ~0xf;    /* start must be 16B aligned */
1833         } else {
1834                 pf = V_PFNUM(sc->pf);
1835                 start = addr & ~0x7f;   /* start must be 128B aligned */
1836         }
1837         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1838
1839         t4_write_reg(sc, reg, start | pf);
1840         t4_read_reg(sc, reg);
1841
1842         return (addr - start);
1843 }
1844
1845 static int
1846 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1847     struct intrs_and_queues *iaq)
1848 {
1849         int rc, itype, navail, nrxq10g, nrxq1g, n;
1850         int nofldrxq10g = 0, nofldrxq1g = 0;
1851         int nnmrxq10g = 0, nnmrxq1g = 0;
1852
1853         bzero(iaq, sizeof(*iaq));
1854
1855         iaq->ntxq10g = t4_ntxq10g;
1856         iaq->ntxq1g = t4_ntxq1g;
1857         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1858         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1859         iaq->rsrv_noflowq = t4_rsrv_noflowq;
1860 #ifdef TCP_OFFLOAD
1861         if (is_offload(sc)) {
1862                 iaq->nofldtxq10g = t4_nofldtxq10g;
1863                 iaq->nofldtxq1g = t4_nofldtxq1g;
1864                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1865                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1866         }
1867 #endif
1868 #ifdef DEV_NETMAP
1869         iaq->nnmtxq10g = t4_nnmtxq10g;
1870         iaq->nnmtxq1g = t4_nnmtxq1g;
1871         iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g;
1872         iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g;
1873 #endif
1874
1875         for (itype = INTR_MSIX; itype; itype >>= 1) {
1876
1877                 if ((itype & t4_intr_types) == 0)
1878                         continue;       /* not allowed */
1879
1880                 if (itype == INTR_MSIX)
1881                         navail = pci_msix_count(sc->dev);
1882                 else if (itype == INTR_MSI)
1883                         navail = pci_msi_count(sc->dev);
1884                 else
1885                         navail = 1;
1886 restart:
1887                 if (navail == 0)
1888                         continue;
1889
1890                 iaq->intr_type = itype;
1891                 iaq->intr_flags_10g = 0;
1892                 iaq->intr_flags_1g = 0;
1893
1894                 /*
1895                  * Best option: an interrupt vector for errors, one for the
1896                  * firmware event queue, and one for every rxq (NIC, TOE, and
1897                  * netmap).
1898                  */
1899                 iaq->nirq = T4_EXTRA_INTR;
1900                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g);
1901                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g);
1902                 if (iaq->nirq <= navail &&
1903                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
1904                         iaq->intr_flags_10g = INTR_ALL;
1905                         iaq->intr_flags_1g = INTR_ALL;
1906                         goto allocate;
1907                 }
1908
1909                 /*
1910                  * Second best option: a vector for errors, one for the firmware
1911                  * event queue, and vectors for either all the NIC rx queues or
1912                  * all the TOE rx queues.  The queues that don't get vectors
1913                  * will forward their interrupts to those that do.
1914                  *
1915                  * Note: netmap rx queues cannot be created early and so they
1916                  * can't be setup to receive forwarded interrupts for others.
1917                  */
1918                 iaq->nirq = T4_EXTRA_INTR;
1919                 if (nrxq10g >= nofldrxq10g) {
1920                         iaq->intr_flags_10g = INTR_RXQ;
1921                         iaq->nirq += n10g * nrxq10g;
1922 #ifdef DEV_NETMAP
1923                         iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g);
1924 #endif
1925                 } else {
1926                         iaq->intr_flags_10g = INTR_OFLD_RXQ;
1927                         iaq->nirq += n10g * nofldrxq10g;
1928 #ifdef DEV_NETMAP
1929                         iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g);
1930 #endif
1931                 }
1932                 if (nrxq1g >= nofldrxq1g) {
1933                         iaq->intr_flags_1g = INTR_RXQ;
1934                         iaq->nirq += n1g * nrxq1g;
1935 #ifdef DEV_NETMAP
1936                         iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g);
1937 #endif
1938                 } else {
1939                         iaq->intr_flags_1g = INTR_OFLD_RXQ;
1940                         iaq->nirq += n1g * nofldrxq1g;
1941 #ifdef DEV_NETMAP
1942                         iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g);
1943 #endif
1944                 }
1945                 if (iaq->nirq <= navail &&
1946                     (itype != INTR_MSI || powerof2(iaq->nirq)))
1947                         goto allocate;
1948
1949                 /*
1950                  * Next best option: an interrupt vector for errors, one for the
1951                  * firmware event queue, and at least one per port.  At this
1952                  * point we know we'll have to downsize nrxq and/or nofldrxq
1953                  * and/or nnmrxq to fit what's available to us.
1954                  */
1955                 iaq->nirq = T4_EXTRA_INTR;
1956                 iaq->nirq += n10g + n1g;
1957                 if (iaq->nirq <= navail) {
1958                         int leftover = navail - iaq->nirq;
1959
1960                         if (n10g > 0) {
1961                                 int target = max(nrxq10g, nofldrxq10g);
1962
1963                                 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
1964                                     INTR_RXQ : INTR_OFLD_RXQ;
1965
1966                                 n = 1;
1967                                 while (n < target && leftover >= n10g) {
1968                                         leftover -= n10g;
1969                                         iaq->nirq += n10g;
1970                                         n++;
1971                                 }
1972                                 iaq->nrxq10g = min(n, nrxq10g);
1973 #ifdef TCP_OFFLOAD
1974                                 iaq->nofldrxq10g = min(n, nofldrxq10g);
1975 #endif
1976 #ifdef DEV_NETMAP
1977                                 iaq->nnmrxq10g = min(n, nnmrxq10g);
1978 #endif
1979                         }
1980
1981                         if (n1g > 0) {
1982                                 int target = max(nrxq1g, nofldrxq1g);
1983
1984                                 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
1985                                     INTR_RXQ : INTR_OFLD_RXQ;
1986
1987                                 n = 1;
1988                                 while (n < target && leftover >= n1g) {
1989                                         leftover -= n1g;
1990                                         iaq->nirq += n1g;
1991                                         n++;
1992                                 }
1993                                 iaq->nrxq1g = min(n, nrxq1g);
1994 #ifdef TCP_OFFLOAD
1995                                 iaq->nofldrxq1g = min(n, nofldrxq1g);
1996 #endif
1997 #ifdef DEV_NETMAP
1998                                 iaq->nnmrxq1g = min(n, nnmrxq1g);
1999 #endif
2000                         }
2001
2002                         if (itype != INTR_MSI || powerof2(iaq->nirq))
2003                                 goto allocate;
2004                 }
2005
2006                 /*
2007                  * Least desirable option: one interrupt vector for everything.
2008                  */
2009                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
2010                 iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
2011 #ifdef TCP_OFFLOAD
2012                 if (is_offload(sc))
2013                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2014 #endif
2015 #ifdef DEV_NETMAP
2016                 iaq->nnmrxq10g = iaq->nnmrxq1g = 1;
2017 #endif
2018
2019 allocate:
2020                 navail = iaq->nirq;
2021                 rc = 0;
2022                 if (itype == INTR_MSIX)
2023                         rc = pci_alloc_msix(sc->dev, &navail);
2024                 else if (itype == INTR_MSI)
2025                         rc = pci_alloc_msi(sc->dev, &navail);
2026
2027                 if (rc == 0) {
2028                         if (navail == iaq->nirq)
2029                                 return (0);
2030
2031                         /*
2032                          * Didn't get the number requested.  Use whatever number
2033                          * the kernel is willing to allocate (it's in navail).
2034                          */
2035                         device_printf(sc->dev, "fewer vectors than requested, "
2036                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
2037                             itype, iaq->nirq, navail);
2038                         pci_release_msi(sc->dev);
2039                         goto restart;
2040                 }
2041
2042                 device_printf(sc->dev,
2043                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
2044                     itype, rc, iaq->nirq, navail);
2045         }
2046
2047         device_printf(sc->dev,
2048             "failed to find a usable interrupt type.  "
2049             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
2050             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
2051
2052         return (ENXIO);
2053 }
2054
2055 #define FW_VERSION(chip) ( \
2056     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
2057     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
2058     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
2059     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
2060 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
2061
2062 struct fw_info {
2063         uint8_t chip;
2064         char *kld_name;
2065         char *fw_mod_name;
2066         struct fw_hdr fw_hdr;   /* XXX: waste of space, need a sparse struct */
2067 } fw_info[] = {
2068         {
2069                 .chip = CHELSIO_T4,
2070                 .kld_name = "t4fw_cfg",
2071                 .fw_mod_name = "t4fw",
2072                 .fw_hdr = {
2073                         .chip = FW_HDR_CHIP_T4,
2074                         .fw_ver = htobe32_const(FW_VERSION(T4)),
2075                         .intfver_nic = FW_INTFVER(T4, NIC),
2076                         .intfver_vnic = FW_INTFVER(T4, VNIC),
2077                         .intfver_ofld = FW_INTFVER(T4, OFLD),
2078                         .intfver_ri = FW_INTFVER(T4, RI),
2079                         .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
2080                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
2081                         .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
2082                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
2083                 },
2084         }, {
2085                 .chip = CHELSIO_T5,
2086                 .kld_name = "t5fw_cfg",
2087                 .fw_mod_name = "t5fw",
2088                 .fw_hdr = {
2089                         .chip = FW_HDR_CHIP_T5,
2090                         .fw_ver = htobe32_const(FW_VERSION(T5)),
2091                         .intfver_nic = FW_INTFVER(T5, NIC),
2092                         .intfver_vnic = FW_INTFVER(T5, VNIC),
2093                         .intfver_ofld = FW_INTFVER(T5, OFLD),
2094                         .intfver_ri = FW_INTFVER(T5, RI),
2095                         .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
2096                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
2097                         .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
2098                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
2099                 },
2100         }
2101 };
2102
2103 static struct fw_info *
2104 find_fw_info(int chip)
2105 {
2106         int i;
2107
2108         for (i = 0; i < nitems(fw_info); i++) {
2109                 if (fw_info[i].chip == chip)
2110                         return (&fw_info[i]);
2111         }
2112         return (NULL);
2113 }
2114
2115 /*
2116  * Is the given firmware API compatible with the one the driver was compiled
2117  * with?
2118  */
2119 static int
2120 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
2121 {
2122
2123         /* short circuit if it's the exact same firmware version */
2124         if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
2125                 return (1);
2126
2127         /*
2128          * XXX: Is this too conservative?  Perhaps I should limit this to the
2129          * features that are supported in the driver.
2130          */
2131 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
2132         if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
2133             SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
2134             SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
2135                 return (1);
2136 #undef SAME_INTF
2137
2138         return (0);
2139 }
2140
2141 /*
2142  * The firmware in the KLD is usable, but should it be installed?  This routine
2143  * explains itself in detail if it indicates the KLD firmware should be
2144  * installed.
2145  */
2146 static int
2147 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
2148 {
2149         const char *reason;
2150
2151         if (!card_fw_usable) {
2152                 reason = "incompatible or unusable";
2153                 goto install;
2154         }
2155
2156         if (k > c) {
2157                 reason = "older than the version bundled with this driver";
2158                 goto install;
2159         }
2160
2161         if (t4_fw_install == 2 && k != c) {
2162                 reason = "different than the version bundled with this driver";
2163                 goto install;
2164         }
2165
2166         return (0);
2167
2168 install:
2169         if (t4_fw_install == 0) {
2170                 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2171                     "but the driver is prohibited from installing a different "
2172                     "firmware on the card.\n",
2173                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2174                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2175
2176                 return (0);
2177         }
2178
2179         device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2180             "installing firmware %u.%u.%u.%u on card.\n",
2181             G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2182             G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2183             G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2184             G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2185
2186         return (1);
2187 }
2188 /*
2189  * Establish contact with the firmware and determine if we are the master driver
2190  * or not, and whether we are responsible for chip initialization.
2191  */
2192 static int
2193 prep_firmware(struct adapter *sc)
2194 {
2195         const struct firmware *fw = NULL, *default_cfg;
2196         int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2197         enum dev_state state;
2198         struct fw_info *fw_info;
2199         struct fw_hdr *card_fw;         /* fw on the card */
2200         const struct fw_hdr *kld_fw;    /* fw in the KLD */
2201         const struct fw_hdr *drv_fw;    /* fw header the driver was compiled
2202                                            against */
2203
2204         /* Contact firmware. */
2205         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2206         if (rc < 0 || state == DEV_STATE_ERR) {
2207                 rc = -rc;
2208                 device_printf(sc->dev,
2209                     "failed to connect to the firmware: %d, %d.\n", rc, state);
2210                 return (rc);
2211         }
2212         pf = rc;
2213         if (pf == sc->mbox)
2214                 sc->flags |= MASTER_PF;
2215         else if (state == DEV_STATE_UNINIT) {
2216                 /*
2217                  * We didn't get to be the master so we definitely won't be
2218                  * configuring the chip.  It's a bug if someone else hasn't
2219                  * configured it already.
2220                  */
2221                 device_printf(sc->dev, "couldn't be master(%d), "
2222                     "device not already initialized either(%d).\n", rc, state);
2223                 return (EDOOFUS);
2224         }
2225
2226         /* This is the firmware whose headers the driver was compiled against */
2227         fw_info = find_fw_info(chip_id(sc));
2228         if (fw_info == NULL) {
2229                 device_printf(sc->dev,
2230                     "unable to look up firmware information for chip %d.\n",
2231                     chip_id(sc));
2232                 return (EINVAL);
2233         }
2234         drv_fw = &fw_info->fw_hdr;
2235
2236         /*
2237          * The firmware KLD contains many modules.  The KLD name is also the
2238          * name of the module that contains the default config file.
2239          */
2240         default_cfg = firmware_get(fw_info->kld_name);
2241
2242         /* Read the header of the firmware on the card */
2243         card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2244         rc = -t4_read_flash(sc, FLASH_FW_START,
2245             sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2246         if (rc == 0)
2247                 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2248         else {
2249                 device_printf(sc->dev,
2250                     "Unable to read card's firmware header: %d\n", rc);
2251                 card_fw_usable = 0;
2252         }
2253
2254         /* This is the firmware in the KLD */
2255         fw = firmware_get(fw_info->fw_mod_name);
2256         if (fw != NULL) {
2257                 kld_fw = (const void *)fw->data;
2258                 kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2259         } else {
2260                 kld_fw = NULL;
2261                 kld_fw_usable = 0;
2262         }
2263
2264         if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2265             (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2266                 /*
2267                  * Common case: the firmware on the card is an exact match and
2268                  * the KLD is an exact match too, or the KLD is
2269                  * absent/incompatible.  Note that t4_fw_install = 2 is ignored
2270                  * here -- use cxgbetool loadfw if you want to reinstall the
2271                  * same firmware as the one on the card.
2272                  */
2273         } else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2274             should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2275             be32toh(card_fw->fw_ver))) {
2276
2277                 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
2278                 if (rc != 0) {
2279                         device_printf(sc->dev,
2280                             "failed to install firmware: %d\n", rc);
2281                         goto done;
2282                 }
2283
2284                 /* Installed successfully, update the cached header too. */
2285                 memcpy(card_fw, kld_fw, sizeof(*card_fw));
2286                 card_fw_usable = 1;
2287                 need_fw_reset = 0;      /* already reset as part of load_fw */
2288         }
2289
2290         if (!card_fw_usable) {
2291                 uint32_t d, c, k;
2292
2293                 d = ntohl(drv_fw->fw_ver);
2294                 c = ntohl(card_fw->fw_ver);
2295                 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
2296
2297                 device_printf(sc->dev, "Cannot find a usable firmware: "
2298                     "fw_install %d, chip state %d, "
2299                     "driver compiled with %d.%d.%d.%d, "
2300                     "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
2301                     t4_fw_install, state,
2302                     G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
2303                     G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
2304                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2305                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
2306                     G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2307                     G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2308                 rc = EINVAL;
2309                 goto done;
2310         }
2311
2312         /* We're using whatever's on the card and it's known to be good. */
2313         sc->params.fw_vers = ntohl(card_fw->fw_ver);
2314         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
2315             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2316             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2317             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2318             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2319         t4_get_tp_version(sc, &sc->params.tp_vers);
2320
2321         /* Reset device */
2322         if (need_fw_reset &&
2323             (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
2324                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
2325                 if (rc != ETIMEDOUT && rc != EIO)
2326                         t4_fw_bye(sc, sc->mbox);
2327                 goto done;
2328         }
2329         sc->flags |= FW_OK;
2330
2331         rc = get_params__pre_init(sc);
2332         if (rc != 0)
2333                 goto done; /* error message displayed already */
2334
2335         /* Partition adapter resources as specified in the config file. */
2336         if (state == DEV_STATE_UNINIT) {
2337
2338                 KASSERT(sc->flags & MASTER_PF,
2339                     ("%s: trying to change chip settings when not master.",
2340                     __func__));
2341
2342                 rc = partition_resources(sc, default_cfg, fw_info->kld_name);
2343                 if (rc != 0)
2344                         goto done;      /* error message displayed already */
2345
2346                 t4_tweak_chip_settings(sc);
2347
2348                 /* get basic stuff going */
2349                 rc = -t4_fw_initialize(sc, sc->mbox);
2350                 if (rc != 0) {
2351                         device_printf(sc->dev, "fw init failed: %d.\n", rc);
2352                         goto done;
2353                 }
2354         } else {
2355                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
2356                 sc->cfcsum = 0;
2357         }
2358
2359 done:
2360         free(card_fw, M_CXGBE);
2361         if (fw != NULL)
2362                 firmware_put(fw, FIRMWARE_UNLOAD);
2363         if (default_cfg != NULL)
2364                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
2365
2366         return (rc);
2367 }
2368
2369 #define FW_PARAM_DEV(param) \
2370         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2371          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2372 #define FW_PARAM_PFVF(param) \
2373         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2374          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2375
2376 /*
2377  * Partition chip resources for use between various PFs, VFs, etc.
2378  */
2379 static int
2380 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
2381     const char *name_prefix)
2382 {
2383         const struct firmware *cfg = NULL;
2384         int rc = 0;
2385         struct fw_caps_config_cmd caps;
2386         uint32_t mtype, moff, finicsum, cfcsum;
2387
2388         /*
2389          * Figure out what configuration file to use.  Pick the default config
2390          * file for the card if the user hasn't specified one explicitly.
2391          */
2392         snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
2393         if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
2394                 /* Card specific overrides go here. */
2395                 if (pci_get_device(sc->dev) == 0x440a)
2396                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
2397                 if (is_fpga(sc))
2398                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
2399         }
2400
2401         /*
2402          * We need to load another module if the profile is anything except
2403          * "default" or "flash".
2404          */
2405         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
2406             strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2407                 char s[32];
2408
2409                 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
2410                 cfg = firmware_get(s);
2411                 if (cfg == NULL) {
2412                         if (default_cfg != NULL) {
2413                                 device_printf(sc->dev,
2414                                     "unable to load module \"%s\" for "
2415                                     "configuration profile \"%s\", will use "
2416                                     "the default config file instead.\n",
2417                                     s, sc->cfg_file);
2418                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2419                                     "%s", DEFAULT_CF);
2420                         } else {
2421                                 device_printf(sc->dev,
2422                                     "unable to load module \"%s\" for "
2423                                     "configuration profile \"%s\", will use "
2424                                     "the config file on the card's flash "
2425                                     "instead.\n", s, sc->cfg_file);
2426                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2427                                     "%s", FLASH_CF);
2428                         }
2429                 }
2430         }
2431
2432         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
2433             default_cfg == NULL) {
2434                 device_printf(sc->dev,
2435                     "default config file not available, will use the config "
2436                     "file on the card's flash instead.\n");
2437                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
2438         }
2439
2440         if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2441                 u_int cflen, i, n;
2442                 const uint32_t *cfdata;
2443                 uint32_t param, val, addr, off, mw_base, mw_aperture;
2444
2445                 KASSERT(cfg != NULL || default_cfg != NULL,
2446                     ("%s: no config to upload", __func__));
2447
2448                 /*
2449                  * Ask the firmware where it wants us to upload the config file.
2450                  */
2451                 param = FW_PARAM_DEV(CF);
2452                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2453                 if (rc != 0) {
2454                         /* No support for config file?  Shouldn't happen. */
2455                         device_printf(sc->dev,
2456                             "failed to query config file location: %d.\n", rc);
2457                         goto done;
2458                 }
2459                 mtype = G_FW_PARAMS_PARAM_Y(val);
2460                 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
2461
2462                 /*
2463                  * XXX: sheer laziness.  We deliberately added 4 bytes of
2464                  * useless stuffing/comments at the end of the config file so
2465                  * it's ok to simply throw away the last remaining bytes when
2466                  * the config file is not an exact multiple of 4.  This also
2467                  * helps with the validate_mt_off_len check.
2468                  */
2469                 if (cfg != NULL) {
2470                         cflen = cfg->datasize & ~3;
2471                         cfdata = cfg->data;
2472                 } else {
2473                         cflen = default_cfg->datasize & ~3;
2474                         cfdata = default_cfg->data;
2475                 }
2476
2477                 if (cflen > FLASH_CFG_MAX_SIZE) {
2478                         device_printf(sc->dev,
2479                             "config file too long (%d, max allowed is %d).  "
2480                             "Will try to use the config on the card, if any.\n",
2481                             cflen, FLASH_CFG_MAX_SIZE);
2482                         goto use_config_on_flash;
2483                 }
2484
2485                 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
2486                 if (rc != 0) {
2487                         device_printf(sc->dev,
2488                             "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
2489                             "Will try to use the config on the card, if any.\n",
2490                             __func__, mtype, moff, cflen, rc);
2491                         goto use_config_on_flash;
2492                 }
2493
2494                 memwin_info(sc, 2, &mw_base, &mw_aperture);
2495                 while (cflen) {
2496                         off = position_memwin(sc, 2, addr);
2497                         n = min(cflen, mw_aperture - off);
2498                         for (i = 0; i < n; i += 4)
2499                                 t4_write_reg(sc, mw_base + off + i, *cfdata++);
2500                         cflen -= n;
2501                         addr += n;
2502                 }
2503         } else {
2504 use_config_on_flash:
2505                 mtype = FW_MEMTYPE_FLASH;
2506                 moff = t4_flash_cfg_addr(sc);
2507         }
2508
2509         bzero(&caps, sizeof(caps));
2510         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2511             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2512         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
2513             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
2514             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
2515         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2516         if (rc != 0) {
2517                 device_printf(sc->dev,
2518                     "failed to pre-process config file: %d "
2519                     "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
2520                 goto done;
2521         }
2522
2523         finicsum = be32toh(caps.finicsum);
2524         cfcsum = be32toh(caps.cfcsum);
2525         if (finicsum != cfcsum) {
2526                 device_printf(sc->dev,
2527                     "WARNING: config file checksum mismatch: %08x %08x\n",
2528                     finicsum, cfcsum);
2529         }
2530         sc->cfcsum = cfcsum;
2531
2532 #define LIMIT_CAPS(x) do { \
2533         caps.x &= htobe16(t4_##x##_allowed); \
2534 } while (0)
2535
2536         /*
2537          * Let the firmware know what features will (not) be used so it can tune
2538          * things accordingly.
2539          */
2540         LIMIT_CAPS(linkcaps);
2541         LIMIT_CAPS(niccaps);
2542         LIMIT_CAPS(toecaps);
2543         LIMIT_CAPS(rdmacaps);
2544         LIMIT_CAPS(iscsicaps);
2545         LIMIT_CAPS(fcoecaps);
2546 #undef LIMIT_CAPS
2547
2548         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2549             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
2550         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2551         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
2552         if (rc != 0) {
2553                 device_printf(sc->dev,
2554                     "failed to process config file: %d.\n", rc);
2555         }
2556 done:
2557         if (cfg != NULL)
2558                 firmware_put(cfg, FIRMWARE_UNLOAD);
2559         return (rc);
2560 }
2561
2562 /*
2563  * Retrieve parameters that are needed (or nice to have) very early.
2564  */
2565 static int
2566 get_params__pre_init(struct adapter *sc)
2567 {
2568         int rc;
2569         uint32_t param[2], val[2];
2570         struct fw_devlog_cmd cmd;
2571         struct devlog_params *dlog = &sc->params.devlog;
2572
2573         param[0] = FW_PARAM_DEV(PORTVEC);
2574         param[1] = FW_PARAM_DEV(CCLK);
2575         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2576         if (rc != 0) {
2577                 device_printf(sc->dev,
2578                     "failed to query parameters (pre_init): %d.\n", rc);
2579                 return (rc);
2580         }
2581
2582         sc->params.portvec = val[0];
2583         sc->params.nports = bitcount32(val[0]);
2584         sc->params.vpd.cclk = val[1];
2585
2586         /* Read device log parameters. */
2587         bzero(&cmd, sizeof(cmd));
2588         cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
2589             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2590         cmd.retval_len16 = htobe32(FW_LEN16(cmd));
2591         rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
2592         if (rc != 0) {
2593                 device_printf(sc->dev,
2594                     "failed to get devlog parameters: %d.\n", rc);
2595                 bzero(dlog, sizeof (*dlog));
2596                 rc = 0; /* devlog isn't critical for device operation */
2597         } else {
2598                 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
2599                 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
2600                 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
2601                 dlog->size = be32toh(cmd.memsize_devlog);
2602         }
2603
2604         return (rc);
2605 }
2606
2607 /*
2608  * Retrieve various parameters that are of interest to the driver.  The device
2609  * has been initialized by the firmware at this point.
2610  */
2611 static int
2612 get_params__post_init(struct adapter *sc)
2613 {
2614         int rc;
2615         uint32_t param[7], val[7];
2616         struct fw_caps_config_cmd caps;
2617
2618         param[0] = FW_PARAM_PFVF(IQFLINT_START);
2619         param[1] = FW_PARAM_PFVF(EQ_START);
2620         param[2] = FW_PARAM_PFVF(FILTER_START);
2621         param[3] = FW_PARAM_PFVF(FILTER_END);
2622         param[4] = FW_PARAM_PFVF(L2T_START);
2623         param[5] = FW_PARAM_PFVF(L2T_END);
2624         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2625         if (rc != 0) {
2626                 device_printf(sc->dev,
2627                     "failed to query parameters (post_init): %d.\n", rc);
2628                 return (rc);
2629         }
2630
2631         sc->sge.iq_start = val[0];
2632         sc->sge.eq_start = val[1];
2633         sc->tids.ftid_base = val[2];
2634         sc->tids.nftids = val[3] - val[2] + 1;
2635         sc->params.ftid_min = val[2];
2636         sc->params.ftid_max = val[3];
2637         sc->vres.l2t.start = val[4];
2638         sc->vres.l2t.size = val[5] - val[4] + 1;
2639         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
2640             ("%s: L2 table size (%u) larger than expected (%u)",
2641             __func__, sc->vres.l2t.size, L2T_SIZE));
2642
2643         /* get capabilites */
2644         bzero(&caps, sizeof(caps));
2645         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2646             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2647         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2648         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2649         if (rc != 0) {
2650                 device_printf(sc->dev,
2651                     "failed to get card capabilities: %d.\n", rc);
2652                 return (rc);
2653         }
2654
2655 #define READ_CAPS(x) do { \
2656         sc->x = htobe16(caps.x); \
2657 } while (0)
2658         READ_CAPS(linkcaps);
2659         READ_CAPS(niccaps);
2660         READ_CAPS(toecaps);
2661         READ_CAPS(rdmacaps);
2662         READ_CAPS(iscsicaps);
2663         READ_CAPS(fcoecaps);
2664
2665         if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
2666                 param[0] = FW_PARAM_PFVF(ETHOFLD_START);
2667                 param[1] = FW_PARAM_PFVF(ETHOFLD_END);
2668                 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2669                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
2670                 if (rc != 0) {
2671                         device_printf(sc->dev,
2672                             "failed to query NIC parameters: %d.\n", rc);
2673                         return (rc);
2674                 }
2675                 sc->tids.etid_base = val[0];
2676                 sc->params.etid_min = val[0];
2677                 sc->tids.netids = val[1] - val[0] + 1;
2678                 sc->params.netids = sc->tids.netids;
2679                 sc->params.eo_wr_cred = val[2];
2680                 sc->params.ethoffload = 1;
2681         }
2682
2683         if (sc->toecaps) {
2684                 /* query offload-related parameters */
2685                 param[0] = FW_PARAM_DEV(NTID);
2686                 param[1] = FW_PARAM_PFVF(SERVER_START);
2687                 param[2] = FW_PARAM_PFVF(SERVER_END);
2688                 param[3] = FW_PARAM_PFVF(TDDP_START);
2689                 param[4] = FW_PARAM_PFVF(TDDP_END);
2690                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2691                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2692                 if (rc != 0) {
2693                         device_printf(sc->dev,
2694                             "failed to query TOE parameters: %d.\n", rc);
2695                         return (rc);
2696                 }
2697                 sc->tids.ntids = val[0];
2698                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
2699                 sc->tids.stid_base = val[1];
2700                 sc->tids.nstids = val[2] - val[1] + 1;
2701                 sc->vres.ddp.start = val[3];
2702                 sc->vres.ddp.size = val[4] - val[3] + 1;
2703                 sc->params.ofldq_wr_cred = val[5];
2704                 sc->params.offload = 1;
2705         }
2706         if (sc->rdmacaps) {
2707                 param[0] = FW_PARAM_PFVF(STAG_START);
2708                 param[1] = FW_PARAM_PFVF(STAG_END);
2709                 param[2] = FW_PARAM_PFVF(RQ_START);
2710                 param[3] = FW_PARAM_PFVF(RQ_END);
2711                 param[4] = FW_PARAM_PFVF(PBL_START);
2712                 param[5] = FW_PARAM_PFVF(PBL_END);
2713                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2714                 if (rc != 0) {
2715                         device_printf(sc->dev,
2716                             "failed to query RDMA parameters(1): %d.\n", rc);
2717                         return (rc);
2718                 }
2719                 sc->vres.stag.start = val[0];
2720                 sc->vres.stag.size = val[1] - val[0] + 1;
2721                 sc->vres.rq.start = val[2];
2722                 sc->vres.rq.size = val[3] - val[2] + 1;
2723                 sc->vres.pbl.start = val[4];
2724                 sc->vres.pbl.size = val[5] - val[4] + 1;
2725
2726                 param[0] = FW_PARAM_PFVF(SQRQ_START);
2727                 param[1] = FW_PARAM_PFVF(SQRQ_END);
2728                 param[2] = FW_PARAM_PFVF(CQ_START);
2729                 param[3] = FW_PARAM_PFVF(CQ_END);
2730                 param[4] = FW_PARAM_PFVF(OCQ_START);
2731                 param[5] = FW_PARAM_PFVF(OCQ_END);
2732                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2733                 if (rc != 0) {
2734                         device_printf(sc->dev,
2735                             "failed to query RDMA parameters(2): %d.\n", rc);
2736                         return (rc);
2737                 }
2738                 sc->vres.qp.start = val[0];
2739                 sc->vres.qp.size = val[1] - val[0] + 1;
2740                 sc->vres.cq.start = val[2];
2741                 sc->vres.cq.size = val[3] - val[2] + 1;
2742                 sc->vres.ocq.start = val[4];
2743                 sc->vres.ocq.size = val[5] - val[4] + 1;
2744         }
2745         if (sc->iscsicaps) {
2746                 param[0] = FW_PARAM_PFVF(ISCSI_START);
2747                 param[1] = FW_PARAM_PFVF(ISCSI_END);
2748                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2749                 if (rc != 0) {
2750                         device_printf(sc->dev,
2751                             "failed to query iSCSI parameters: %d.\n", rc);
2752                         return (rc);
2753                 }
2754                 sc->vres.iscsi.start = val[0];
2755                 sc->vres.iscsi.size = val[1] - val[0] + 1;
2756         }
2757
2758         /*
2759          * We've got the params we wanted to query via the firmware.  Now grab
2760          * some others directly from the chip.
2761          */
2762         rc = t4_read_chip_settings(sc);
2763
2764         return (rc);
2765 }
2766
2767 static int
2768 set_params__post_init(struct adapter *sc)
2769 {
2770         uint32_t param, val;
2771
2772         /* ask for encapsulated CPLs */
2773         param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
2774         val = 1;
2775         (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2776
2777         return (0);
2778 }
2779
2780 #undef FW_PARAM_PFVF
2781 #undef FW_PARAM_DEV
2782
2783 static void
2784 t4_set_desc(struct adapter *sc)
2785 {
2786         char buf[128];
2787         struct adapter_params *p = &sc->params;
2788
2789         snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
2790             "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
2791             chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
2792
2793         device_set_desc_copy(sc->dev, buf);
2794 }
2795
2796 static void
2797 build_medialist(struct port_info *pi, struct ifmedia *media)
2798 {
2799         int data, m;
2800
2801         PORT_LOCK(pi);
2802
2803         ifmedia_removeall(media);
2804
2805         m = IFM_ETHER | IFM_FDX;
2806         data = (pi->port_type << 8) | pi->mod_type;
2807
2808         switch(pi->port_type) {
2809         case FW_PORT_TYPE_BT_XFI:
2810         case FW_PORT_TYPE_BT_XAUI:
2811                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2812                 /* fall through */
2813
2814         case FW_PORT_TYPE_BT_SGMII:
2815                 ifmedia_add(media, m | IFM_1000_T, data, NULL);
2816                 ifmedia_add(media, m | IFM_100_TX, data, NULL);
2817                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL);
2818                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2819                 break;
2820
2821         case FW_PORT_TYPE_CX4:
2822                 ifmedia_add(media, m | IFM_10G_CX4, data, NULL);
2823                 ifmedia_set(media, m | IFM_10G_CX4);
2824                 break;
2825
2826         case FW_PORT_TYPE_QSFP_10G:
2827         case FW_PORT_TYPE_SFP:
2828         case FW_PORT_TYPE_FIBER_XFI:
2829         case FW_PORT_TYPE_FIBER_XAUI:
2830                 switch (pi->mod_type) {
2831
2832                 case FW_PORT_MOD_TYPE_LR:
2833                         ifmedia_add(media, m | IFM_10G_LR, data, NULL);
2834                         ifmedia_set(media, m | IFM_10G_LR);
2835                         break;
2836
2837                 case FW_PORT_MOD_TYPE_SR:
2838                         ifmedia_add(media, m | IFM_10G_SR, data, NULL);
2839                         ifmedia_set(media, m | IFM_10G_SR);
2840                         break;
2841
2842                 case FW_PORT_MOD_TYPE_LRM:
2843                         ifmedia_add(media, m | IFM_10G_LRM, data, NULL);
2844                         ifmedia_set(media, m | IFM_10G_LRM);
2845                         break;
2846
2847                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2848                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2849                         ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL);
2850                         ifmedia_set(media, m | IFM_10G_TWINAX);
2851                         break;
2852
2853                 case FW_PORT_MOD_TYPE_NONE:
2854                         m &= ~IFM_FDX;
2855                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2856                         ifmedia_set(media, m | IFM_NONE);
2857                         break;
2858
2859                 case FW_PORT_MOD_TYPE_NA:
2860                 case FW_PORT_MOD_TYPE_ER:
2861                 default:
2862                         device_printf(pi->dev,
2863                             "unknown port_type (%d), mod_type (%d)\n",
2864                             pi->port_type, pi->mod_type);
2865                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2866                         ifmedia_set(media, m | IFM_UNKNOWN);
2867                         break;
2868                 }
2869                 break;
2870
2871         case FW_PORT_TYPE_QSFP:
2872                 switch (pi->mod_type) {
2873
2874                 case FW_PORT_MOD_TYPE_LR:
2875                         ifmedia_add(media, m | IFM_40G_LR4, data, NULL);
2876                         ifmedia_set(media, m | IFM_40G_LR4);
2877                         break;
2878
2879                 case FW_PORT_MOD_TYPE_SR:
2880                         ifmedia_add(media, m | IFM_40G_SR4, data, NULL);
2881                         ifmedia_set(media, m | IFM_40G_SR4);
2882                         break;
2883
2884                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2885                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2886                         ifmedia_add(media, m | IFM_40G_CR4, data, NULL);
2887                         ifmedia_set(media, m | IFM_40G_CR4);
2888                         break;
2889
2890                 case FW_PORT_MOD_TYPE_NONE:
2891                         m &= ~IFM_FDX;
2892                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2893                         ifmedia_set(media, m | IFM_NONE);
2894                         break;
2895
2896                 default:
2897                         device_printf(pi->dev,
2898                             "unknown port_type (%d), mod_type (%d)\n",
2899                             pi->port_type, pi->mod_type);
2900                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2901                         ifmedia_set(media, m | IFM_UNKNOWN);
2902                         break;
2903                 }
2904                 break;
2905
2906         default:
2907                 device_printf(pi->dev,
2908                     "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
2909                     pi->mod_type);
2910                 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2911                 ifmedia_set(media, m | IFM_UNKNOWN);
2912                 break;
2913         }
2914
2915         PORT_UNLOCK(pi);
2916 }
2917
2918 #define FW_MAC_EXACT_CHUNK      7
2919
2920 /*
2921  * Program the port's XGMAC based on parameters in ifnet.  The caller also
2922  * indicates which parameters should be programmed (the rest are left alone).
2923  */
2924 int
2925 update_mac_settings(struct ifnet *ifp, int flags)
2926 {
2927         int rc = 0;
2928         struct port_info *pi = ifp->if_softc;
2929         struct adapter *sc = pi->adapter;
2930         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
2931         uint16_t viid = 0xffff;
2932         int16_t *xact_addr_filt = NULL;
2933
2934         ASSERT_SYNCHRONIZED_OP(sc);
2935         KASSERT(flags, ("%s: not told what to update.", __func__));
2936
2937         if (ifp == pi->ifp) {
2938                 viid = pi->viid;
2939                 xact_addr_filt = &pi->xact_addr_filt;
2940         }
2941 #ifdef DEV_NETMAP
2942         else if (ifp == pi->nm_ifp) {
2943                 viid = pi->nm_viid;
2944                 xact_addr_filt = &pi->nm_xact_addr_filt;
2945         }
2946 #endif
2947         if (flags & XGMAC_MTU)
2948                 mtu = ifp->if_mtu;
2949
2950         if (flags & XGMAC_PROMISC)
2951                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
2952
2953         if (flags & XGMAC_ALLMULTI)
2954                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
2955
2956         if (flags & XGMAC_VLANEX)
2957                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
2958
2959         if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
2960                 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti,
2961                     1, vlanex, false);
2962                 if (rc) {
2963                         if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
2964                             rc);
2965                         return (rc);
2966                 }
2967         }
2968
2969         if (flags & XGMAC_UCADDR) {
2970                 uint8_t ucaddr[ETHER_ADDR_LEN];
2971
2972                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
2973                 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr,
2974                     true, true);
2975                 if (rc < 0) {
2976                         rc = -rc;
2977                         if_printf(ifp, "change_mac failed: %d\n", rc);
2978                         return (rc);
2979                 } else {
2980                         *xact_addr_filt = rc;
2981                         rc = 0;
2982                 }
2983         }
2984
2985         if (flags & XGMAC_MCADDRS) {
2986                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
2987                 int del = 1;
2988                 uint64_t hash = 0;
2989                 struct ifmultiaddr *ifma;
2990                 int i = 0, j;
2991
2992                 if_maddr_rlock(ifp);
2993                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2994                         if (ifma->ifma_addr->sa_family != AF_LINK)
2995                                 continue;
2996                         mcaddr[i++] =
2997                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
2998
2999                         if (i == FW_MAC_EXACT_CHUNK) {
3000                                 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del,
3001                                     i, mcaddr, NULL, &hash, 0);
3002                                 if (rc < 0) {
3003                                         rc = -rc;
3004                                         for (j = 0; j < i; j++) {
3005                                                 if_printf(ifp,
3006                                                     "failed to add mc address"
3007                                                     " %02x:%02x:%02x:"
3008                                                     "%02x:%02x:%02x rc=%d\n",
3009                                                     mcaddr[j][0], mcaddr[j][1],
3010                                                     mcaddr[j][2], mcaddr[j][3],
3011                                                     mcaddr[j][4], mcaddr[j][5],
3012                                                     rc);
3013                                         }
3014                                         goto mcfail;
3015                                 }
3016                                 del = 0;
3017                                 i = 0;
3018                         }
3019                 }
3020                 if (i > 0) {
3021                         rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i,
3022                             mcaddr, NULL, &hash, 0);
3023                         if (rc < 0) {
3024                                 rc = -rc;
3025                                 for (j = 0; j < i; j++) {
3026                                         if_printf(ifp,
3027                                             "failed to add mc address"
3028                                             " %02x:%02x:%02x:"
3029                                             "%02x:%02x:%02x rc=%d\n",
3030                                             mcaddr[j][0], mcaddr[j][1],
3031                                             mcaddr[j][2], mcaddr[j][3],
3032                                             mcaddr[j][4], mcaddr[j][5],
3033                                             rc);
3034                                 }
3035                                 goto mcfail;
3036                         }
3037                 }
3038
3039                 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0);
3040                 if (rc != 0)
3041                         if_printf(ifp, "failed to set mc address hash: %d", rc);
3042 mcfail:
3043                 if_maddr_runlock(ifp);
3044         }
3045
3046         return (rc);
3047 }
3048
3049 /*
3050  * {begin|end}_synchronized_op must be called from the same thread.
3051  */
3052 int
3053 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
3054     char *wmesg)
3055 {
3056         int rc, pri;
3057
3058 #ifdef WITNESS
3059         /* the caller thinks it's ok to sleep, but is it really? */
3060         if (flags & SLEEP_OK)
3061                 pause("t4slptst", 1);
3062 #endif
3063
3064         if (INTR_OK)
3065                 pri = PCATCH;
3066         else
3067                 pri = 0;
3068
3069         ADAPTER_LOCK(sc);
3070         for (;;) {
3071
3072                 if (pi && IS_DOOMED(pi)) {
3073                         rc = ENXIO;
3074                         goto done;
3075                 }
3076
3077                 if (!IS_BUSY(sc)) {
3078                         rc = 0;
3079                         break;
3080                 }
3081
3082                 if (!(flags & SLEEP_OK)) {
3083                         rc = EBUSY;
3084                         goto done;
3085                 }
3086
3087                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
3088                         rc = EINTR;
3089                         goto done;
3090                 }
3091         }
3092
3093         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
3094         SET_BUSY(sc);
3095 #ifdef INVARIANTS
3096         sc->last_op = wmesg;
3097         sc->last_op_thr = curthread;
3098 #endif
3099
3100 done:
3101         if (!(flags & HOLD_LOCK) || rc)
3102                 ADAPTER_UNLOCK(sc);
3103
3104         return (rc);
3105 }
3106
3107 /*
3108  * {begin|end}_synchronized_op must be called from the same thread.
3109  */
3110 void
3111 end_synchronized_op(struct adapter *sc, int flags)
3112 {
3113
3114         if (flags & LOCK_HELD)
3115                 ADAPTER_LOCK_ASSERT_OWNED(sc);
3116         else
3117                 ADAPTER_LOCK(sc);
3118
3119         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
3120         CLR_BUSY(sc);
3121         wakeup(&sc->flags);
3122         ADAPTER_UNLOCK(sc);
3123 }
3124
3125 static int
3126 cxgbe_init_synchronized(struct port_info *pi)
3127 {
3128         struct adapter *sc = pi->adapter;
3129         struct ifnet *ifp = pi->ifp;
3130         int rc = 0;
3131
3132         ASSERT_SYNCHRONIZED_OP(sc);
3133
3134         if (isset(&sc->open_device_map, pi->port_id)) {
3135                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
3136                     ("mismatch between open_device_map and if_drv_flags"));
3137                 return (0);     /* already running */
3138         }
3139
3140         if (!(sc->flags & FULL_INIT_DONE) &&
3141             ((rc = adapter_full_init(sc)) != 0))
3142                 return (rc);    /* error message displayed already */
3143
3144         if (!(pi->flags & PORT_INIT_DONE) &&
3145             ((rc = port_full_init(pi)) != 0))
3146                 return (rc); /* error message displayed already */
3147
3148         rc = update_mac_settings(ifp, XGMAC_ALL);
3149         if (rc)
3150                 goto done;      /* error message displayed already */
3151
3152         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
3153         if (rc != 0) {
3154                 if_printf(ifp, "enable_vi failed: %d\n", rc);
3155                 goto done;
3156         }
3157
3158         /*
3159          * The first iq of the first port to come up is used for tracing.
3160          */
3161         if (sc->traceq < 0) {
3162                 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
3163                 t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
3164                     A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
3165                     V_QUEUENUMBER(sc->traceq));
3166                 pi->flags |= HAS_TRACEQ;
3167         }
3168
3169         /* all ok */
3170         setbit(&sc->open_device_map, pi->port_id);
3171         PORT_LOCK(pi);
3172         ifp->if_drv_flags |= IFF_DRV_RUNNING;
3173         PORT_UNLOCK(pi);
3174
3175         callout_reset(&pi->tick, hz, cxgbe_tick, pi);
3176 done:
3177         if (rc != 0)
3178                 cxgbe_uninit_synchronized(pi);
3179
3180         return (rc);
3181 }
3182
3183 /*
3184  * Idempotent.
3185  */
3186 static int
3187 cxgbe_uninit_synchronized(struct port_info *pi)
3188 {
3189         struct adapter *sc = pi->adapter;
3190         struct ifnet *ifp = pi->ifp;
3191         int rc;
3192
3193         ASSERT_SYNCHRONIZED_OP(sc);
3194
3195         /*
3196          * Disable the VI so that all its data in either direction is discarded
3197          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
3198          * tick) intact as the TP can deliver negative advice or data that it's
3199          * holding in its RAM (for an offloaded connection) even after the VI is
3200          * disabled.
3201          */
3202         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
3203         if (rc) {
3204                 if_printf(ifp, "disable_vi failed: %d\n", rc);
3205                 return (rc);
3206         }
3207
3208         clrbit(&sc->open_device_map, pi->port_id);
3209         PORT_LOCK(pi);
3210         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3211         PORT_UNLOCK(pi);
3212
3213         pi->link_cfg.link_ok = 0;
3214         pi->link_cfg.speed = 0;
3215         pi->linkdnrc = -1;
3216         t4_os_link_changed(sc, pi->port_id, 0, -1);
3217
3218         return (0);
3219 }
3220
3221 /*
3222  * It is ok for this function to fail midway and return right away.  t4_detach
3223  * will walk the entire sc->irq list and clean up whatever is valid.
3224  */
3225 static int
3226 setup_intr_handlers(struct adapter *sc)
3227 {
3228         int rc, rid, p, q;
3229         char s[8];
3230         struct irq *irq;
3231         struct port_info *pi;
3232         struct sge_rxq *rxq;
3233 #ifdef TCP_OFFLOAD
3234         struct sge_ofld_rxq *ofld_rxq;
3235 #endif
3236 #ifdef DEV_NETMAP
3237         struct sge_nm_rxq *nm_rxq;
3238 #endif
3239
3240         /*
3241          * Setup interrupts.
3242          */
3243         irq = &sc->irq[0];
3244         rid = sc->intr_type == INTR_INTX ? 0 : 1;
3245         if (sc->intr_count == 1)
3246                 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
3247
3248         /* Multiple interrupts. */
3249         KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
3250             ("%s: too few intr.", __func__));
3251
3252         /* The first one is always error intr */
3253         rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
3254         if (rc != 0)
3255                 return (rc);
3256         irq++;
3257         rid++;
3258
3259         /* The second one is always the firmware event queue */
3260         rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
3261         if (rc != 0)
3262                 return (rc);
3263         irq++;
3264         rid++;
3265
3266         for_each_port(sc, p) {
3267                 pi = sc->port[p];
3268
3269                 if (pi->flags & INTR_RXQ) {
3270                         for_each_rxq(pi, q, rxq) {
3271                                 snprintf(s, sizeof(s), "%d.%d", p, q);
3272                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
3273                                     s);
3274                                 if (rc != 0)
3275                                         return (rc);
3276                                 irq++;
3277                                 rid++;
3278                         }
3279                 }
3280 #ifdef TCP_OFFLOAD
3281                 if (pi->flags & INTR_OFLD_RXQ) {
3282                         for_each_ofld_rxq(pi, q, ofld_rxq) {
3283                                 snprintf(s, sizeof(s), "%d,%d", p, q);
3284                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr,
3285                                     ofld_rxq, s);
3286                                 if (rc != 0)
3287                                         return (rc);
3288                                 irq++;
3289                                 rid++;
3290                         }
3291                 }
3292 #endif
3293 #ifdef DEV_NETMAP
3294                 if (pi->flags & INTR_NM_RXQ) {
3295                         for_each_nm_rxq(pi, q, nm_rxq) {
3296                                 snprintf(s, sizeof(s), "%d-%d", p, q);
3297                                 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr,
3298                                     nm_rxq, s);
3299                                 if (rc != 0)
3300                                         return (rc);
3301                                 irq++;
3302                                 rid++;
3303                         }
3304                 }
3305 #endif
3306         }
3307         MPASS(irq == &sc->irq[sc->intr_count]);
3308
3309         return (0);
3310 }
3311
3312 int
3313 adapter_full_init(struct adapter *sc)
3314 {
3315         int rc, i;
3316
3317         ASSERT_SYNCHRONIZED_OP(sc);
3318         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3319         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
3320             ("%s: FULL_INIT_DONE already", __func__));
3321
3322         /*
3323          * queues that belong to the adapter (not any particular port).
3324          */
3325         rc = t4_setup_adapter_queues(sc);
3326         if (rc != 0)
3327                 goto done;
3328
3329         for (i = 0; i < nitems(sc->tq); i++) {
3330                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
3331                     taskqueue_thread_enqueue, &sc->tq[i]);
3332                 if (sc->tq[i] == NULL) {
3333                         device_printf(sc->dev,
3334                             "failed to allocate task queue %d\n", i);
3335                         rc = ENOMEM;
3336                         goto done;
3337                 }
3338                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
3339                     device_get_nameunit(sc->dev), i);
3340         }
3341
3342         t4_intr_enable(sc);
3343         sc->flags |= FULL_INIT_DONE;
3344 done:
3345         if (rc != 0)
3346                 adapter_full_uninit(sc);
3347
3348         return (rc);
3349 }
3350
3351 int
3352 adapter_full_uninit(struct adapter *sc)
3353 {
3354         int i;
3355
3356         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3357
3358         t4_teardown_adapter_queues(sc);
3359
3360         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
3361                 taskqueue_free(sc->tq[i]);
3362                 sc->tq[i] = NULL;
3363         }
3364
3365         sc->flags &= ~FULL_INIT_DONE;
3366
3367         return (0);
3368 }
3369
3370 int
3371 port_full_init(struct port_info *pi)
3372 {
3373         struct adapter *sc = pi->adapter;
3374         struct ifnet *ifp = pi->ifp;
3375         uint16_t *rss;
3376         struct sge_rxq *rxq;
3377         int rc, i, j;
3378
3379         ASSERT_SYNCHRONIZED_OP(sc);
3380         KASSERT((pi->flags & PORT_INIT_DONE) == 0,
3381             ("%s: PORT_INIT_DONE already", __func__));
3382
3383         sysctl_ctx_init(&pi->ctx);
3384         pi->flags |= PORT_SYSCTL_CTX;
3385
3386         /*
3387          * Allocate tx/rx/fl queues for this port.
3388          */
3389         rc = t4_setup_port_queues(pi);
3390         if (rc != 0)
3391                 goto done;      /* error message displayed already */
3392
3393         /*
3394          * Setup RSS for this port.  Save a copy of the RSS table for later use.
3395          */
3396         rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
3397         for (i = 0; i < pi->rss_size;) {
3398                 for_each_rxq(pi, j, rxq) {
3399                         rss[i++] = rxq->iq.abs_id;
3400                         if (i == pi->rss_size)
3401                                 break;
3402                 }
3403         }
3404
3405         rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
3406             pi->rss_size);
3407         if (rc != 0) {
3408                 if_printf(ifp, "rss_config failed: %d\n", rc);
3409                 goto done;
3410         }
3411
3412         pi->rss = rss;
3413         pi->flags |= PORT_INIT_DONE;
3414 done:
3415         if (rc != 0)
3416                 port_full_uninit(pi);
3417
3418         return (rc);
3419 }
3420
3421 /*
3422  * Idempotent.
3423  */
3424 int
3425 port_full_uninit(struct port_info *pi)
3426 {
3427         struct adapter *sc = pi->adapter;
3428         int i;
3429         struct sge_rxq *rxq;
3430         struct sge_txq *txq;
3431 #ifdef TCP_OFFLOAD
3432         struct sge_ofld_rxq *ofld_rxq;
3433         struct sge_wrq *ofld_txq;
3434 #endif
3435
3436         if (pi->flags & PORT_INIT_DONE) {
3437
3438                 /* Need to quiesce queues.  XXX: ctrl queues? */
3439
3440                 for_each_txq(pi, i, txq) {
3441                         quiesce_eq(sc, &txq->eq);
3442                 }
3443
3444 #ifdef TCP_OFFLOAD
3445                 for_each_ofld_txq(pi, i, ofld_txq) {
3446                         quiesce_eq(sc, &ofld_txq->eq);
3447                 }
3448 #endif
3449
3450                 for_each_rxq(pi, i, rxq) {
3451                         quiesce_iq(sc, &rxq->iq);
3452                         quiesce_fl(sc, &rxq->fl);
3453                 }
3454
3455 #ifdef TCP_OFFLOAD
3456                 for_each_ofld_rxq(pi, i, ofld_rxq) {
3457                         quiesce_iq(sc, &ofld_rxq->iq);
3458                         quiesce_fl(sc, &ofld_rxq->fl);
3459                 }
3460 #endif
3461                 free(pi->rss, M_CXGBE);
3462         }
3463
3464         t4_teardown_port_queues(pi);
3465         pi->flags &= ~PORT_INIT_DONE;
3466
3467         return (0);
3468 }
3469
3470 static void
3471 quiesce_eq(struct adapter *sc, struct sge_eq *eq)
3472 {
3473         EQ_LOCK(eq);
3474         eq->flags |= EQ_DOOMED;
3475
3476         /*
3477          * Wait for the response to a credit flush if one's
3478          * pending.
3479          */
3480         while (eq->flags & EQ_CRFLUSHED)
3481                 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0);
3482         EQ_UNLOCK(eq);
3483
3484         callout_drain(&eq->tx_callout); /* XXX: iffy */
3485         pause("callout", 10);           /* Still iffy */
3486
3487         taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task);
3488 }
3489
3490 static void
3491 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
3492 {
3493         (void) sc;      /* unused */
3494
3495         /* Synchronize with the interrupt handler */
3496         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
3497                 pause("iqfree", 1);
3498 }
3499
3500 static void
3501 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
3502 {
3503         mtx_lock(&sc->sfl_lock);
3504         FL_LOCK(fl);
3505         fl->flags |= FL_DOOMED;
3506         FL_UNLOCK(fl);
3507         mtx_unlock(&sc->sfl_lock);
3508
3509         callout_drain(&sc->sfl_callout);
3510         KASSERT((fl->flags & FL_STARVING) == 0,
3511             ("%s: still starving", __func__));
3512 }
3513
3514 static int
3515 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
3516     driver_intr_t *handler, void *arg, char *name)
3517 {
3518         int rc;
3519
3520         irq->rid = rid;
3521         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
3522             RF_SHAREABLE | RF_ACTIVE);
3523         if (irq->res == NULL) {
3524                 device_printf(sc->dev,
3525                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
3526                 return (ENOMEM);
3527         }
3528
3529         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
3530             NULL, handler, arg, &irq->tag);
3531         if (rc != 0) {
3532                 device_printf(sc->dev,
3533                     "failed to setup interrupt for rid %d, name %s: %d\n",
3534                     rid, name, rc);
3535         } else if (name)
3536                 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
3537
3538         return (rc);
3539 }
3540
3541 static int
3542 t4_free_irq(struct adapter *sc, struct irq *irq)
3543 {
3544         if (irq->tag)
3545                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
3546         if (irq->res)
3547                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
3548
3549         bzero(irq, sizeof(*irq));
3550
3551         return (0);
3552 }
3553
3554 static void
3555 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
3556     unsigned int end)
3557 {
3558         uint32_t *p = (uint32_t *)(buf + start);
3559
3560         for ( ; start <= end; start += sizeof(uint32_t))
3561                 *p++ = t4_read_reg(sc, start);
3562 }
3563
3564 static void
3565 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
3566 {
3567         int i, n;
3568         const unsigned int *reg_ranges;
3569         static const unsigned int t4_reg_ranges[] = {
3570                 0x1008, 0x1108,
3571                 0x1180, 0x11b4,
3572                 0x11fc, 0x123c,
3573                 0x1300, 0x173c,
3574                 0x1800, 0x18fc,
3575                 0x3000, 0x30d8,
3576                 0x30e0, 0x5924,
3577                 0x5960, 0x59d4,
3578                 0x5a00, 0x5af8,
3579                 0x6000, 0x6098,
3580                 0x6100, 0x6150,
3581                 0x6200, 0x6208,
3582                 0x6240, 0x6248,
3583                 0x6280, 0x6338,
3584                 0x6370, 0x638c,
3585                 0x6400, 0x643c,
3586                 0x6500, 0x6524,
3587                 0x6a00, 0x6a38,
3588                 0x6a60, 0x6a78,
3589                 0x6b00, 0x6b84,
3590                 0x6bf0, 0x6c84,
3591                 0x6cf0, 0x6d84,
3592                 0x6df0, 0x6e84,
3593                 0x6ef0, 0x6f84,
3594                 0x6ff0, 0x7084,
3595                 0x70f0, 0x7184,
3596                 0x71f0, 0x7284,
3597                 0x72f0, 0x7384,
3598                 0x73f0, 0x7450,
3599                 0x7500, 0x7530,
3600                 0x7600, 0x761c,
3601                 0x7680, 0x76cc,
3602                 0x7700, 0x7798,
3603                 0x77c0, 0x77fc,
3604                 0x7900, 0x79fc,
3605                 0x7b00, 0x7c38,
3606                 0x7d00, 0x7efc,
3607                 0x8dc0, 0x8e1c,
3608                 0x8e30, 0x8e78,
3609                 0x8ea0, 0x8f6c,
3610                 0x8fc0, 0x9074,
3611                 0x90fc, 0x90fc,
3612                 0x9400, 0x9458,
3613                 0x9600, 0x96bc,
3614                 0x9800, 0x9808,
3615                 0x9820, 0x983c,
3616                 0x9850, 0x9864,
3617                 0x9c00, 0x9c6c,
3618                 0x9c80, 0x9cec,
3619                 0x9d00, 0x9d6c,
3620                 0x9d80, 0x9dec,
3621                 0x9e00, 0x9e6c,
3622                 0x9e80, 0x9eec,
3623                 0x9f00, 0x9f6c,
3624                 0x9f80, 0x9fec,
3625                 0xd004, 0xd03c,
3626                 0xdfc0, 0xdfe0,
3627                 0xe000, 0xea7c,
3628                 0xf000, 0x11110,
3629                 0x11118, 0x11190,
3630                 0x19040, 0x1906c,
3631                 0x19078, 0x19080,
3632                 0x1908c, 0x19124,
3633                 0x19150, 0x191b0,
3634                 0x191d0, 0x191e8,
3635                 0x19238, 0x1924c,
3636                 0x193f8, 0x19474,
3637                 0x19490, 0x194f8,
3638                 0x19800, 0x19f30,
3639                 0x1a000, 0x1a06c,
3640                 0x1a0b0, 0x1a120,
3641                 0x1a128, 0x1a138,
3642                 0x1a190, 0x1a1c4,
3643                 0x1a1fc, 0x1a1fc,
3644                 0x1e040, 0x1e04c,
3645                 0x1e284, 0x1e28c,
3646                 0x1e2c0, 0x1e2c0,
3647                 0x1e2e0, 0x1e2e0,
3648                 0x1e300, 0x1e384,
3649                 0x1e3c0, 0x1e3c8,
3650                 0x1e440, 0x1e44c,
3651                 0x1e684, 0x1e68c,
3652                 0x1e6c0, 0x1e6c0,
3653                 0x1e6e0, 0x1e6e0,
3654                 0x1e700, 0x1e784,
3655                 0x1e7c0, 0x1e7c8,
3656                 0x1e840, 0x1e84c,
3657                 0x1ea84, 0x1ea8c,
3658                 0x1eac0, 0x1eac0,
3659                 0x1eae0, 0x1eae0,
3660                 0x1eb00, 0x1eb84,
3661                 0x1ebc0, 0x1ebc8,
3662                 0x1ec40, 0x1ec4c,
3663                 0x1ee84, 0x1ee8c,
3664                 0x1eec0, 0x1eec0,
3665                 0x1eee0, 0x1eee0,
3666                 0x1ef00, 0x1ef84,
3667                 0x1efc0, 0x1efc8,
3668                 0x1f040, 0x1f04c,
3669                 0x1f284, 0x1f28c,
3670                 0x1f2c0, 0x1f2c0,
3671                 0x1f2e0, 0x1f2e0,
3672                 0x1f300, 0x1f384,
3673                 0x1f3c0, 0x1f3c8,
3674                 0x1f440, 0x1f44c,
3675                 0x1f684, 0x1f68c,
3676                 0x1f6c0, 0x1f6c0,
3677                 0x1f6e0, 0x1f6e0,
3678                 0x1f700, 0x1f784,
3679                 0x1f7c0, 0x1f7c8,
3680                 0x1f840, 0x1f84c,
3681                 0x1fa84, 0x1fa8c,
3682                 0x1fac0, 0x1fac0,
3683                 0x1fae0, 0x1fae0,
3684                 0x1fb00, 0x1fb84,
3685                 0x1fbc0, 0x1fbc8,
3686                 0x1fc40, 0x1fc4c,
3687                 0x1fe84, 0x1fe8c,
3688                 0x1fec0, 0x1fec0,
3689                 0x1fee0, 0x1fee0,
3690                 0x1ff00, 0x1ff84,
3691                 0x1ffc0, 0x1ffc8,
3692                 0x20000, 0x2002c,
3693                 0x20100, 0x2013c,
3694                 0x20190, 0x201c8,
3695                 0x20200, 0x20318,
3696                 0x20400, 0x20528,
3697                 0x20540, 0x20614,
3698                 0x21000, 0x21040,
3699                 0x2104c, 0x21060,
3700                 0x210c0, 0x210ec,
3701                 0x21200, 0x21268,
3702                 0x21270, 0x21284,
3703                 0x212fc, 0x21388,
3704                 0x21400, 0x21404,
3705                 0x21500, 0x21518,
3706                 0x2152c, 0x2153c,
3707                 0x21550, 0x21554,
3708                 0x21600, 0x21600,
3709                 0x21608, 0x21628,
3710                 0x21630, 0x2163c,
3711                 0x21700, 0x2171c,
3712                 0x21780, 0x2178c,
3713                 0x21800, 0x21c38,
3714                 0x21c80, 0x21d7c,
3715                 0x21e00, 0x21e04,
3716                 0x22000, 0x2202c,
3717                 0x22100, 0x2213c,
3718                 0x22190, 0x221c8,
3719                 0x22200, 0x22318,
3720                 0x22400, 0x22528,
3721                 0x22540, 0x22614,
3722                 0x23000, 0x23040,
3723                 0x2304c, 0x23060,
3724                 0x230c0, 0x230ec,
3725                 0x23200, 0x23268,
3726                 0x23270, 0x23284,
3727                 0x232fc, 0x23388,
3728                 0x23400, 0x23404,
3729                 0x23500, 0x23518,
3730                 0x2352c, 0x2353c,
3731                 0x23550, 0x23554,
3732                 0x23600, 0x23600,
3733                 0x23608, 0x23628,
3734                 0x23630, 0x2363c,
3735                 0x23700, 0x2371c,
3736                 0x23780, 0x2378c,
3737                 0x23800, 0x23c38,
3738                 0x23c80, 0x23d7c,
3739                 0x23e00, 0x23e04,
3740                 0x24000, 0x2402c,
3741                 0x24100, 0x2413c,
3742                 0x24190, 0x241c8,
3743                 0x24200, 0x24318,
3744                 0x24400, 0x24528,
3745                 0x24540, 0x24614,
3746                 0x25000, 0x25040,
3747                 0x2504c, 0x25060,
3748                 0x250c0, 0x250ec,
3749                 0x25200, 0x25268,
3750                 0x25270, 0x25284,
3751                 0x252fc, 0x25388,
3752                 0x25400, 0x25404,
3753                 0x25500, 0x25518,
3754                 0x2552c, 0x2553c,
3755                 0x25550, 0x25554,
3756                 0x25600, 0x25600,
3757                 0x25608, 0x25628,
3758                 0x25630, 0x2563c,
3759                 0x25700, 0x2571c,
3760                 0x25780, 0x2578c,
3761                 0x25800, 0x25c38,
3762                 0x25c80, 0x25d7c,
3763                 0x25e00, 0x25e04,
3764                 0x26000, 0x2602c,
3765                 0x26100, 0x2613c,
3766                 0x26190, 0x261c8,
3767                 0x26200, 0x26318,
3768                 0x26400, 0x26528,
3769                 0x26540, 0x26614,
3770                 0x27000, 0x27040,
3771                 0x2704c, 0x27060,
3772                 0x270c0, 0x270ec,
3773                 0x27200, 0x27268,
3774                 0x27270, 0x27284,
3775                 0x272fc, 0x27388,
3776                 0x27400, 0x27404,
3777                 0x27500, 0x27518,
3778                 0x2752c, 0x2753c,
3779                 0x27550, 0x27554,
3780                 0x27600, 0x27600,
3781                 0x27608, 0x27628,
3782                 0x27630, 0x2763c,
3783                 0x27700, 0x2771c,
3784                 0x27780, 0x2778c,
3785                 0x27800, 0x27c38,
3786                 0x27c80, 0x27d7c,
3787                 0x27e00, 0x27e04
3788         };
3789         static const unsigned int t5_reg_ranges[] = {
3790                 0x1008, 0x1148,
3791                 0x1180, 0x11b4,
3792                 0x11fc, 0x123c,
3793                 0x1280, 0x173c,
3794                 0x1800, 0x18fc,
3795                 0x3000, 0x3028,
3796                 0x3060, 0x30d8,
3797                 0x30e0, 0x30fc,
3798                 0x3140, 0x357c,
3799                 0x35a8, 0x35cc,
3800                 0x35ec, 0x35ec,
3801                 0x3600, 0x5624,
3802                 0x56cc, 0x575c,
3803                 0x580c, 0x5814,
3804                 0x5890, 0x58bc,
3805                 0x5940, 0x59dc,
3806                 0x59fc, 0x5a18,
3807                 0x5a60, 0x5a9c,
3808                 0x5b94, 0x5bfc,
3809                 0x6000, 0x6040,
3810                 0x6058, 0x614c,
3811                 0x7700, 0x7798,
3812                 0x77c0, 0x78fc,
3813                 0x7b00, 0x7c54,
3814                 0x7d00, 0x7efc,
3815                 0x8dc0, 0x8de0,
3816                 0x8df8, 0x8e84,
3817                 0x8ea0, 0x8f84,
3818                 0x8fc0, 0x90f8,
3819                 0x9400, 0x9470,
3820                 0x9600, 0x96f4,
3821                 0x9800, 0x9808,
3822                 0x9820, 0x983c,
3823                 0x9850, 0x9864,
3824                 0x9c00, 0x9c6c,
3825                 0x9c80, 0x9cec,
3826                 0x9d00, 0x9d6c,
3827                 0x9d80, 0x9dec,
3828                 0x9e00, 0x9e6c,
3829                 0x9e80, 0x9eec,
3830                 0x9f00, 0x9f6c,
3831                 0x9f80, 0xa020,
3832                 0xd004, 0xd03c,
3833                 0xdfc0, 0xdfe0,
3834                 0xe000, 0x11088,
3835                 0x1109c, 0x11110,
3836                 0x11118, 0x1117c,
3837                 0x11190, 0x11204,
3838                 0x19040, 0x1906c,
3839                 0x19078, 0x19080,
3840                 0x1908c, 0x19124,
3841                 0x19150, 0x191b0,
3842                 0x191d0, 0x191e8,
3843                 0x19238, 0x19290,
3844                 0x193f8, 0x19474,
3845                 0x19490, 0x194cc,
3846                 0x194f0, 0x194f8,
3847                 0x19c00, 0x19c60,
3848                 0x19c94, 0x19e10,
3849                 0x19e50, 0x19f34,
3850                 0x19f40, 0x19f50,
3851                 0x19f90, 0x19fe4,
3852                 0x1a000, 0x1a06c,
3853                 0x1a0b0, 0x1a120,
3854                 0x1a128, 0x1a138,
3855                 0x1a190, 0x1a1c4,
3856                 0x1a1fc, 0x1a1fc,
3857                 0x1e008, 0x1e00c,
3858                 0x1e040, 0x1e04c,
3859                 0x1e284, 0x1e290,
3860                 0x1e2c0, 0x1e2c0,
3861                 0x1e2e0, 0x1e2e0,
3862                 0x1e300, 0x1e384,
3863                 0x1e3c0, 0x1e3c8,
3864                 0x1e408, 0x1e40c,
3865                 0x1e440, 0x1e44c,
3866                 0x1e684, 0x1e690,
3867                 0x1e6c0, 0x1e6c0,
3868                 0x1e6e0, 0x1e6e0,
3869                 0x1e700, 0x1e784,
3870                 0x1e7c0, 0x1e7c8,
3871                 0x1e808, 0x1e80c,
3872                 0x1e840, 0x1e84c,
3873                 0x1ea84, 0x1ea90,
3874                 0x1eac0, 0x1eac0,
3875                 0x1eae0, 0x1eae0,
3876                 0x1eb00, 0x1eb84,
3877                 0x1ebc0, 0x1ebc8,
3878                 0x1ec08, 0x1ec0c,
3879                 0x1ec40, 0x1ec4c,
3880                 0x1ee84, 0x1ee90,
3881                 0x1eec0, 0x1eec0,
3882                 0x1eee0, 0x1eee0,
3883                 0x1ef00, 0x1ef84,
3884                 0x1efc0, 0x1efc8,
3885                 0x1f008, 0x1f00c,
3886                 0x1f040, 0x1f04c,
3887                 0x1f284, 0x1f290,
3888                 0x1f2c0, 0x1f2c0,
3889                 0x1f2e0, 0x1f2e0,
3890                 0x1f300, 0x1f384,
3891                 0x1f3c0, 0x1f3c8,
3892                 0x1f408, 0x1f40c,
3893                 0x1f440, 0x1f44c,
3894                 0x1f684, 0x1f690,
3895                 0x1f6c0, 0x1f6c0,
3896                 0x1f6e0, 0x1f6e0,
3897                 0x1f700, 0x1f784,
3898                 0x1f7c0, 0x1f7c8,
3899                 0x1f808, 0x1f80c,
3900                 0x1f840, 0x1f84c,
3901                 0x1fa84, 0x1fa90,
3902                 0x1fac0, 0x1fac0,
3903                 0x1fae0, 0x1fae0,
3904                 0x1fb00, 0x1fb84,
3905                 0x1fbc0, 0x1fbc8,
3906                 0x1fc08, 0x1fc0c,
3907                 0x1fc40, 0x1fc4c,
3908                 0x1fe84, 0x1fe90,
3909                 0x1fec0, 0x1fec0,
3910                 0x1fee0, 0x1fee0,
3911                 0x1ff00, 0x1ff84,
3912                 0x1ffc0, 0x1ffc8,
3913                 0x30000, 0x30030,
3914                 0x30100, 0x30144,
3915                 0x30190, 0x301d0,
3916                 0x30200, 0x30318,
3917                 0x30400, 0x3052c,
3918                 0x30540, 0x3061c,
3919                 0x30800, 0x30834,
3920                 0x308c0, 0x30908,
3921                 0x30910, 0x309ac,
3922                 0x30a00, 0x30a2c,
3923                 0x30a44, 0x30a50,
3924                 0x30a74, 0x30c24,
3925                 0x30d00, 0x30d00,
3926                 0x30d08, 0x30d14,
3927                 0x30d1c, 0x30d20,
3928                 0x30d3c, 0x30d50,
3929                 0x31200, 0x3120c,
3930                 0x31220, 0x31220,
3931                 0x31240, 0x31240,
3932                 0x31600, 0x3160c,
3933                 0x31a00, 0x31a1c,
3934                 0x31e00, 0x31e20,
3935                 0x31e38, 0x31e3c,
3936                 0x31e80, 0x31e80,
3937                 0x31e88, 0x31ea8,
3938                 0x31eb0, 0x31eb4,
3939                 0x31ec8, 0x31ed4,
3940                 0x31fb8, 0x32004,
3941                 0x32200, 0x32200,
3942                 0x32208, 0x32240,
3943                 0x32248, 0x32280,
3944                 0x32288, 0x322c0,
3945                 0x322c8, 0x322fc,
3946                 0x32600, 0x32630,
3947                 0x32a00, 0x32abc,
3948                 0x32b00, 0x32b70,
3949                 0x33000, 0x33048,
3950                 0x33060, 0x3309c,
3951                 0x330f0, 0x33148,
3952                 0x33160, 0x3319c,
3953                 0x331f0, 0x332e4,
3954                 0x332f8, 0x333e4,
3955                 0x333f8, 0x33448,
3956                 0x33460, 0x3349c,
3957                 0x334f0, 0x33548,
3958                 0x33560, 0x3359c,
3959                 0x335f0, 0x336e4,
3960                 0x336f8, 0x337e4,
3961                 0x337f8, 0x337fc,
3962                 0x33814, 0x33814,
3963                 0x3382c, 0x3382c,
3964                 0x33880, 0x3388c,
3965                 0x338e8, 0x338ec,
3966                 0x33900, 0x33948,
3967                 0x33960, 0x3399c,
3968                 0x339f0, 0x33ae4,
3969                 0x33af8, 0x33b10,
3970                 0x33b28, 0x33b28,
3971                 0x33b3c, 0x33b50,
3972                 0x33bf0, 0x33c10,
3973                 0x33c28, 0x33c28,
3974                 0x33c3c, 0x33c50,
3975                 0x33cf0, 0x33cfc,
3976                 0x34000, 0x34030,
3977                 0x34100, 0x34144,
3978                 0x34190, 0x341d0,
3979                 0x34200, 0x34318,
3980                 0x34400, 0x3452c,
3981                 0x34540, 0x3461c,
3982                 0x34800, 0x34834,
3983                 0x348c0, 0x34908,
3984                 0x34910, 0x349ac,
3985                 0x34a00, 0x34a2c,
3986                 0x34a44, 0x34a50,
3987                 0x34a74, 0x34c24,
3988                 0x34d00, 0x34d00,
3989                 0x34d08, 0x34d14,
3990                 0x34d1c, 0x34d20,
3991                 0x34d3c, 0x34d50,
3992                 0x35200, 0x3520c,
3993                 0x35220, 0x35220,
3994                 0x35240, 0x35240,
3995                 0x35600, 0x3560c,
3996                 0x35a00, 0x35a1c,
3997                 0x35e00, 0x35e20,
3998                 0x35e38, 0x35e3c,
3999                 0x35e80, 0x35e80,
4000                 0x35e88, 0x35ea8,
4001                 0x35eb0, 0x35eb4,
4002                 0x35ec8, 0x35ed4,
4003                 0x35fb8, 0x36004,
4004                 0x36200, 0x36200,
4005                 0x36208, 0x36240,
4006                 0x36248, 0x36280,
4007                 0x36288, 0x362c0,
4008                 0x362c8, 0x362fc,
4009                 0x36600, 0x36630,
4010                 0x36a00, 0x36abc,
4011                 0x36b00, 0x36b70,
4012                 0x37000, 0x37048,
4013                 0x37060, 0x3709c,
4014                 0x370f0, 0x37148,
4015                 0x37160, 0x3719c,
4016                 0x371f0, 0x372e4,
4017                 0x372f8, 0x373e4,
4018                 0x373f8, 0x37448,
4019                 0x37460, 0x3749c,
4020                 0x374f0, 0x37548,
4021                 0x37560, 0x3759c,
4022                 0x375f0, 0x376e4,
4023                 0x376f8, 0x377e4,
4024                 0x377f8, 0x377fc,
4025                 0x37814, 0x37814,
4026                 0x3782c, 0x3782c,
4027                 0x37880, 0x3788c,
4028                 0x378e8, 0x378ec,
4029                 0x37900, 0x37948,
4030                 0x37960, 0x3799c,
4031                 0x379f0, 0x37ae4,
4032                 0x37af8, 0x37b10,
4033                 0x37b28, 0x37b28,
4034                 0x37b3c, 0x37b50,
4035                 0x37bf0, 0x37c10,
4036                 0x37c28, 0x37c28,
4037                 0x37c3c, 0x37c50,
4038                 0x37cf0, 0x37cfc,
4039                 0x38000, 0x38030,
4040                 0x38100, 0x38144,
4041                 0x38190, 0x381d0,
4042                 0x38200, 0x38318,
4043                 0x38400, 0x3852c,
4044                 0x38540, 0x3861c,
4045                 0x38800, 0x38834,
4046                 0x388c0, 0x38908,
4047                 0x38910, 0x389ac,
4048                 0x38a00, 0x38a2c,
4049                 0x38a44, 0x38a50,
4050                 0x38a74, 0x38c24,
4051                 0x38d00, 0x38d00,
4052                 0x38d08, 0x38d14,
4053                 0x38d1c, 0x38d20,
4054                 0x38d3c, 0x38d50,
4055                 0x39200, 0x3920c,
4056                 0x39220, 0x39220,
4057                 0x39240, 0x39240,
4058                 0x39600, 0x3960c,
4059                 0x39a00, 0x39a1c,
4060                 0x39e00, 0x39e20,
4061                 0x39e38, 0x39e3c,
4062                 0x39e80, 0x39e80,
4063                 0x39e88, 0x39ea8,
4064                 0x39eb0, 0x39eb4,
4065                 0x39ec8, 0x39ed4,
4066                 0x39fb8, 0x3a004,
4067                 0x3a200, 0x3a200,
4068                 0x3a208, 0x3a240,
4069                 0x3a248, 0x3a280,
4070                 0x3a288, 0x3a2c0,
4071                 0x3a2c8, 0x3a2fc,
4072                 0x3a600, 0x3a630,
4073                 0x3aa00, 0x3aabc,
4074                 0x3ab00, 0x3ab70,
4075                 0x3b000, 0x3b048,
4076                 0x3b060, 0x3b09c,
4077                 0x3b0f0, 0x3b148,
4078                 0x3b160, 0x3b19c,
4079                 0x3b1f0, 0x3b2e4,
4080                 0x3b2f8, 0x3b3e4,
4081                 0x3b3f8, 0x3b448,
4082                 0x3b460, 0x3b49c,
4083                 0x3b4f0, 0x3b548,
4084                 0x3b560, 0x3b59c,
4085                 0x3b5f0, 0x3b6e4,
4086                 0x3b6f8, 0x3b7e4,
4087                 0x3b7f8, 0x3b7fc,
4088                 0x3b814, 0x3b814,
4089                 0x3b82c, 0x3b82c,
4090                 0x3b880, 0x3b88c,
4091                 0x3b8e8, 0x3b8ec,
4092                 0x3b900, 0x3b948,
4093                 0x3b960, 0x3b99c,
4094                 0x3b9f0, 0x3bae4,
4095                 0x3baf8, 0x3bb10,
4096                 0x3bb28, 0x3bb28,
4097                 0x3bb3c, 0x3bb50,
4098                 0x3bbf0, 0x3bc10,
4099                 0x3bc28, 0x3bc28,
4100                 0x3bc3c, 0x3bc50,
4101                 0x3bcf0, 0x3bcfc,
4102                 0x3c000, 0x3c030,
4103                 0x3c100, 0x3c144,
4104                 0x3c190, 0x3c1d0,
4105                 0x3c200, 0x3c318,
4106                 0x3c400, 0x3c52c,
4107                 0x3c540, 0x3c61c,
4108                 0x3c800, 0x3c834,
4109                 0x3c8c0, 0x3c908,
4110                 0x3c910, 0x3c9ac,
4111                 0x3ca00, 0x3ca2c,
4112                 0x3ca44, 0x3ca50,
4113                 0x3ca74, 0x3cc24,
4114                 0x3cd00, 0x3cd00,
4115                 0x3cd08, 0x3cd14,
4116                 0x3cd1c, 0x3cd20,
4117                 0x3cd3c, 0x3cd50,
4118                 0x3d200, 0x3d20c,
4119                 0x3d220, 0x3d220,
4120                 0x3d240, 0x3d240,
4121                 0x3d600, 0x3d60c,
4122                 0x3da00, 0x3da1c,
4123                 0x3de00, 0x3de20,
4124                 0x3de38, 0x3de3c,
4125                 0x3de80, 0x3de80,
4126                 0x3de88, 0x3dea8,
4127                 0x3deb0, 0x3deb4,
4128                 0x3dec8, 0x3ded4,
4129                 0x3dfb8, 0x3e004,
4130                 0x3e200, 0x3e200,
4131                 0x3e208, 0x3e240,
4132                 0x3e248, 0x3e280,
4133                 0x3e288, 0x3e2c0,
4134                 0x3e2c8, 0x3e2fc,
4135                 0x3e600, 0x3e630,
4136                 0x3ea00, 0x3eabc,
4137                 0x3eb00, 0x3eb70,
4138                 0x3f000, 0x3f048,
4139                 0x3f060, 0x3f09c,
4140                 0x3f0f0, 0x3f148,
4141                 0x3f160, 0x3f19c,
4142                 0x3f1f0, 0x3f2e4,
4143                 0x3f2f8, 0x3f3e4,
4144                 0x3f3f8, 0x3f448,
4145                 0x3f460, 0x3f49c,
4146                 0x3f4f0, 0x3f548,
4147                 0x3f560, 0x3f59c,
4148                 0x3f5f0, 0x3f6e4,
4149                 0x3f6f8, 0x3f7e4,
4150                 0x3f7f8, 0x3f7fc,
4151                 0x3f814, 0x3f814,
4152                 0x3f82c, 0x3f82c,
4153                 0x3f880, 0x3f88c,
4154                 0x3f8e8, 0x3f8ec,
4155                 0x3f900, 0x3f948,
4156                 0x3f960, 0x3f99c,
4157                 0x3f9f0, 0x3fae4,
4158                 0x3faf8, 0x3fb10,
4159                 0x3fb28, 0x3fb28,
4160                 0x3fb3c, 0x3fb50,
4161                 0x3fbf0, 0x3fc10,
4162                 0x3fc28, 0x3fc28,
4163                 0x3fc3c, 0x3fc50,
4164                 0x3fcf0, 0x3fcfc,
4165                 0x40000, 0x4000c,
4166                 0x40040, 0x40068,
4167                 0x4007c, 0x40144,
4168                 0x40180, 0x4018c,
4169                 0x40200, 0x40298,
4170                 0x402ac, 0x4033c,
4171                 0x403f8, 0x403fc,
4172                 0x41304, 0x413c4,
4173                 0x41400, 0x4141c,
4174                 0x41480, 0x414d0,
4175                 0x44000, 0x44078,
4176                 0x440c0, 0x44278,
4177                 0x442c0, 0x44478,
4178                 0x444c0, 0x44678,
4179                 0x446c0, 0x44878,
4180                 0x448c0, 0x449fc,
4181                 0x45000, 0x45068,
4182                 0x45080, 0x45084,
4183                 0x450a0, 0x450b0,
4184                 0x45200, 0x45268,
4185                 0x45280, 0x45284,
4186                 0x452a0, 0x452b0,
4187                 0x460c0, 0x460e4,
4188                 0x47000, 0x4708c,
4189                 0x47200, 0x47250,
4190                 0x47400, 0x47420,
4191                 0x47600, 0x47618,
4192                 0x47800, 0x47814,
4193                 0x48000, 0x4800c,
4194                 0x48040, 0x48068,
4195                 0x4807c, 0x48144,
4196                 0x48180, 0x4818c,
4197                 0x48200, 0x48298,
4198                 0x482ac, 0x4833c,
4199                 0x483f8, 0x483fc,
4200                 0x49304, 0x493c4,
4201                 0x49400, 0x4941c,
4202                 0x49480, 0x494d0,
4203                 0x4c000, 0x4c078,
4204                 0x4c0c0, 0x4c278,
4205                 0x4c2c0, 0x4c478,
4206                 0x4c4c0, 0x4c678,
4207                 0x4c6c0, 0x4c878,
4208                 0x4c8c0, 0x4c9fc,
4209                 0x4d000, 0x4d068,
4210                 0x4d080, 0x4d084,
4211                 0x4d0a0, 0x4d0b0,
4212                 0x4d200, 0x4d268,
4213                 0x4d280, 0x4d284,
4214                 0x4d2a0, 0x4d2b0,
4215                 0x4e0c0, 0x4e0e4,
4216                 0x4f000, 0x4f08c,
4217                 0x4f200, 0x4f250,
4218                 0x4f400, 0x4f420,
4219                 0x4f600, 0x4f618,
4220                 0x4f800, 0x4f814,
4221                 0x50000, 0x500cc,
4222                 0x50400, 0x50400,
4223                 0x50800, 0x508cc,
4224                 0x50c00, 0x50c00,
4225                 0x51000, 0x5101c,
4226                 0x51300, 0x51308,
4227         };
4228
4229         if (is_t4(sc)) {
4230                 reg_ranges = &t4_reg_ranges[0];
4231                 n = nitems(t4_reg_ranges);
4232         } else {
4233                 reg_ranges = &t5_reg_ranges[0];
4234                 n = nitems(t5_reg_ranges);
4235         }
4236
4237         regs->version = chip_id(sc) | chip_rev(sc) << 10;
4238         for (i = 0; i < n; i += 2)
4239                 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
4240 }
4241
4242 static void
4243 cxgbe_tick(void *arg)
4244 {
4245         struct port_info *pi = arg;
4246         struct adapter *sc = pi->adapter;
4247         struct ifnet *ifp = pi->ifp;
4248         struct sge_txq *txq;
4249         int i, drops;
4250         struct port_stats *s = &pi->stats;
4251
4252         PORT_LOCK(pi);
4253         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4254                 PORT_UNLOCK(pi);
4255                 return; /* without scheduling another callout */
4256         }
4257
4258         t4_get_port_stats(sc, pi->tx_chan, s);
4259
4260         ifp->if_opackets = s->tx_frames - s->tx_pause;
4261         ifp->if_ipackets = s->rx_frames - s->rx_pause;
4262         ifp->if_obytes = s->tx_octets - s->tx_pause * 64;
4263         ifp->if_ibytes = s->rx_octets - s->rx_pause * 64;
4264         ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause;
4265         ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause;
4266         ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
4267             s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
4268             s->rx_trunc3;
4269         for (i = 0; i < 4; i++) {
4270                 if (pi->rx_chan_map & (1 << i)) {
4271                         uint32_t v;
4272
4273                         /*
4274                          * XXX: indirect reads from the same ADDR/DATA pair can
4275                          * race with each other.
4276                          */
4277                         t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
4278                             1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4279                         ifp->if_iqdrops += v;
4280                 }
4281         }
4282
4283         drops = s->tx_drop;
4284         for_each_txq(pi, i, txq)
4285                 drops += txq->br->br_drops;
4286         ifp->if_snd.ifq_drops = drops;
4287
4288         ifp->if_oerrors = s->tx_error_frames;
4289         ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long +
4290             s->rx_fcs_err + s->rx_len_err;
4291
4292         callout_schedule(&pi->tick, hz);
4293         PORT_UNLOCK(pi);
4294 }
4295
4296 static void
4297 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4298 {
4299         struct ifnet *vlan;
4300
4301         if (arg != ifp || ifp->if_type != IFT_ETHER)
4302                 return;
4303
4304         vlan = VLAN_DEVAT(ifp, vid);
4305         VLAN_SETCOOKIE(vlan, ifp);
4306 }
4307
4308 static int
4309 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4310 {
4311
4312 #ifdef INVARIANTS
4313         panic("%s: opcode 0x%02x on iq %p with payload %p",
4314             __func__, rss->opcode, iq, m);
4315 #else
4316         log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
4317             __func__, rss->opcode, iq, m);
4318         m_freem(m);
4319 #endif
4320         return (EDOOFUS);
4321 }
4322
4323 int
4324 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
4325 {
4326         uintptr_t *loc, new;
4327
4328         if (opcode >= nitems(sc->cpl_handler))
4329                 return (EINVAL);
4330
4331         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
4332         loc = (uintptr_t *) &sc->cpl_handler[opcode];
4333         atomic_store_rel_ptr(loc, new);
4334
4335         return (0);
4336 }
4337
4338 static int
4339 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
4340 {
4341
4342 #ifdef INVARIANTS
4343         panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
4344 #else
4345         log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
4346             __func__, iq, ctrl);
4347 #endif
4348         return (EDOOFUS);
4349 }
4350
4351 int
4352 t4_register_an_handler(struct adapter *sc, an_handler_t h)
4353 {
4354         uintptr_t *loc, new;
4355
4356         new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
4357         loc = (uintptr_t *) &sc->an_handler;
4358         atomic_store_rel_ptr(loc, new);
4359
4360         return (0);
4361 }
4362
4363 static int
4364 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
4365 {
4366         const struct cpl_fw6_msg *cpl =
4367             __containerof(rpl, struct cpl_fw6_msg, data[0]);
4368
4369 #ifdef INVARIANTS
4370         panic("%s: fw_msg type %d", __func__, cpl->type);
4371 #else
4372         log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
4373 #endif
4374         return (EDOOFUS);
4375 }
4376
4377 int
4378 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
4379 {
4380         uintptr_t *loc, new;
4381
4382         if (type >= nitems(sc->fw_msg_handler))
4383                 return (EINVAL);
4384
4385         /*
4386          * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
4387          * handler dispatch table.  Reject any attempt to install a handler for
4388          * this subtype.
4389          */
4390         if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
4391                 return (EINVAL);
4392
4393         new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
4394         loc = (uintptr_t *) &sc->fw_msg_handler[type];
4395         atomic_store_rel_ptr(loc, new);
4396
4397         return (0);
4398 }
4399
4400 static int
4401 t4_sysctls(struct adapter *sc)
4402 {
4403         struct sysctl_ctx_list *ctx;
4404         struct sysctl_oid *oid;
4405         struct sysctl_oid_list *children, *c0;
4406         static char *caps[] = {
4407                 "\20\1PPP\2QFC\3DCBX",                  /* caps[0] linkcaps */
4408                 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL"        /* caps[1] niccaps */
4409                     "\6HASHFILTER\7ETHOFLD",
4410                 "\20\1TOE",                             /* caps[2] toecaps */
4411                 "\20\1RDDP\2RDMAC",                     /* caps[3] rdmacaps */
4412                 "\20\1INITIATOR_PDU\2TARGET_PDU"        /* caps[4] iscsicaps */
4413                     "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
4414                     "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
4415                 "\20\1INITIATOR\2TARGET\3CTRL_OFLD"     /* caps[5] fcoecaps */
4416                     "\4PO_INITIAOR\5PO_TARGET"
4417         };
4418         static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
4419
4420         ctx = device_get_sysctl_ctx(sc->dev);
4421
4422         /*
4423          * dev.t4nex.X.
4424          */
4425         oid = device_get_sysctl_tree(sc->dev);
4426         c0 = children = SYSCTL_CHILDREN(oid);
4427
4428         sc->sc_do_rxcopy = 1;
4429         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
4430             &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
4431
4432         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
4433             sc->params.nports, "# of ports");
4434
4435         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
4436             NULL, chip_rev(sc), "chip hardware revision");
4437
4438         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
4439             CTLFLAG_RD, sc->fw_version, 0, "firmware version");
4440
4441         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
4442             CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
4443
4444         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
4445             sc->cfcsum, "config file checksum");
4446
4447         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
4448             CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
4449             sysctl_bitfield, "A", "available doorbells");
4450
4451         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
4452             CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
4453             sysctl_bitfield, "A", "available link capabilities");
4454
4455         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
4456             CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
4457             sysctl_bitfield, "A", "available NIC capabilities");
4458
4459         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
4460             CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
4461             sysctl_bitfield, "A", "available TCP offload capabilities");
4462
4463         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
4464             CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
4465             sysctl_bitfield, "A", "available RDMA capabilities");
4466
4467         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
4468             CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
4469             sysctl_bitfield, "A", "available iSCSI capabilities");
4470
4471         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
4472             CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
4473             sysctl_bitfield, "A", "available FCoE capabilities");
4474
4475         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
4476             sc->params.vpd.cclk, "core clock frequency (in KHz)");
4477
4478         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
4479             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
4480             sizeof(sc->sge.timer_val), sysctl_int_array, "A",
4481             "interrupt holdoff timer values (us)");
4482
4483         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
4484             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
4485             sizeof(sc->sge.counter_val), sysctl_int_array, "A",
4486             "interrupt holdoff packet counter values");
4487
4488         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
4489             NULL, sc->tids.nftids, "number of filters");
4490
4491         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
4492             CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
4493             "chip temperature (in Celsius)");
4494
4495         t4_sge_sysctls(sc, ctx, children);
4496
4497         sc->lro_timeout = 100;
4498         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
4499             &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
4500
4501 #ifdef SBUF_DRAIN
4502         /*
4503          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
4504          */
4505         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
4506             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
4507             "logs and miscellaneous information");
4508         children = SYSCTL_CHILDREN(oid);
4509
4510         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
4511             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4512             sysctl_cctrl, "A", "congestion control");
4513
4514         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
4515             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4516             sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
4517
4518         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
4519             CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
4520             sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
4521
4522         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
4523             CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
4524             sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
4525
4526         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
4527             CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
4528             sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
4529
4530         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
4531             CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
4532             sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
4533
4534         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
4535             CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
4536             sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
4537
4538         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
4539             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4540             sysctl_cim_la, "A", "CIM logic analyzer");
4541
4542         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
4543             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4544             sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
4545
4546         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
4547             CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
4548             sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
4549
4550         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
4551             CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
4552             sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
4553
4554         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
4555             CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
4556             sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
4557
4558         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
4559             CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
4560             sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
4561
4562         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
4563             CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
4564             sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
4565
4566         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
4567             CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
4568             sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
4569
4570         if (is_t5(sc)) {
4571                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
4572                     CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
4573                     sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
4574
4575                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
4576                     CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
4577                     sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
4578         }
4579
4580         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
4581             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4582             sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
4583
4584         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
4585             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4586             sysctl_cim_qcfg, "A", "CIM queue configuration");
4587
4588         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
4589             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4590             sysctl_cpl_stats, "A", "CPL statistics");
4591
4592         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
4593             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4594             sysctl_ddp_stats, "A", "non-TCP DDP statistics");
4595
4596         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
4597             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4598             sysctl_devlog, "A", "firmware's device log");
4599
4600         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
4601             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4602             sysctl_fcoe_stats, "A", "FCoE statistics");
4603
4604         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
4605             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4606             sysctl_hw_sched, "A", "hardware scheduler ");
4607
4608         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
4609             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4610             sysctl_l2t, "A", "hardware L2 table");
4611
4612         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
4613             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4614             sysctl_lb_stats, "A", "loopback statistics");
4615
4616         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
4617             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4618             sysctl_meminfo, "A", "memory regions");
4619
4620         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
4621             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4622             sysctl_mps_tcam, "A", "MPS TCAM entries");
4623
4624         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
4625             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4626             sysctl_path_mtus, "A", "path MTUs");
4627
4628         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
4629             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4630             sysctl_pm_stats, "A", "PM statistics");
4631
4632         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
4633             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4634             sysctl_rdma_stats, "A", "RDMA statistics");
4635
4636         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
4637             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4638             sysctl_tcp_stats, "A", "TCP statistics");
4639
4640         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
4641             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4642             sysctl_tids, "A", "TID information");
4643
4644         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
4645             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4646             sysctl_tp_err_stats, "A", "TP error statistics");
4647
4648         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
4649             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4650             sysctl_tp_la, "A", "TP logic analyzer");
4651
4652         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
4653             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4654             sysctl_tx_rate, "A", "Tx rate");
4655
4656         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
4657             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4658             sysctl_ulprx_la, "A", "ULPRX logic analyzer");
4659
4660         if (is_t5(sc)) {
4661                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
4662                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4663                     sysctl_wcwr_stats, "A", "write combined work requests");
4664         }
4665 #endif
4666
4667 #ifdef TCP_OFFLOAD
4668         if (is_offload(sc)) {
4669                 /*
4670                  * dev.t4nex.X.toe.
4671                  */
4672                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
4673                     NULL, "TOE parameters");
4674                 children = SYSCTL_CHILDREN(oid);
4675
4676                 sc->tt.sndbuf = 256 * 1024;
4677                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
4678                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
4679
4680                 sc->tt.ddp = 0;
4681                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
4682                     &sc->tt.ddp, 0, "DDP allowed");
4683
4684                 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
4685                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
4686                     &sc->tt.indsz, 0, "DDP max indicate size allowed");
4687
4688                 sc->tt.ddp_thres =
4689                     G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
4690                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
4691                     &sc->tt.ddp_thres, 0, "DDP threshold");
4692
4693                 sc->tt.rx_coalesce = 1;
4694                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
4695                     CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
4696
4697                 sc->tt.tx_align = 1;
4698                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align",
4699                     CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload");
4700         }
4701 #endif
4702
4703
4704         return (0);
4705 }
4706
4707 static int
4708 cxgbe_sysctls(struct port_info *pi)
4709 {
4710         struct sysctl_ctx_list *ctx;
4711         struct sysctl_oid *oid;
4712         struct sysctl_oid_list *children;
4713         struct adapter *sc = pi->adapter;
4714
4715         ctx = device_get_sysctl_ctx(pi->dev);
4716
4717         /*
4718          * dev.cxgbe.X.
4719          */
4720         oid = device_get_sysctl_tree(pi->dev);
4721         children = SYSCTL_CHILDREN(oid);
4722
4723         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
4724            CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
4725         if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
4726                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
4727                     CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
4728                     "PHY temperature (in Celsius)");
4729                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
4730                     CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
4731                     "PHY firmware version");
4732         }
4733         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
4734             &pi->nrxq, 0, "# of rx queues");
4735         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
4736             &pi->ntxq, 0, "# of tx queues");
4737         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
4738             &pi->first_rxq, 0, "index of first rx queue");
4739         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
4740             &pi->first_txq, 0, "index of first tx queue");
4741         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
4742             CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU",
4743             "Reserve queue 0 for non-flowid packets");
4744
4745 #ifdef TCP_OFFLOAD
4746         if (is_offload(sc)) {
4747                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
4748                     &pi->nofldrxq, 0,
4749                     "# of rx queues for offloaded TCP connections");
4750                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
4751                     &pi->nofldtxq, 0,
4752                     "# of tx queues for offloaded TCP connections");
4753                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
4754                     CTLFLAG_RD, &pi->first_ofld_rxq, 0,
4755                     "index of first TOE rx queue");
4756                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
4757                     CTLFLAG_RD, &pi->first_ofld_txq, 0,
4758                     "index of first TOE tx queue");
4759         }
4760 #endif
4761 #ifdef DEV_NETMAP
4762         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
4763             &pi->nnmrxq, 0, "# of rx queues for netmap");
4764         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
4765             &pi->nnmtxq, 0, "# of tx queues for netmap");
4766         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
4767             CTLFLAG_RD, &pi->first_nm_rxq, 0,
4768             "index of first netmap rx queue");
4769         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
4770             CTLFLAG_RD, &pi->first_nm_txq, 0,
4771             "index of first netmap tx queue");
4772 #endif
4773
4774         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
4775             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
4776             "holdoff timer index");
4777         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
4778             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
4779             "holdoff packet counter index");
4780
4781         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
4782             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
4783             "rx queue size");
4784         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
4785             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
4786             "tx queue size");
4787
4788         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
4789             CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings,
4790             "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
4791
4792         /*
4793          * dev.cxgbe.X.stats.
4794          */
4795         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
4796             NULL, "port statistics");
4797         children = SYSCTL_CHILDREN(oid);
4798
4799 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
4800         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
4801             CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
4802             sysctl_handle_t4_reg64, "QU", desc)
4803
4804         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
4805             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
4806         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
4807             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
4808         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
4809             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
4810         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
4811             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
4812         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
4813             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
4814         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
4815             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
4816         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
4817             "# of tx frames in this range",
4818             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
4819         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
4820             "# of tx frames in this range",
4821             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
4822         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
4823             "# of tx frames in this range",
4824             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
4825         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
4826             "# of tx frames in this range",
4827             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
4828         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
4829             "# of tx frames in this range",
4830             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
4831         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
4832             "# of tx frames in this range",
4833             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
4834         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
4835             "# of tx frames in this range",
4836             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
4837         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
4838             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
4839         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
4840             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
4841         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
4842             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
4843         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
4844             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
4845         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
4846             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
4847         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
4848             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
4849         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
4850             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
4851         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
4852             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
4853         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
4854             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
4855         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
4856             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
4857
4858         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
4859             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
4860         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
4861             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
4862         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
4863             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
4864         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
4865             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
4866         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
4867             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
4868         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
4869             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
4870         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
4871             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
4872         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
4873             "# of frames received with bad FCS",
4874             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
4875         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
4876             "# of frames received with length error",
4877             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
4878         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
4879             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
4880         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
4881             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
4882         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
4883             "# of rx frames in this range",
4884             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
4885         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
4886             "# of rx frames in this range",
4887             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
4888         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
4889             "# of rx frames in this range",
4890             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
4891         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
4892             "# of rx frames in this range",
4893             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
4894         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
4895             "# of rx frames in this range",
4896             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
4897         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
4898             "# of rx frames in this range",
4899             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
4900         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
4901             "# of rx frames in this range",
4902             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
4903         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
4904             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
4905         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
4906             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
4907         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
4908             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
4909         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
4910             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
4911         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
4912             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
4913         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
4914             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
4915         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
4916             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
4917         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
4918             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
4919         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
4920             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
4921
4922 #undef SYSCTL_ADD_T4_REG64
4923
4924 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
4925         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
4926             &pi->stats.name, desc)
4927
4928         /* We get these from port_stats and they may be stale by upto 1s */
4929         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
4930             "# drops due to buffer-group 0 overflows");
4931         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
4932             "# drops due to buffer-group 1 overflows");
4933         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
4934             "# drops due to buffer-group 2 overflows");
4935         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
4936             "# drops due to buffer-group 3 overflows");
4937         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
4938             "# of buffer-group 0 truncated packets");
4939         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
4940             "# of buffer-group 1 truncated packets");
4941         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
4942             "# of buffer-group 2 truncated packets");
4943         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
4944             "# of buffer-group 3 truncated packets");
4945
4946 #undef SYSCTL_ADD_T4_PORTSTAT
4947
4948         return (0);
4949 }
4950
4951 static int
4952 sysctl_int_array(SYSCTL_HANDLER_ARGS)
4953 {
4954         int rc, *i, space = 0;
4955         struct sbuf sb;
4956
4957         sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
4958         for (i = arg1; arg2; arg2 -= sizeof(int), i++) {
4959                 if (space)
4960                         sbuf_printf(&sb, " ");
4961                 sbuf_printf(&sb, "%d", *i);
4962                 space = 1;
4963         }
4964         sbuf_finish(&sb);
4965         rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
4966         sbuf_delete(&sb);
4967         return (rc);
4968 }
4969
4970 static int
4971 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
4972 {
4973         int rc;
4974         struct sbuf *sb;
4975
4976         rc = sysctl_wire_old_buffer(req, 0);
4977         if (rc != 0)
4978                 return(rc);
4979
4980         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
4981         if (sb == NULL)
4982                 return (ENOMEM);
4983
4984         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
4985         rc = sbuf_finish(sb);
4986         sbuf_delete(sb);
4987
4988         return (rc);
4989 }
4990
4991 static int
4992 sysctl_btphy(SYSCTL_HANDLER_ARGS)
4993 {
4994         struct port_info *pi = arg1;
4995         int op = arg2;
4996         struct adapter *sc = pi->adapter;
4997         u_int v;
4998         int rc;
4999
5000         rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt");
5001         if (rc)
5002                 return (rc);
5003         /* XXX: magic numbers */
5004         rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
5005             &v);
5006         end_synchronized_op(sc, 0);
5007         if (rc)
5008                 return (rc);
5009         if (op == 0)
5010                 v /= 256;
5011
5012         rc = sysctl_handle_int(oidp, &v, 0, req);
5013         return (rc);
5014 }
5015
5016 static int
5017 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
5018 {
5019         struct port_info *pi = arg1;
5020         int rc, val;
5021
5022         val = pi->rsrv_noflowq;
5023         rc = sysctl_handle_int(oidp, &val, 0, req);
5024         if (rc != 0 || req->newptr == NULL)
5025                 return (rc);
5026
5027         if ((val >= 1) && (pi->ntxq > 1))
5028                 pi->rsrv_noflowq = 1;
5029         else
5030                 pi->rsrv_noflowq = 0;
5031
5032         return (rc);
5033 }
5034
5035 static int
5036 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
5037 {
5038         struct port_info *pi = arg1;
5039         struct adapter *sc = pi->adapter;
5040         int idx, rc, i;
5041         struct sge_rxq *rxq;
5042 #ifdef TCP_OFFLOAD
5043         struct sge_ofld_rxq *ofld_rxq;
5044 #endif
5045         uint8_t v;
5046
5047         idx = pi->tmr_idx;
5048
5049         rc = sysctl_handle_int(oidp, &idx, 0, req);
5050         if (rc != 0 || req->newptr == NULL)
5051                 return (rc);
5052
5053         if (idx < 0 || idx >= SGE_NTIMERS)
5054                 return (EINVAL);
5055
5056         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5057             "t4tmr");
5058         if (rc)
5059                 return (rc);
5060
5061         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
5062         for_each_rxq(pi, i, rxq) {
5063 #ifdef atomic_store_rel_8
5064                 atomic_store_rel_8(&rxq->iq.intr_params, v);
5065 #else
5066                 rxq->iq.intr_params = v;
5067 #endif
5068         }
5069 #ifdef TCP_OFFLOAD
5070         for_each_ofld_rxq(pi, i, ofld_rxq) {
5071 #ifdef atomic_store_rel_8
5072                 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
5073 #else
5074                 ofld_rxq->iq.intr_params = v;
5075 #endif
5076         }
5077 #endif
5078         pi->tmr_idx = idx;
5079
5080         end_synchronized_op(sc, LOCK_HELD);
5081         return (0);
5082 }
5083
5084 static int
5085 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
5086 {
5087         struct port_info *pi = arg1;
5088         struct adapter *sc = pi->adapter;
5089         int idx, rc;
5090
5091         idx = pi->pktc_idx;
5092
5093         rc = sysctl_handle_int(oidp, &idx, 0, req);
5094         if (rc != 0 || req->newptr == NULL)
5095                 return (rc);
5096
5097         if (idx < -1 || idx >= SGE_NCOUNTERS)
5098                 return (EINVAL);
5099
5100         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5101             "t4pktc");
5102         if (rc)
5103                 return (rc);
5104
5105         if (pi->flags & PORT_INIT_DONE)
5106                 rc = EBUSY; /* cannot be changed once the queues are created */
5107         else
5108                 pi->pktc_idx = idx;
5109
5110         end_synchronized_op(sc, LOCK_HELD);
5111         return (rc);
5112 }
5113
5114 static int
5115 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
5116 {
5117         struct port_info *pi = arg1;
5118         struct adapter *sc = pi->adapter;
5119         int qsize, rc;
5120
5121         qsize = pi->qsize_rxq;
5122
5123         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5124         if (rc != 0 || req->newptr == NULL)
5125                 return (rc);
5126
5127         if (qsize < 128 || (qsize & 7))
5128                 return (EINVAL);
5129
5130         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5131             "t4rxqs");
5132         if (rc)
5133                 return (rc);
5134
5135         if (pi->flags & PORT_INIT_DONE)
5136                 rc = EBUSY; /* cannot be changed once the queues are created */
5137         else
5138                 pi->qsize_rxq = qsize;
5139
5140         end_synchronized_op(sc, LOCK_HELD);
5141         return (rc);
5142 }
5143
5144 static int
5145 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
5146 {
5147         struct port_info *pi = arg1;
5148         struct adapter *sc = pi->adapter;
5149         int qsize, rc;
5150
5151         qsize = pi->qsize_txq;
5152
5153         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5154         if (rc != 0 || req->newptr == NULL)
5155                 return (rc);
5156
5157         /* bufring size must be powerof2 */
5158         if (qsize < 128 || !powerof2(qsize))
5159                 return (EINVAL);
5160
5161         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5162             "t4txqs");
5163         if (rc)
5164                 return (rc);
5165
5166         if (pi->flags & PORT_INIT_DONE)
5167                 rc = EBUSY; /* cannot be changed once the queues are created */
5168         else
5169                 pi->qsize_txq = qsize;
5170
5171         end_synchronized_op(sc, LOCK_HELD);
5172         return (rc);
5173 }
5174
5175 static int
5176 sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
5177 {
5178         struct port_info *pi = arg1;
5179         struct adapter *sc = pi->adapter;
5180         struct link_config *lc = &pi->link_cfg;
5181         int rc;
5182
5183         if (req->newptr == NULL) {
5184                 struct sbuf *sb;
5185                 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX";
5186
5187                 rc = sysctl_wire_old_buffer(req, 0);
5188                 if (rc != 0)
5189                         return(rc);
5190
5191                 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5192                 if (sb == NULL)
5193                         return (ENOMEM);
5194
5195                 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits);
5196                 rc = sbuf_finish(sb);
5197                 sbuf_delete(sb);
5198         } else {
5199                 char s[2];
5200                 int n;
5201
5202                 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX));
5203                 s[1] = 0;
5204
5205                 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
5206                 if (rc != 0)
5207                         return(rc);
5208
5209                 if (s[1] != 0)
5210                         return (EINVAL);
5211                 if (s[0] < '0' || s[0] > '9')
5212                         return (EINVAL);        /* not a number */
5213                 n = s[0] - '0';
5214                 if (n & ~(PAUSE_TX | PAUSE_RX))
5215                         return (EINVAL);        /* some other bit is set too */
5216
5217                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE");
5218                 if (rc)
5219                         return (rc);
5220                 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) {
5221                         int link_ok = lc->link_ok;
5222
5223                         lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX);
5224                         lc->requested_fc |= n;
5225                         rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc);
5226                         lc->link_ok = link_ok;  /* restore */
5227                 }
5228                 end_synchronized_op(sc, 0);
5229         }
5230
5231         return (rc);
5232 }
5233
5234 static int
5235 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
5236 {
5237         struct adapter *sc = arg1;
5238         int reg = arg2;
5239         uint64_t val;
5240
5241         val = t4_read_reg64(sc, reg);
5242
5243         return (sysctl_handle_64(oidp, &val, 0, req));
5244 }
5245
5246 static int
5247 sysctl_temperature(SYSCTL_HANDLER_ARGS)
5248 {
5249         struct adapter *sc = arg1;
5250         int rc, t;
5251         uint32_t param, val;
5252
5253         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
5254         if (rc)
5255                 return (rc);
5256         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5257             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5258             V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
5259         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
5260         end_synchronized_op(sc, 0);
5261         if (rc)
5262                 return (rc);
5263
5264         /* unknown is returned as 0 but we display -1 in that case */
5265         t = val == 0 ? -1 : val;
5266
5267         rc = sysctl_handle_int(oidp, &t, 0, req);
5268         return (rc);
5269 }
5270
5271 #ifdef SBUF_DRAIN
5272 static int
5273 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
5274 {
5275         struct adapter *sc = arg1;
5276         struct sbuf *sb;
5277         int rc, i;
5278         uint16_t incr[NMTUS][NCCTRL_WIN];
5279         static const char *dec_fac[] = {
5280                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
5281                 "0.9375"
5282         };
5283
5284         rc = sysctl_wire_old_buffer(req, 0);
5285         if (rc != 0)
5286                 return (rc);
5287
5288         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5289         if (sb == NULL)
5290                 return (ENOMEM);
5291
5292         t4_read_cong_tbl(sc, incr);
5293
5294         for (i = 0; i < NCCTRL_WIN; ++i) {
5295                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
5296                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
5297                     incr[5][i], incr[6][i], incr[7][i]);
5298                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
5299                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
5300                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
5301                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
5302         }
5303
5304         rc = sbuf_finish(sb);
5305         sbuf_delete(sb);
5306
5307         return (rc);
5308 }
5309
5310 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
5311         "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",   /* ibq's */
5312         "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
5313         "SGE0-RX", "SGE1-RX"    /* additional obq's (T5 onwards) */
5314 };
5315
5316 static int
5317 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
5318 {
5319         struct adapter *sc = arg1;
5320         struct sbuf *sb;
5321         int rc, i, n, qid = arg2;
5322         uint32_t *buf, *p;
5323         char *qtype;
5324         u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
5325
5326         KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
5327             ("%s: bad qid %d\n", __func__, qid));
5328
5329         if (qid < CIM_NUM_IBQ) {
5330                 /* inbound queue */
5331                 qtype = "IBQ";
5332                 n = 4 * CIM_IBQ_SIZE;
5333                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5334                 rc = t4_read_cim_ibq(sc, qid, buf, n);
5335         } else {
5336                 /* outbound queue */
5337                 qtype = "OBQ";
5338                 qid -= CIM_NUM_IBQ;
5339                 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
5340                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5341                 rc = t4_read_cim_obq(sc, qid, buf, n);
5342         }
5343
5344         if (rc < 0) {
5345                 rc = -rc;
5346                 goto done;
5347         }
5348         n = rc * sizeof(uint32_t);      /* rc has # of words actually read */
5349
5350         rc = sysctl_wire_old_buffer(req, 0);
5351         if (rc != 0)
5352                 goto done;
5353
5354         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5355         if (sb == NULL) {
5356                 rc = ENOMEM;
5357                 goto done;
5358         }
5359
5360         sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
5361         for (i = 0, p = buf; i < n; i += 16, p += 4)
5362                 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
5363                     p[2], p[3]);
5364
5365         rc = sbuf_finish(sb);
5366         sbuf_delete(sb);
5367 done:
5368         free(buf, M_CXGBE);
5369         return (rc);
5370 }
5371
5372 static int
5373 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
5374 {
5375         struct adapter *sc = arg1;
5376         u_int cfg;
5377         struct sbuf *sb;
5378         uint32_t *buf, *p;
5379         int rc;
5380
5381         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
5382         if (rc != 0)
5383                 return (rc);
5384
5385         rc = sysctl_wire_old_buffer(req, 0);
5386         if (rc != 0)
5387                 return (rc);
5388
5389         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5390         if (sb == NULL)
5391                 return (ENOMEM);
5392
5393         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
5394             M_ZERO | M_WAITOK);
5395
5396         rc = -t4_cim_read_la(sc, buf, NULL);
5397         if (rc != 0)
5398                 goto done;
5399
5400         sbuf_printf(sb, "Status   Data      PC%s",
5401             cfg & F_UPDBGLACAPTPCONLY ? "" :
5402             "     LS0Stat  LS0Addr             LS0Data");
5403
5404         KASSERT((sc->params.cim_la_size & 7) == 0,
5405             ("%s: p will walk off the end of buf", __func__));
5406
5407         for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
5408                 if (cfg & F_UPDBGLACAPTPCONLY) {
5409                         sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
5410                             p[6], p[7]);
5411                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
5412                             (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
5413                             p[4] & 0xff, p[5] >> 8);
5414                         sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
5415                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5416                             p[1] & 0xf, p[2] >> 4);
5417                 } else {
5418                         sbuf_printf(sb,
5419                             "\n  %02x   %x%07x %x%07x %08x %08x "
5420                             "%08x%08x%08x%08x",
5421                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5422                             p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
5423                             p[6], p[7]);
5424                 }
5425         }
5426
5427         rc = sbuf_finish(sb);
5428         sbuf_delete(sb);
5429 done:
5430         free(buf, M_CXGBE);
5431         return (rc);
5432 }
5433
5434 static int
5435 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
5436 {
5437         struct adapter *sc = arg1;
5438         u_int i;
5439         struct sbuf *sb;
5440         uint32_t *buf, *p;
5441         int rc;
5442
5443         rc = sysctl_wire_old_buffer(req, 0);
5444         if (rc != 0)
5445                 return (rc);
5446
5447         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5448         if (sb == NULL)
5449                 return (ENOMEM);
5450
5451         buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
5452             M_ZERO | M_WAITOK);
5453
5454         t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
5455         p = buf;
5456
5457         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5458                 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
5459                     p[1], p[0]);
5460         }
5461
5462         sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
5463         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5464                 sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
5465                     (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
5466                     (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
5467                     (p[1] >> 2) | ((p[2] & 3) << 30),
5468                     (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
5469                     p[0] & 1);
5470         }
5471
5472         rc = sbuf_finish(sb);
5473         sbuf_delete(sb);
5474         free(buf, M_CXGBE);
5475         return (rc);
5476 }
5477
5478 static int
5479 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
5480 {
5481         struct adapter *sc = arg1;
5482         u_int i;
5483         struct sbuf *sb;
5484         uint32_t *buf, *p;
5485         int rc;
5486
5487         rc = sysctl_wire_old_buffer(req, 0);
5488         if (rc != 0)
5489                 return (rc);
5490
5491         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5492         if (sb == NULL)
5493                 return (ENOMEM);
5494
5495         buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
5496             M_ZERO | M_WAITOK);
5497
5498         t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
5499         p = buf;
5500
5501         sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
5502         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5503                 sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
5504                     (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
5505                     p[4], p[3], p[2], p[1], p[0]);
5506         }
5507
5508         sbuf_printf(sb, "\n\nCntl ID               Data");
5509         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5510                 sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
5511                     (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
5512         }
5513
5514         rc = sbuf_finish(sb);
5515         sbuf_delete(sb);
5516         free(buf, M_CXGBE);
5517         return (rc);
5518 }
5519
5520 static int
5521 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
5522 {
5523         struct adapter *sc = arg1;
5524         struct sbuf *sb;
5525         int rc, i;
5526         uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5527         uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5528         uint16_t thres[CIM_NUM_IBQ];
5529         uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
5530         uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
5531         u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
5532
5533         if (is_t4(sc)) {
5534                 cim_num_obq = CIM_NUM_OBQ;
5535                 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
5536                 obq_rdaddr = A_UP_OBQ_0_REALADDR;
5537         } else {
5538                 cim_num_obq = CIM_NUM_OBQ_T5;
5539                 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
5540                 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
5541         }
5542         nq = CIM_NUM_IBQ + cim_num_obq;
5543
5544         rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
5545         if (rc == 0)
5546                 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
5547         if (rc != 0)
5548                 return (rc);
5549
5550         t4_read_cimq_cfg(sc, base, size, thres);
5551
5552         rc = sysctl_wire_old_buffer(req, 0);
5553         if (rc != 0)
5554                 return (rc);
5555
5556         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5557         if (sb == NULL)
5558                 return (ENOMEM);
5559
5560         sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
5561
5562         for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
5563                 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
5564                     qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
5565                     G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5566                     G_QUEREMFLITS(p[2]) * 16);
5567         for ( ; i < nq; i++, p += 4, wr += 2)
5568                 sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
5569                     base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
5570                     wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5571                     G_QUEREMFLITS(p[2]) * 16);
5572
5573         rc = sbuf_finish(sb);
5574         sbuf_delete(sb);
5575
5576         return (rc);
5577 }
5578
5579 static int
5580 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
5581 {
5582         struct adapter *sc = arg1;
5583         struct sbuf *sb;
5584         int rc;
5585         struct tp_cpl_stats stats;
5586
5587         rc = sysctl_wire_old_buffer(req, 0);
5588         if (rc != 0)
5589                 return (rc);
5590
5591         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5592         if (sb == NULL)
5593                 return (ENOMEM);
5594
5595         t4_tp_get_cpl_stats(sc, &stats);
5596
5597         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
5598             "channel 3\n");
5599         sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
5600                    stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
5601         sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
5602                    stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
5603
5604         rc = sbuf_finish(sb);
5605         sbuf_delete(sb);
5606
5607         return (rc);
5608 }
5609
5610 static int
5611 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
5612 {
5613         struct adapter *sc = arg1;
5614         struct sbuf *sb;
5615         int rc;
5616         struct tp_usm_stats stats;
5617
5618         rc = sysctl_wire_old_buffer(req, 0);
5619         if (rc != 0)
5620                 return(rc);
5621
5622         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5623         if (sb == NULL)
5624                 return (ENOMEM);
5625
5626         t4_get_usm_stats(sc, &stats);
5627
5628         sbuf_printf(sb, "Frames: %u\n", stats.frames);
5629         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
5630         sbuf_printf(sb, "Drops:  %u", stats.drops);
5631
5632         rc = sbuf_finish(sb);
5633         sbuf_delete(sb);
5634
5635         return (rc);
5636 }
5637
5638 const char *devlog_level_strings[] = {
5639         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
5640         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
5641         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
5642         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
5643         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
5644         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
5645 };
5646
5647 const char *devlog_facility_strings[] = {
5648         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
5649         [FW_DEVLOG_FACILITY_CF]         = "CF",
5650         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
5651         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
5652         [FW_DEVLOG_FACILITY_RES]        = "RES",
5653         [FW_DEVLOG_FACILITY_HW]         = "HW",
5654         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
5655         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
5656         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
5657         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
5658         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
5659         [FW_DEVLOG_FACILITY_VI]         = "VI",
5660         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
5661         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
5662         [FW_DEVLOG_FACILITY_TM]         = "TM",
5663         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
5664         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
5665         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
5666         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
5667         [FW_DEVLOG_FACILITY_RI]         = "RI",
5668         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
5669         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
5670         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
5671         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE"
5672 };
5673
5674 static int
5675 sysctl_devlog(SYSCTL_HANDLER_ARGS)
5676 {
5677         struct adapter *sc = arg1;
5678         struct devlog_params *dparams = &sc->params.devlog;
5679         struct fw_devlog_e *buf, *e;
5680         int i, j, rc, nentries, first = 0, m;
5681         struct sbuf *sb;
5682         uint64_t ftstamp = UINT64_MAX;
5683
5684         if (dparams->start == 0) {
5685                 dparams->memtype = FW_MEMTYPE_EDC0;
5686                 dparams->start = 0x84000;
5687                 dparams->size = 32768;
5688         }
5689
5690         nentries = dparams->size / sizeof(struct fw_devlog_e);
5691
5692         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
5693         if (buf == NULL)
5694                 return (ENOMEM);
5695
5696         m = fwmtype_to_hwmtype(dparams->memtype);
5697         rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
5698         if (rc != 0)
5699                 goto done;
5700
5701         for (i = 0; i < nentries; i++) {
5702                 e = &buf[i];
5703
5704                 if (e->timestamp == 0)
5705                         break;  /* end */
5706
5707                 e->timestamp = be64toh(e->timestamp);
5708                 e->seqno = be32toh(e->seqno);
5709                 for (j = 0; j < 8; j++)
5710                         e->params[j] = be32toh(e->params[j]);
5711
5712                 if (e->timestamp < ftstamp) {
5713                         ftstamp = e->timestamp;
5714                         first = i;
5715                 }
5716         }
5717
5718         if (buf[first].timestamp == 0)
5719                 goto done;      /* nothing in the log */
5720
5721         rc = sysctl_wire_old_buffer(req, 0);
5722         if (rc != 0)
5723                 goto done;
5724
5725         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5726         if (sb == NULL) {
5727                 rc = ENOMEM;
5728                 goto done;
5729         }
5730         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
5731             "Seq#", "Tstamp", "Level", "Facility", "Message");
5732
5733         i = first;
5734         do {
5735                 e = &buf[i];
5736                 if (e->timestamp == 0)
5737                         break;  /* end */
5738
5739                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
5740                     e->seqno, e->timestamp,
5741                     (e->level < nitems(devlog_level_strings) ?
5742                         devlog_level_strings[e->level] : "UNKNOWN"),
5743                     (e->facility < nitems(devlog_facility_strings) ?
5744                         devlog_facility_strings[e->facility] : "UNKNOWN"));
5745                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
5746                     e->params[2], e->params[3], e->params[4],
5747                     e->params[5], e->params[6], e->params[7]);
5748
5749                 if (++i == nentries)
5750                         i = 0;
5751         } while (i != first);
5752
5753         rc = sbuf_finish(sb);
5754         sbuf_delete(sb);
5755 done:
5756         free(buf, M_CXGBE);
5757         return (rc);
5758 }
5759
5760 static int
5761 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
5762 {
5763         struct adapter *sc = arg1;
5764         struct sbuf *sb;
5765         int rc;
5766         struct tp_fcoe_stats stats[4];
5767
5768         rc = sysctl_wire_old_buffer(req, 0);
5769         if (rc != 0)
5770                 return (rc);
5771
5772         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5773         if (sb == NULL)
5774                 return (ENOMEM);
5775
5776         t4_get_fcoe_stats(sc, 0, &stats[0]);
5777         t4_get_fcoe_stats(sc, 1, &stats[1]);
5778         t4_get_fcoe_stats(sc, 2, &stats[2]);
5779         t4_get_fcoe_stats(sc, 3, &stats[3]);
5780
5781         sbuf_printf(sb, "                   channel 0        channel 1        "
5782             "channel 2        channel 3\n");
5783         sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
5784             stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
5785             stats[3].octetsDDP);
5786         sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
5787             stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
5788         sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
5789             stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
5790             stats[3].framesDrop);
5791
5792         rc = sbuf_finish(sb);
5793         sbuf_delete(sb);
5794
5795         return (rc);
5796 }
5797
5798 static int
5799 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
5800 {
5801         struct adapter *sc = arg1;
5802         struct sbuf *sb;
5803         int rc, i;
5804         unsigned int map, kbps, ipg, mode;
5805         unsigned int pace_tab[NTX_SCHED];
5806
5807         rc = sysctl_wire_old_buffer(req, 0);
5808         if (rc != 0)
5809                 return (rc);
5810
5811         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5812         if (sb == NULL)
5813                 return (ENOMEM);
5814
5815         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
5816         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
5817         t4_read_pace_tbl(sc, pace_tab);
5818
5819         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
5820             "Class IPG (0.1 ns)   Flow IPG (us)");
5821
5822         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
5823                 t4_get_tx_sched(sc, i, &kbps, &ipg);
5824                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
5825                     (mode & (1 << i)) ? "flow" : "class", map & 3);
5826                 if (kbps)
5827                         sbuf_printf(sb, "%9u     ", kbps);
5828                 else
5829                         sbuf_printf(sb, " disabled     ");
5830
5831                 if (ipg)
5832                         sbuf_printf(sb, "%13u        ", ipg);
5833                 else
5834                         sbuf_printf(sb, "     disabled        ");
5835
5836                 if (pace_tab[i])
5837                         sbuf_printf(sb, "%10u", pace_tab[i]);
5838                 else
5839                         sbuf_printf(sb, "  disabled");
5840         }
5841
5842         rc = sbuf_finish(sb);
5843         sbuf_delete(sb);
5844
5845         return (rc);
5846 }
5847
5848 static int
5849 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
5850 {
5851         struct adapter *sc = arg1;
5852         struct sbuf *sb;
5853         int rc, i, j;
5854         uint64_t *p0, *p1;
5855         struct lb_port_stats s[2];
5856         static const char *stat_name[] = {
5857                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
5858                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
5859                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
5860                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
5861                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
5862                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
5863                 "BG2FramesTrunc:", "BG3FramesTrunc:"
5864         };
5865
5866         rc = sysctl_wire_old_buffer(req, 0);
5867         if (rc != 0)
5868                 return (rc);
5869
5870         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5871         if (sb == NULL)
5872                 return (ENOMEM);
5873
5874         memset(s, 0, sizeof(s));
5875
5876         for (i = 0; i < 4; i += 2) {
5877                 t4_get_lb_stats(sc, i, &s[0]);
5878                 t4_get_lb_stats(sc, i + 1, &s[1]);
5879
5880                 p0 = &s[0].octets;
5881                 p1 = &s[1].octets;
5882                 sbuf_printf(sb, "%s                       Loopback %u"
5883                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
5884
5885                 for (j = 0; j < nitems(stat_name); j++)
5886                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
5887                                    *p0++, *p1++);
5888         }
5889
5890         rc = sbuf_finish(sb);
5891         sbuf_delete(sb);
5892
5893         return (rc);
5894 }
5895
5896 static int
5897 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
5898 {
5899         int rc = 0;
5900         struct port_info *pi = arg1;
5901         struct sbuf *sb;
5902         static const char *linkdnreasons[] = {
5903                 "non-specific", "remote fault", "autoneg failed", "reserved3",
5904                 "PHY overheated", "unknown", "rx los", "reserved7"
5905         };
5906
5907         rc = sysctl_wire_old_buffer(req, 0);
5908         if (rc != 0)
5909                 return(rc);
5910         sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
5911         if (sb == NULL)
5912                 return (ENOMEM);
5913
5914         if (pi->linkdnrc < 0)
5915                 sbuf_printf(sb, "n/a");
5916         else if (pi->linkdnrc < nitems(linkdnreasons))
5917                 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
5918         else
5919                 sbuf_printf(sb, "%d", pi->linkdnrc);
5920
5921         rc = sbuf_finish(sb);
5922         sbuf_delete(sb);
5923
5924         return (rc);
5925 }
5926
5927 struct mem_desc {
5928         unsigned int base;
5929         unsigned int limit;
5930         unsigned int idx;
5931 };
5932
5933 static int
5934 mem_desc_cmp(const void *a, const void *b)
5935 {
5936         return ((const struct mem_desc *)a)->base -
5937                ((const struct mem_desc *)b)->base;
5938 }
5939
5940 static void
5941 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
5942     unsigned int to)
5943 {
5944         unsigned int size;
5945
5946         size = to - from + 1;
5947         if (size == 0)
5948                 return;
5949
5950         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
5951         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
5952 }
5953
5954 static int
5955 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
5956 {
5957         struct adapter *sc = arg1;
5958         struct sbuf *sb;
5959         int rc, i, n;
5960         uint32_t lo, hi, used, alloc;
5961         static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
5962         static const char *region[] = {
5963                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
5964                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
5965                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
5966                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
5967                 "RQUDP region:", "PBL region:", "TXPBL region:",
5968                 "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
5969                 "On-chip queues:"
5970         };
5971         struct mem_desc avail[4];
5972         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
5973         struct mem_desc *md = mem;
5974
5975         rc = sysctl_wire_old_buffer(req, 0);
5976         if (rc != 0)
5977                 return (rc);
5978
5979         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5980         if (sb == NULL)
5981                 return (ENOMEM);
5982
5983         for (i = 0; i < nitems(mem); i++) {
5984                 mem[i].limit = 0;
5985                 mem[i].idx = i;
5986         }
5987
5988         /* Find and sort the populated memory ranges */
5989         i = 0;
5990         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
5991         if (lo & F_EDRAM0_ENABLE) {
5992                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
5993                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
5994                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
5995                 avail[i].idx = 0;
5996                 i++;
5997         }
5998         if (lo & F_EDRAM1_ENABLE) {
5999                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
6000                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
6001                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
6002                 avail[i].idx = 1;
6003                 i++;
6004         }
6005         if (lo & F_EXT_MEM_ENABLE) {
6006                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
6007                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
6008                 avail[i].limit = avail[i].base +
6009                     (G_EXT_MEM_SIZE(hi) << 20);
6010                 avail[i].idx = is_t4(sc) ? 2 : 3;       /* Call it MC for T4 */
6011                 i++;
6012         }
6013         if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
6014                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
6015                 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
6016                 avail[i].limit = avail[i].base +
6017                     (G_EXT_MEM1_SIZE(hi) << 20);
6018                 avail[i].idx = 4;
6019                 i++;
6020         }
6021         if (!i)                                    /* no memory available */
6022                 return 0;
6023         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
6024
6025         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
6026         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
6027         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
6028         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
6029         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
6030         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
6031         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
6032         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
6033         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
6034
6035         /* the next few have explicit upper bounds */
6036         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
6037         md->limit = md->base - 1 +
6038                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
6039                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
6040         md++;
6041
6042         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
6043         md->limit = md->base - 1 +
6044                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
6045                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
6046         md++;
6047
6048         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6049                 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
6050                 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
6051                 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
6052         } else {
6053                 md->base = 0;
6054                 md->idx = nitems(region);  /* hide it */
6055         }
6056         md++;
6057
6058 #define ulp_region(reg) \
6059         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
6060         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
6061
6062         ulp_region(RX_ISCSI);
6063         ulp_region(RX_TDDP);
6064         ulp_region(TX_TPT);
6065         ulp_region(RX_STAG);
6066         ulp_region(RX_RQ);
6067         ulp_region(RX_RQUDP);
6068         ulp_region(RX_PBL);
6069         ulp_region(TX_PBL);
6070 #undef ulp_region
6071
6072         md->base = 0;
6073         md->idx = nitems(region);
6074         if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
6075                 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
6076                 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
6077                     A_SGE_DBVFIFO_SIZE))) << 2) - 1;
6078         }
6079         md++;
6080
6081         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
6082         md->limit = md->base + sc->tids.ntids - 1;
6083         md++;
6084         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
6085         md->limit = md->base + sc->tids.ntids - 1;
6086         md++;
6087
6088         md->base = sc->vres.ocq.start;
6089         if (sc->vres.ocq.size)
6090                 md->limit = md->base + sc->vres.ocq.size - 1;
6091         else
6092                 md->idx = nitems(region);  /* hide it */
6093         md++;
6094
6095         /* add any address-space holes, there can be up to 3 */
6096         for (n = 0; n < i - 1; n++)
6097                 if (avail[n].limit < avail[n + 1].base)
6098                         (md++)->base = avail[n].limit;
6099         if (avail[n].limit)
6100                 (md++)->base = avail[n].limit;
6101
6102         n = md - mem;
6103         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
6104
6105         for (lo = 0; lo < i; lo++)
6106                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
6107                                 avail[lo].limit - 1);
6108
6109         sbuf_printf(sb, "\n");
6110         for (i = 0; i < n; i++) {
6111                 if (mem[i].idx >= nitems(region))
6112                         continue;                        /* skip holes */
6113                 if (!mem[i].limit)
6114                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
6115                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
6116                                 mem[i].limit);
6117         }
6118
6119         sbuf_printf(sb, "\n");
6120         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
6121         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
6122         mem_region_show(sb, "uP RAM:", lo, hi);
6123
6124         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
6125         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
6126         mem_region_show(sb, "uP Extmem2:", lo, hi);
6127
6128         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
6129         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
6130                    G_PMRXMAXPAGE(lo),
6131                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
6132                    (lo & F_PMRXNUMCHN) ? 2 : 1);
6133
6134         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
6135         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
6136         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
6137                    G_PMTXMAXPAGE(lo),
6138                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
6139                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
6140         sbuf_printf(sb, "%u p-structs\n",
6141                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
6142
6143         for (i = 0; i < 4; i++) {
6144                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
6145                 if (is_t4(sc)) {
6146                         used = G_USED(lo);
6147                         alloc = G_ALLOC(lo);
6148                 } else {
6149                         used = G_T5_USED(lo);
6150                         alloc = G_T5_ALLOC(lo);
6151                 }
6152                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
6153                            i, used, alloc);
6154         }
6155         for (i = 0; i < 4; i++) {
6156                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
6157                 if (is_t4(sc)) {
6158                         used = G_USED(lo);
6159                         alloc = G_ALLOC(lo);
6160                 } else {
6161                         used = G_T5_USED(lo);
6162                         alloc = G_T5_ALLOC(lo);
6163                 }
6164                 sbuf_printf(sb,
6165                            "\nLoopback %d using %u pages out of %u allocated",
6166                            i, used, alloc);
6167         }
6168
6169         rc = sbuf_finish(sb);
6170         sbuf_delete(sb);
6171
6172         return (rc);
6173 }
6174
6175 static inline void
6176 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
6177 {
6178         *mask = x | y;
6179         y = htobe64(y);
6180         memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
6181 }
6182
6183 static int
6184 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
6185 {
6186         struct adapter *sc = arg1;
6187         struct sbuf *sb;
6188         int rc, i, n;
6189
6190         rc = sysctl_wire_old_buffer(req, 0);
6191         if (rc != 0)
6192                 return (rc);
6193
6194         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6195         if (sb == NULL)
6196                 return (ENOMEM);
6197
6198         sbuf_printf(sb,
6199             "Idx  Ethernet address     Mask     Vld Ports PF"
6200             "  VF              Replication             P0 P1 P2 P3  ML");
6201         n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
6202             NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
6203         for (i = 0; i < n; i++) {
6204                 uint64_t tcamx, tcamy, mask;
6205                 uint32_t cls_lo, cls_hi;
6206                 uint8_t addr[ETHER_ADDR_LEN];
6207
6208                 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
6209                 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
6210                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
6211                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
6212
6213                 if (tcamx & tcamy)
6214                         continue;
6215
6216                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
6217                 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
6218                            "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
6219                            addr[3], addr[4], addr[5], (uintmax_t)mask,
6220                            (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
6221                            G_PORTMAP(cls_hi), G_PF(cls_lo),
6222                            (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
6223
6224                 if (cls_lo & F_REPLICATE) {
6225                         struct fw_ldst_cmd ldst_cmd;
6226
6227                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
6228                         ldst_cmd.op_to_addrspace =
6229                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
6230                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
6231                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
6232                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
6233                         ldst_cmd.u.mps.fid_ctl =
6234                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
6235                                 V_FW_LDST_CMD_CTL(i));
6236
6237                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
6238                             "t4mps");
6239                         if (rc)
6240                                 break;
6241                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
6242                             sizeof(ldst_cmd), &ldst_cmd);
6243                         end_synchronized_op(sc, 0);
6244
6245                         if (rc != 0) {
6246                                 sbuf_printf(sb,
6247                                     " ------------ error %3u ------------", rc);
6248                                 rc = 0;
6249                         } else {
6250                                 sbuf_printf(sb, " %08x %08x %08x %08x",
6251                                     be32toh(ldst_cmd.u.mps.rplc127_96),
6252                                     be32toh(ldst_cmd.u.mps.rplc95_64),
6253                                     be32toh(ldst_cmd.u.mps.rplc63_32),
6254                                     be32toh(ldst_cmd.u.mps.rplc31_0));
6255                         }
6256                 } else
6257                         sbuf_printf(sb, "%36s", "");
6258
6259                 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
6260                     G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
6261                     G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
6262         }
6263
6264         if (rc)
6265                 (void) sbuf_finish(sb);
6266         else
6267                 rc = sbuf_finish(sb);
6268         sbuf_delete(sb);
6269
6270         return (rc);
6271 }
6272
6273 static int
6274 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
6275 {
6276         struct adapter *sc = arg1;
6277         struct sbuf *sb;
6278         int rc;
6279         uint16_t mtus[NMTUS];
6280
6281         rc = sysctl_wire_old_buffer(req, 0);
6282         if (rc != 0)
6283                 return (rc);
6284
6285         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6286         if (sb == NULL)
6287                 return (ENOMEM);
6288
6289         t4_read_mtu_tbl(sc, mtus, NULL);
6290
6291         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
6292             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
6293             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
6294             mtus[14], mtus[15]);
6295
6296         rc = sbuf_finish(sb);
6297         sbuf_delete(sb);
6298
6299         return (rc);
6300 }
6301
6302 static int
6303 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
6304 {
6305         struct adapter *sc = arg1;
6306         struct sbuf *sb;
6307         int rc, i;
6308         uint32_t cnt[PM_NSTATS];
6309         uint64_t cyc[PM_NSTATS];
6310         static const char *rx_stats[] = {
6311                 "Read:", "Write bypass:", "Write mem:", "Flush:"
6312         };
6313         static const char *tx_stats[] = {
6314                 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
6315         };
6316
6317         rc = sysctl_wire_old_buffer(req, 0);
6318         if (rc != 0)
6319                 return (rc);
6320
6321         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6322         if (sb == NULL)
6323                 return (ENOMEM);
6324
6325         t4_pmtx_get_stats(sc, cnt, cyc);
6326         sbuf_printf(sb, "                Tx pcmds             Tx bytes");
6327         for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
6328                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
6329                     cyc[i]);
6330
6331         t4_pmrx_get_stats(sc, cnt, cyc);
6332         sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
6333         for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
6334                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
6335                     cyc[i]);
6336
6337         rc = sbuf_finish(sb);
6338         sbuf_delete(sb);
6339
6340         return (rc);
6341 }
6342
6343 static int
6344 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
6345 {
6346         struct adapter *sc = arg1;
6347         struct sbuf *sb;
6348         int rc;
6349         struct tp_rdma_stats stats;
6350
6351         rc = sysctl_wire_old_buffer(req, 0);
6352         if (rc != 0)
6353                 return (rc);
6354
6355         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6356         if (sb == NULL)
6357                 return (ENOMEM);
6358
6359         t4_tp_get_rdma_stats(sc, &stats);
6360         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
6361         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
6362
6363         rc = sbuf_finish(sb);
6364         sbuf_delete(sb);
6365
6366         return (rc);
6367 }
6368
6369 static int
6370 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
6371 {
6372         struct adapter *sc = arg1;
6373         struct sbuf *sb;
6374         int rc;
6375         struct tp_tcp_stats v4, v6;
6376
6377         rc = sysctl_wire_old_buffer(req, 0);
6378         if (rc != 0)
6379                 return (rc);
6380
6381         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6382         if (sb == NULL)
6383                 return (ENOMEM);
6384
6385         t4_tp_get_tcp_stats(sc, &v4, &v6);
6386         sbuf_printf(sb,
6387             "                                IP                 IPv6\n");
6388         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
6389             v4.tcpOutRsts, v6.tcpOutRsts);
6390         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
6391             v4.tcpInSegs, v6.tcpInSegs);
6392         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
6393             v4.tcpOutSegs, v6.tcpOutSegs);
6394         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
6395             v4.tcpRetransSegs, v6.tcpRetransSegs);
6396
6397         rc = sbuf_finish(sb);
6398         sbuf_delete(sb);
6399
6400         return (rc);
6401 }
6402
6403 static int
6404 sysctl_tids(SYSCTL_HANDLER_ARGS)
6405 {
6406         struct adapter *sc = arg1;
6407         struct sbuf *sb;
6408         int rc;
6409         struct tid_info *t = &sc->tids;
6410
6411         rc = sysctl_wire_old_buffer(req, 0);
6412         if (rc != 0)
6413                 return (rc);
6414
6415         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6416         if (sb == NULL)
6417                 return (ENOMEM);
6418
6419         if (t->natids) {
6420                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
6421                     t->atids_in_use);
6422         }
6423
6424         if (t->ntids) {
6425                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6426                         uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
6427
6428                         if (b) {
6429                                 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
6430                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6431                                     t->ntids - 1);
6432                         } else {
6433                                 sbuf_printf(sb, "TID range: %u-%u",
6434                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6435                                     t->ntids - 1);
6436                         }
6437                 } else
6438                         sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
6439                 sbuf_printf(sb, ", in use: %u\n",
6440                     atomic_load_acq_int(&t->tids_in_use));
6441         }
6442
6443         if (t->nstids) {
6444                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
6445                     t->stid_base + t->nstids - 1, t->stids_in_use);
6446         }
6447
6448         if (t->nftids) {
6449                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
6450                     t->ftid_base + t->nftids - 1);
6451         }
6452
6453         if (t->netids) {
6454                 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
6455                     t->etid_base + t->netids - 1);
6456         }
6457
6458         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
6459             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
6460             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
6461
6462         rc = sbuf_finish(sb);
6463         sbuf_delete(sb);
6464
6465         return (rc);
6466 }
6467
6468 static int
6469 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
6470 {
6471         struct adapter *sc = arg1;
6472         struct sbuf *sb;
6473         int rc;
6474         struct tp_err_stats stats;
6475
6476         rc = sysctl_wire_old_buffer(req, 0);
6477         if (rc != 0)
6478                 return (rc);
6479
6480         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6481         if (sb == NULL)
6482                 return (ENOMEM);
6483
6484         t4_tp_get_err_stats(sc, &stats);
6485
6486         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
6487                       "channel 3\n");
6488         sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
6489             stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
6490             stats.macInErrs[3]);
6491         sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
6492             stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
6493             stats.hdrInErrs[3]);
6494         sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
6495             stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
6496             stats.tcpInErrs[3]);
6497         sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
6498             stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
6499             stats.tcp6InErrs[3]);
6500         sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
6501             stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
6502             stats.tnlCongDrops[3]);
6503         sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
6504             stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
6505             stats.tnlTxDrops[3]);
6506         sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
6507             stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
6508             stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
6509         sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
6510             stats.ofldChanDrops[0], stats.ofldChanDrops[1],
6511             stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
6512         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
6513             stats.ofldNoNeigh, stats.ofldCongDefer);
6514
6515         rc = sbuf_finish(sb);
6516         sbuf_delete(sb);
6517
6518         return (rc);
6519 }
6520
6521 struct field_desc {
6522         const char *name;
6523         u_int start;
6524         u_int width;
6525 };
6526
6527 static void
6528 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
6529 {
6530         char buf[32];
6531         int line_size = 0;
6532
6533         while (f->name) {
6534                 uint64_t mask = (1ULL << f->width) - 1;
6535                 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
6536                     ((uintmax_t)v >> f->start) & mask);
6537
6538                 if (line_size + len >= 79) {
6539                         line_size = 8;
6540                         sbuf_printf(sb, "\n        ");
6541                 }
6542                 sbuf_printf(sb, "%s ", buf);
6543                 line_size += len + 1;
6544                 f++;
6545         }
6546         sbuf_printf(sb, "\n");
6547 }
6548
6549 static struct field_desc tp_la0[] = {
6550         { "RcfOpCodeOut", 60, 4 },
6551         { "State", 56, 4 },
6552         { "WcfState", 52, 4 },
6553         { "RcfOpcSrcOut", 50, 2 },
6554         { "CRxError", 49, 1 },
6555         { "ERxError", 48, 1 },
6556         { "SanityFailed", 47, 1 },
6557         { "SpuriousMsg", 46, 1 },
6558         { "FlushInputMsg", 45, 1 },
6559         { "FlushInputCpl", 44, 1 },
6560         { "RssUpBit", 43, 1 },
6561         { "RssFilterHit", 42, 1 },
6562         { "Tid", 32, 10 },
6563         { "InitTcb", 31, 1 },
6564         { "LineNumber", 24, 7 },
6565         { "Emsg", 23, 1 },
6566         { "EdataOut", 22, 1 },
6567         { "Cmsg", 21, 1 },
6568         { "CdataOut", 20, 1 },
6569         { "EreadPdu", 19, 1 },
6570         { "CreadPdu", 18, 1 },
6571         { "TunnelPkt", 17, 1 },
6572         { "RcfPeerFin", 16, 1 },
6573         { "RcfReasonOut", 12, 4 },
6574         { "TxCchannel", 10, 2 },
6575         { "RcfTxChannel", 8, 2 },
6576         { "RxEchannel", 6, 2 },
6577         { "RcfRxChannel", 5, 1 },
6578         { "RcfDataOutSrdy", 4, 1 },
6579         { "RxDvld", 3, 1 },
6580         { "RxOoDvld", 2, 1 },
6581         { "RxCongestion", 1, 1 },
6582         { "TxCongestion", 0, 1 },
6583         { NULL }
6584 };
6585
6586 static struct field_desc tp_la1[] = {
6587         { "CplCmdIn", 56, 8 },
6588         { "CplCmdOut", 48, 8 },
6589         { "ESynOut", 47, 1 },
6590         { "EAckOut", 46, 1 },
6591         { "EFinOut", 45, 1 },
6592         { "ERstOut", 44, 1 },
6593         { "SynIn", 43, 1 },
6594         { "AckIn", 42, 1 },
6595         { "FinIn", 41, 1 },
6596         { "RstIn", 40, 1 },
6597         { "DataIn", 39, 1 },
6598         { "DataInVld", 38, 1 },
6599         { "PadIn", 37, 1 },
6600         { "RxBufEmpty", 36, 1 },
6601         { "RxDdp", 35, 1 },
6602         { "RxFbCongestion", 34, 1 },
6603         { "TxFbCongestion", 33, 1 },
6604         { "TxPktSumSrdy", 32, 1 },
6605         { "RcfUlpType", 28, 4 },
6606         { "Eread", 27, 1 },
6607         { "Ebypass", 26, 1 },
6608         { "Esave", 25, 1 },
6609         { "Static0", 24, 1 },
6610         { "Cread", 23, 1 },
6611         { "Cbypass", 22, 1 },
6612         { "Csave", 21, 1 },
6613         { "CPktOut", 20, 1 },
6614         { "RxPagePoolFull", 18, 2 },
6615         { "RxLpbkPkt", 17, 1 },
6616         { "TxLpbkPkt", 16, 1 },
6617         { "RxVfValid", 15, 1 },
6618         { "SynLearned", 14, 1 },
6619         { "SetDelEntry", 13, 1 },
6620         { "SetInvEntry", 12, 1 },
6621         { "CpcmdDvld", 11, 1 },
6622         { "CpcmdSave", 10, 1 },
6623         { "RxPstructsFull", 8, 2 },
6624         { "EpcmdDvld", 7, 1 },
6625         { "EpcmdFlush", 6, 1 },
6626         { "EpcmdTrimPrefix", 5, 1 },
6627         { "EpcmdTrimPostfix", 4, 1 },
6628         { "ERssIp4Pkt", 3, 1 },
6629         { "ERssIp6Pkt", 2, 1 },
6630         { "ERssTcpUdpPkt", 1, 1 },
6631         { "ERssFceFipPkt", 0, 1 },
6632         { NULL }
6633 };
6634
6635 static struct field_desc tp_la2[] = {
6636         { "CplCmdIn", 56, 8 },
6637         { "MpsVfVld", 55, 1 },
6638         { "MpsPf", 52, 3 },
6639         { "MpsVf", 44, 8 },
6640         { "SynIn", 43, 1 },
6641         { "AckIn", 42, 1 },
6642         { "FinIn", 41, 1 },
6643         { "RstIn", 40, 1 },
6644         { "DataIn", 39, 1 },
6645         { "DataInVld", 38, 1 },
6646         { "PadIn", 37, 1 },
6647         { "RxBufEmpty", 36, 1 },
6648         { "RxDdp", 35, 1 },
6649         { "RxFbCongestion", 34, 1 },
6650         { "TxFbCongestion", 33, 1 },
6651         { "TxPktSumSrdy", 32, 1 },
6652         { "RcfUlpType", 28, 4 },
6653         { "Eread", 27, 1 },
6654         { "Ebypass", 26, 1 },
6655         { "Esave", 25, 1 },
6656         { "Static0", 24, 1 },
6657         { "Cread", 23, 1 },
6658         { "Cbypass", 22, 1 },
6659         { "Csave", 21, 1 },
6660         { "CPktOut", 20, 1 },
6661         { "RxPagePoolFull", 18, 2 },
6662         { "RxLpbkPkt", 17, 1 },
6663         { "TxLpbkPkt", 16, 1 },
6664         { "RxVfValid", 15, 1 },
6665         { "SynLearned", 14, 1 },
6666         { "SetDelEntry", 13, 1 },
6667         { "SetInvEntry", 12, 1 },
6668         { "CpcmdDvld", 11, 1 },
6669         { "CpcmdSave", 10, 1 },
6670         { "RxPstructsFull", 8, 2 },
6671         { "EpcmdDvld", 7, 1 },
6672         { "EpcmdFlush", 6, 1 },
6673         { "EpcmdTrimPrefix", 5, 1 },
6674         { "EpcmdTrimPostfix", 4, 1 },
6675         { "ERssIp4Pkt", 3, 1 },
6676         { "ERssIp6Pkt", 2, 1 },
6677         { "ERssTcpUdpPkt", 1, 1 },
6678         { "ERssFceFipPkt", 0, 1 },
6679         { NULL }
6680 };
6681
6682 static void
6683 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
6684 {
6685
6686         field_desc_show(sb, *p, tp_la0);
6687 }
6688
6689 static void
6690 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
6691 {
6692
6693         if (idx)
6694                 sbuf_printf(sb, "\n");
6695         field_desc_show(sb, p[0], tp_la0);
6696         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6697                 field_desc_show(sb, p[1], tp_la0);
6698 }
6699
6700 static void
6701 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
6702 {
6703
6704         if (idx)
6705                 sbuf_printf(sb, "\n");
6706         field_desc_show(sb, p[0], tp_la0);
6707         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6708                 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
6709 }
6710
6711 static int
6712 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
6713 {
6714         struct adapter *sc = arg1;
6715         struct sbuf *sb;
6716         uint64_t *buf, *p;
6717         int rc;
6718         u_int i, inc;
6719         void (*show_func)(struct sbuf *, uint64_t *, int);
6720
6721         rc = sysctl_wire_old_buffer(req, 0);
6722         if (rc != 0)
6723                 return (rc);
6724
6725         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6726         if (sb == NULL)
6727                 return (ENOMEM);
6728
6729         buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
6730
6731         t4_tp_read_la(sc, buf, NULL);
6732         p = buf;
6733
6734         switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
6735         case 2:
6736                 inc = 2;
6737                 show_func = tp_la_show2;
6738                 break;
6739         case 3:
6740                 inc = 2;
6741                 show_func = tp_la_show3;
6742                 break;
6743         default:
6744                 inc = 1;
6745                 show_func = tp_la_show;
6746         }
6747
6748         for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
6749                 (*show_func)(sb, p, i);
6750
6751         rc = sbuf_finish(sb);
6752         sbuf_delete(sb);
6753         free(buf, M_CXGBE);
6754         return (rc);
6755 }
6756
6757 static int
6758 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
6759 {
6760         struct adapter *sc = arg1;
6761         struct sbuf *sb;
6762         int rc;
6763         u64 nrate[NCHAN], orate[NCHAN];
6764
6765         rc = sysctl_wire_old_buffer(req, 0);
6766         if (rc != 0)
6767                 return (rc);
6768
6769         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6770         if (sb == NULL)
6771                 return (ENOMEM);
6772
6773         t4_get_chan_txrate(sc, nrate, orate);
6774         sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
6775                  "channel 3\n");
6776         sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
6777             nrate[0], nrate[1], nrate[2], nrate[3]);
6778         sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
6779             orate[0], orate[1], orate[2], orate[3]);
6780
6781         rc = sbuf_finish(sb);
6782         sbuf_delete(sb);
6783
6784         return (rc);
6785 }
6786
6787 static int
6788 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
6789 {
6790         struct adapter *sc = arg1;
6791         struct sbuf *sb;
6792         uint32_t *buf, *p;
6793         int rc, i;
6794
6795         rc = sysctl_wire_old_buffer(req, 0);
6796         if (rc != 0)
6797                 return (rc);
6798
6799         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6800         if (sb == NULL)
6801                 return (ENOMEM);
6802
6803         buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
6804             M_ZERO | M_WAITOK);
6805
6806         t4_ulprx_read_la(sc, buf);
6807         p = buf;
6808
6809         sbuf_printf(sb, "      Pcmd        Type   Message"
6810             "                Data");
6811         for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
6812                 sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
6813                     p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
6814         }
6815
6816         rc = sbuf_finish(sb);
6817         sbuf_delete(sb);
6818         free(buf, M_CXGBE);
6819         return (rc);
6820 }
6821
6822 static int
6823 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
6824 {
6825         struct adapter *sc = arg1;
6826         struct sbuf *sb;
6827         int rc, v;
6828
6829         rc = sysctl_wire_old_buffer(req, 0);
6830         if (rc != 0)
6831                 return (rc);
6832
6833         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6834         if (sb == NULL)
6835                 return (ENOMEM);
6836
6837         v = t4_read_reg(sc, A_SGE_STAT_CFG);
6838         if (G_STATSOURCE_T5(v) == 7) {
6839                 if (G_STATMODE(v) == 0) {
6840                         sbuf_printf(sb, "total %d, incomplete %d",
6841                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6842                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6843                 } else if (G_STATMODE(v) == 1) {
6844                         sbuf_printf(sb, "total %d, data overflow %d",
6845                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6846                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6847                 }
6848         }
6849         rc = sbuf_finish(sb);
6850         sbuf_delete(sb);
6851
6852         return (rc);
6853 }
6854 #endif
6855
6856 static inline void
6857 txq_start(struct ifnet *ifp, struct sge_txq *txq)
6858 {
6859         struct buf_ring *br;
6860         struct mbuf *m;
6861
6862         TXQ_LOCK_ASSERT_OWNED(txq);
6863
6864         br = txq->br;
6865         m = txq->m ? txq->m : drbr_dequeue(ifp, br);
6866         if (m)
6867                 t4_eth_tx(ifp, txq, m);
6868 }
6869
6870 void
6871 t4_tx_callout(void *arg)
6872 {
6873         struct sge_eq *eq = arg;
6874         struct adapter *sc;
6875
6876         if (EQ_TRYLOCK(eq) == 0)
6877                 goto reschedule;
6878
6879         if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) {
6880                 EQ_UNLOCK(eq);
6881 reschedule:
6882                 if (__predict_true(!(eq->flags && EQ_DOOMED)))
6883                         callout_schedule(&eq->tx_callout, 1);
6884                 return;
6885         }
6886
6887         EQ_LOCK_ASSERT_OWNED(eq);
6888
6889         if (__predict_true((eq->flags & EQ_DOOMED) == 0)) {
6890
6891                 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
6892                         struct sge_txq *txq = arg;
6893                         struct port_info *pi = txq->ifp->if_softc;
6894
6895                         sc = pi->adapter;
6896                 } else {
6897                         struct sge_wrq *wrq = arg;
6898
6899                         sc = wrq->adapter;
6900                 }
6901
6902                 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task);
6903         }
6904
6905         EQ_UNLOCK(eq);
6906 }
6907
6908 void
6909 t4_tx_task(void *arg, int count)
6910 {
6911         struct sge_eq *eq = arg;
6912
6913         EQ_LOCK(eq);
6914         if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
6915                 struct sge_txq *txq = arg;
6916                 txq_start(txq->ifp, txq);
6917         } else {
6918                 struct sge_wrq *wrq = arg;
6919                 t4_wrq_tx_locked(wrq->adapter, wrq, NULL);
6920         }
6921         EQ_UNLOCK(eq);
6922 }
6923
6924 static uint32_t
6925 fconf_to_mode(uint32_t fconf)
6926 {
6927         uint32_t mode;
6928
6929         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
6930             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
6931
6932         if (fconf & F_FRAGMENTATION)
6933                 mode |= T4_FILTER_IP_FRAGMENT;
6934
6935         if (fconf & F_MPSHITTYPE)
6936                 mode |= T4_FILTER_MPS_HIT_TYPE;
6937
6938         if (fconf & F_MACMATCH)
6939                 mode |= T4_FILTER_MAC_IDX;
6940
6941         if (fconf & F_ETHERTYPE)
6942                 mode |= T4_FILTER_ETH_TYPE;
6943
6944         if (fconf & F_PROTOCOL)
6945                 mode |= T4_FILTER_IP_PROTO;
6946
6947         if (fconf & F_TOS)
6948                 mode |= T4_FILTER_IP_TOS;
6949
6950         if (fconf & F_VLAN)
6951                 mode |= T4_FILTER_VLAN;
6952
6953         if (fconf & F_VNIC_ID)
6954                 mode |= T4_FILTER_VNIC;
6955
6956         if (fconf & F_PORT)
6957                 mode |= T4_FILTER_PORT;
6958
6959         if (fconf & F_FCOE)
6960                 mode |= T4_FILTER_FCoE;
6961
6962         return (mode);
6963 }
6964
6965 static uint32_t
6966 mode_to_fconf(uint32_t mode)
6967 {
6968         uint32_t fconf = 0;
6969
6970         if (mode & T4_FILTER_IP_FRAGMENT)
6971                 fconf |= F_FRAGMENTATION;
6972
6973         if (mode & T4_FILTER_MPS_HIT_TYPE)
6974                 fconf |= F_MPSHITTYPE;
6975
6976         if (mode & T4_FILTER_MAC_IDX)
6977                 fconf |= F_MACMATCH;
6978
6979         if (mode & T4_FILTER_ETH_TYPE)
6980                 fconf |= F_ETHERTYPE;
6981
6982         if (mode & T4_FILTER_IP_PROTO)
6983                 fconf |= F_PROTOCOL;
6984
6985         if (mode & T4_FILTER_IP_TOS)
6986                 fconf |= F_TOS;
6987
6988         if (mode & T4_FILTER_VLAN)
6989                 fconf |= F_VLAN;
6990
6991         if (mode & T4_FILTER_VNIC)
6992                 fconf |= F_VNIC_ID;
6993
6994         if (mode & T4_FILTER_PORT)
6995                 fconf |= F_PORT;
6996
6997         if (mode & T4_FILTER_FCoE)
6998                 fconf |= F_FCOE;
6999
7000         return (fconf);
7001 }
7002
7003 static uint32_t
7004 fspec_to_fconf(struct t4_filter_specification *fs)
7005 {
7006         uint32_t fconf = 0;
7007
7008         if (fs->val.frag || fs->mask.frag)
7009                 fconf |= F_FRAGMENTATION;
7010
7011         if (fs->val.matchtype || fs->mask.matchtype)
7012                 fconf |= F_MPSHITTYPE;
7013
7014         if (fs->val.macidx || fs->mask.macidx)
7015                 fconf |= F_MACMATCH;
7016
7017         if (fs->val.ethtype || fs->mask.ethtype)
7018                 fconf |= F_ETHERTYPE;
7019
7020         if (fs->val.proto || fs->mask.proto)
7021                 fconf |= F_PROTOCOL;
7022
7023         if (fs->val.tos || fs->mask.tos)
7024                 fconf |= F_TOS;
7025
7026         if (fs->val.vlan_vld || fs->mask.vlan_vld)
7027                 fconf |= F_VLAN;
7028
7029         if (fs->val.vnic_vld || fs->mask.vnic_vld)
7030                 fconf |= F_VNIC_ID;
7031
7032         if (fs->val.iport || fs->mask.iport)
7033                 fconf |= F_PORT;
7034
7035         if (fs->val.fcoe || fs->mask.fcoe)
7036                 fconf |= F_FCOE;
7037
7038         return (fconf);
7039 }
7040
7041 static int
7042 get_filter_mode(struct adapter *sc, uint32_t *mode)
7043 {
7044         int rc;
7045         uint32_t fconf;
7046
7047         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7048             "t4getfm");
7049         if (rc)
7050                 return (rc);
7051
7052         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
7053             A_TP_VLAN_PRI_MAP);
7054
7055         if (sc->params.tp.vlan_pri_map != fconf) {
7056                 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
7057                     device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
7058                     fconf);
7059                 sc->params.tp.vlan_pri_map = fconf;
7060         }
7061
7062         *mode = fconf_to_mode(sc->params.tp.vlan_pri_map);
7063
7064         end_synchronized_op(sc, LOCK_HELD);
7065         return (0);
7066 }
7067
7068 static int
7069 set_filter_mode(struct adapter *sc, uint32_t mode)
7070 {
7071         uint32_t fconf;
7072         int rc;
7073
7074         fconf = mode_to_fconf(mode);
7075
7076         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7077             "t4setfm");
7078         if (rc)
7079                 return (rc);
7080
7081         if (sc->tids.ftids_in_use > 0) {
7082                 rc = EBUSY;
7083                 goto done;
7084         }
7085
7086 #ifdef TCP_OFFLOAD
7087         if (sc->offload_map) {
7088                 rc = EBUSY;
7089                 goto done;
7090         }
7091 #endif
7092
7093 #ifdef notyet
7094         rc = -t4_set_filter_mode(sc, fconf);
7095         if (rc == 0)
7096                 sc->filter_mode = fconf;
7097 #else
7098         rc = ENOTSUP;
7099 #endif
7100
7101 done:
7102         end_synchronized_op(sc, LOCK_HELD);
7103         return (rc);
7104 }
7105
7106 static inline uint64_t
7107 get_filter_hits(struct adapter *sc, uint32_t fid)
7108 {
7109         uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
7110         uint64_t hits;
7111
7112         memwin_info(sc, 0, &mw_base, NULL);
7113         off = position_memwin(sc, 0,
7114             tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
7115         if (is_t4(sc)) {
7116                 hits = t4_read_reg64(sc, mw_base + off + 16);
7117                 hits = be64toh(hits);
7118         } else {
7119                 hits = t4_read_reg(sc, mw_base + off + 24);
7120                 hits = be32toh(hits);
7121         }
7122
7123         return (hits);
7124 }
7125
7126 static int
7127 get_filter(struct adapter *sc, struct t4_filter *t)
7128 {
7129         int i, rc, nfilters = sc->tids.nftids;
7130         struct filter_entry *f;
7131
7132         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7133             "t4getf");
7134         if (rc)
7135                 return (rc);
7136
7137         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
7138             t->idx >= nfilters) {
7139                 t->idx = 0xffffffff;
7140                 goto done;
7141         }
7142
7143         f = &sc->tids.ftid_tab[t->idx];
7144         for (i = t->idx; i < nfilters; i++, f++) {
7145                 if (f->valid) {
7146                         t->idx = i;
7147                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
7148                         t->smtidx = f->smtidx;
7149                         if (f->fs.hitcnts)
7150                                 t->hits = get_filter_hits(sc, t->idx);
7151                         else
7152                                 t->hits = UINT64_MAX;
7153                         t->fs = f->fs;
7154
7155                         goto done;
7156                 }
7157         }
7158
7159         t->idx = 0xffffffff;
7160 done:
7161         end_synchronized_op(sc, LOCK_HELD);
7162         return (0);
7163 }
7164
7165 static int
7166 set_filter(struct adapter *sc, struct t4_filter *t)
7167 {
7168         unsigned int nfilters, nports;
7169         struct filter_entry *f;
7170         int i, rc;
7171
7172         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
7173         if (rc)
7174                 return (rc);
7175
7176         nfilters = sc->tids.nftids;
7177         nports = sc->params.nports;
7178
7179         if (nfilters == 0) {
7180                 rc = ENOTSUP;
7181                 goto done;
7182         }
7183
7184         if (!(sc->flags & FULL_INIT_DONE)) {
7185                 rc = EAGAIN;
7186                 goto done;
7187         }
7188
7189         if (t->idx >= nfilters) {
7190                 rc = EINVAL;
7191                 goto done;
7192         }
7193
7194         /* Validate against the global filter mode */
7195         if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
7196             sc->params.tp.vlan_pri_map) {
7197                 rc = E2BIG;
7198                 goto done;
7199         }
7200
7201         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
7202                 rc = EINVAL;
7203                 goto done;
7204         }
7205
7206         if (t->fs.val.iport >= nports) {
7207                 rc = EINVAL;
7208                 goto done;
7209         }
7210
7211         /* Can't specify an iq if not steering to it */
7212         if (!t->fs.dirsteer && t->fs.iq) {
7213                 rc = EINVAL;
7214                 goto done;
7215         }
7216
7217         /* IPv6 filter idx must be 4 aligned */
7218         if (t->fs.type == 1 &&
7219             ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
7220                 rc = EINVAL;
7221                 goto done;
7222         }
7223
7224         if (sc->tids.ftid_tab == NULL) {
7225                 KASSERT(sc->tids.ftids_in_use == 0,
7226                     ("%s: no memory allocated but filters_in_use > 0",
7227                     __func__));
7228
7229                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
7230                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
7231                 if (sc->tids.ftid_tab == NULL) {
7232                         rc = ENOMEM;
7233                         goto done;
7234                 }
7235                 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
7236         }
7237
7238         for (i = 0; i < 4; i++) {
7239                 f = &sc->tids.ftid_tab[t->idx + i];
7240
7241                 if (f->pending || f->valid) {
7242                         rc = EBUSY;
7243                         goto done;
7244                 }
7245                 if (f->locked) {
7246                         rc = EPERM;
7247                         goto done;
7248                 }
7249
7250                 if (t->fs.type == 0)
7251                         break;
7252         }
7253
7254         f = &sc->tids.ftid_tab[t->idx];
7255         f->fs = t->fs;
7256
7257         rc = set_filter_wr(sc, t->idx);
7258 done:
7259         end_synchronized_op(sc, 0);
7260
7261         if (rc == 0) {
7262                 mtx_lock(&sc->tids.ftid_lock);
7263                 for (;;) {
7264                         if (f->pending == 0) {
7265                                 rc = f->valid ? 0 : EIO;
7266                                 break;
7267                         }
7268
7269                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7270                             PCATCH, "t4setfw", 0)) {
7271                                 rc = EINPROGRESS;
7272                                 break;
7273                         }
7274                 }
7275                 mtx_unlock(&sc->tids.ftid_lock);
7276         }
7277         return (rc);
7278 }
7279
7280 static int
7281 del_filter(struct adapter *sc, struct t4_filter *t)
7282 {
7283         unsigned int nfilters;
7284         struct filter_entry *f;
7285         int rc;
7286
7287         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
7288         if (rc)
7289                 return (rc);
7290
7291         nfilters = sc->tids.nftids;
7292
7293         if (nfilters == 0) {
7294                 rc = ENOTSUP;
7295                 goto done;
7296         }
7297
7298         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
7299             t->idx >= nfilters) {
7300                 rc = EINVAL;
7301                 goto done;
7302         }
7303
7304         if (!(sc->flags & FULL_INIT_DONE)) {
7305                 rc = EAGAIN;
7306                 goto done;
7307         }
7308
7309         f = &sc->tids.ftid_tab[t->idx];
7310
7311         if (f->pending) {
7312                 rc = EBUSY;
7313                 goto done;
7314         }
7315         if (f->locked) {
7316                 rc = EPERM;
7317                 goto done;
7318         }
7319
7320         if (f->valid) {
7321                 t->fs = f->fs;  /* extra info for the caller */
7322                 rc = del_filter_wr(sc, t->idx);
7323         }
7324
7325 done:
7326         end_synchronized_op(sc, 0);
7327
7328         if (rc == 0) {
7329                 mtx_lock(&sc->tids.ftid_lock);
7330                 for (;;) {
7331                         if (f->pending == 0) {
7332                                 rc = f->valid ? EIO : 0;
7333                                 break;
7334                         }
7335
7336                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7337                             PCATCH, "t4delfw", 0)) {
7338                                 rc = EINPROGRESS;
7339                                 break;
7340                         }
7341                 }
7342                 mtx_unlock(&sc->tids.ftid_lock);
7343         }
7344
7345         return (rc);
7346 }
7347
7348 static void
7349 clear_filter(struct filter_entry *f)
7350 {
7351         if (f->l2t)
7352                 t4_l2t_release(f->l2t);
7353
7354         bzero(f, sizeof (*f));
7355 }
7356
7357 static int
7358 set_filter_wr(struct adapter *sc, int fidx)
7359 {
7360         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7361         struct wrqe *wr;
7362         struct fw_filter_wr *fwr;
7363         unsigned int ftid;
7364
7365         ASSERT_SYNCHRONIZED_OP(sc);
7366
7367         if (f->fs.newdmac || f->fs.newvlan) {
7368                 /* This filter needs an L2T entry; allocate one. */
7369                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
7370                 if (f->l2t == NULL)
7371                         return (EAGAIN);
7372                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
7373                     f->fs.dmac)) {
7374                         t4_l2t_release(f->l2t);
7375                         f->l2t = NULL;
7376                         return (ENOMEM);
7377                 }
7378         }
7379
7380         ftid = sc->tids.ftid_base + fidx;
7381
7382         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7383         if (wr == NULL)
7384                 return (ENOMEM);
7385
7386         fwr = wrtod(wr);
7387         bzero(fwr, sizeof (*fwr));
7388
7389         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
7390         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
7391         fwr->tid_to_iq =
7392             htobe32(V_FW_FILTER_WR_TID(ftid) |
7393                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
7394                 V_FW_FILTER_WR_NOREPLY(0) |
7395                 V_FW_FILTER_WR_IQ(f->fs.iq));
7396         fwr->del_filter_to_l2tix =
7397             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
7398                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
7399                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
7400                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
7401                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
7402                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
7403                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
7404                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
7405                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
7406                     f->fs.newvlan == VLAN_REWRITE) |
7407                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
7408                     f->fs.newvlan == VLAN_REWRITE) |
7409                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
7410                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
7411                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
7412                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
7413         fwr->ethtype = htobe16(f->fs.val.ethtype);
7414         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
7415         fwr->frag_to_ovlan_vldm =
7416             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
7417                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
7418                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
7419                 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
7420                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
7421                 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
7422         fwr->smac_sel = 0;
7423         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
7424             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
7425         fwr->maci_to_matchtypem =
7426             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
7427                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
7428                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
7429                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
7430                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
7431                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
7432                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
7433                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
7434         fwr->ptcl = f->fs.val.proto;
7435         fwr->ptclm = f->fs.mask.proto;
7436         fwr->ttyp = f->fs.val.tos;
7437         fwr->ttypm = f->fs.mask.tos;
7438         fwr->ivlan = htobe16(f->fs.val.vlan);
7439         fwr->ivlanm = htobe16(f->fs.mask.vlan);
7440         fwr->ovlan = htobe16(f->fs.val.vnic);
7441         fwr->ovlanm = htobe16(f->fs.mask.vnic);
7442         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
7443         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
7444         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
7445         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
7446         fwr->lp = htobe16(f->fs.val.dport);
7447         fwr->lpm = htobe16(f->fs.mask.dport);
7448         fwr->fp = htobe16(f->fs.val.sport);
7449         fwr->fpm = htobe16(f->fs.mask.sport);
7450         if (f->fs.newsmac)
7451                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
7452
7453         f->pending = 1;
7454         sc->tids.ftids_in_use++;
7455
7456         t4_wrq_tx(sc, wr);
7457         return (0);
7458 }
7459
7460 static int
7461 del_filter_wr(struct adapter *sc, int fidx)
7462 {
7463         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7464         struct wrqe *wr;
7465         struct fw_filter_wr *fwr;
7466         unsigned int ftid;
7467
7468         ftid = sc->tids.ftid_base + fidx;
7469
7470         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7471         if (wr == NULL)
7472                 return (ENOMEM);
7473         fwr = wrtod(wr);
7474         bzero(fwr, sizeof (*fwr));
7475
7476         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
7477
7478         f->pending = 1;
7479         t4_wrq_tx(sc, wr);
7480         return (0);
7481 }
7482
7483 int
7484 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
7485 {
7486         struct adapter *sc = iq->adapter;
7487         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
7488         unsigned int idx = GET_TID(rpl);
7489         unsigned int rc;
7490         struct filter_entry *f;
7491
7492         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
7493             rss->opcode));
7494
7495         if (is_ftid(sc, idx)) {
7496
7497                 idx -= sc->tids.ftid_base;
7498                 f = &sc->tids.ftid_tab[idx];
7499                 rc = G_COOKIE(rpl->cookie);
7500
7501                 mtx_lock(&sc->tids.ftid_lock);
7502                 if (rc == FW_FILTER_WR_FLT_ADDED) {
7503                         KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
7504                             __func__, idx));
7505                         f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
7506                         f->pending = 0;  /* asynchronous setup completed */
7507                         f->valid = 1;
7508                 } else {
7509                         if (rc != FW_FILTER_WR_FLT_DELETED) {
7510                                 /* Add or delete failed, display an error */
7511                                 log(LOG_ERR,
7512                                     "filter %u setup failed with error %u\n",
7513                                     idx, rc);
7514                         }
7515
7516                         clear_filter(f);
7517                         sc->tids.ftids_in_use--;
7518                 }
7519                 wakeup(&sc->tids.ftid_tab);
7520                 mtx_unlock(&sc->tids.ftid_lock);
7521         }
7522
7523         return (0);
7524 }
7525
7526 static int
7527 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
7528 {
7529         int rc;
7530
7531         if (cntxt->cid > M_CTXTQID)
7532                 return (EINVAL);
7533
7534         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
7535             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
7536                 return (EINVAL);
7537
7538         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
7539         if (rc)
7540                 return (rc);
7541
7542         if (sc->flags & FW_OK) {
7543                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
7544                     &cntxt->data[0]);
7545                 if (rc == 0)
7546                         goto done;
7547         }
7548
7549         /*
7550          * Read via firmware failed or wasn't even attempted.  Read directly via
7551          * the backdoor.
7552          */
7553         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
7554 done:
7555         end_synchronized_op(sc, 0);
7556         return (rc);
7557 }
7558
7559 static int
7560 load_fw(struct adapter *sc, struct t4_data *fw)
7561 {
7562         int rc;
7563         uint8_t *fw_data;
7564
7565         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
7566         if (rc)
7567                 return (rc);
7568
7569         if (sc->flags & FULL_INIT_DONE) {
7570                 rc = EBUSY;
7571                 goto done;
7572         }
7573
7574         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
7575         if (fw_data == NULL) {
7576                 rc = ENOMEM;
7577                 goto done;
7578         }
7579
7580         rc = copyin(fw->data, fw_data, fw->len);
7581         if (rc == 0)
7582                 rc = -t4_load_fw(sc, fw_data, fw->len);
7583
7584         free(fw_data, M_CXGBE);
7585 done:
7586         end_synchronized_op(sc, 0);
7587         return (rc);
7588 }
7589
7590 static int
7591 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
7592 {
7593         uint32_t addr, off, remaining, i, n;
7594         uint32_t *buf, *b;
7595         uint32_t mw_base, mw_aperture;
7596         int rc;
7597         uint8_t *dst;
7598
7599         rc = validate_mem_range(sc, mr->addr, mr->len);
7600         if (rc != 0)
7601                 return (rc);
7602
7603         memwin_info(sc, win, &mw_base, &mw_aperture);
7604         buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
7605         addr = mr->addr;
7606         remaining = mr->len;
7607         dst = (void *)mr->data;
7608
7609         while (remaining) {
7610                 off = position_memwin(sc, win, addr);
7611
7612                 /* number of bytes that we'll copy in the inner loop */
7613                 n = min(remaining, mw_aperture - off);
7614                 for (i = 0; i < n; i += 4)
7615                         *b++ = t4_read_reg(sc, mw_base + off + i);
7616
7617                 rc = copyout(buf, dst, n);
7618                 if (rc != 0)
7619                         break;
7620
7621                 b = buf;
7622                 dst += n;
7623                 remaining -= n;
7624                 addr += n;
7625         }
7626
7627         free(buf, M_CXGBE);
7628         return (rc);
7629 }
7630
7631 static int
7632 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
7633 {
7634         int rc;
7635
7636         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
7637                 return (EINVAL);
7638
7639         if (i2cd->len > sizeof(i2cd->data))
7640                 return (EFBIG);
7641
7642         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
7643         if (rc)
7644                 return (rc);
7645         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
7646             i2cd->offset, i2cd->len, &i2cd->data[0]);
7647         end_synchronized_op(sc, 0);
7648
7649         return (rc);
7650 }
7651
7652 static int
7653 in_range(int val, int lo, int hi)
7654 {
7655
7656         return (val < 0 || (val <= hi && val >= lo));
7657 }
7658
7659 static int
7660 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
7661 {
7662         int fw_subcmd, fw_type, rc;
7663
7664         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
7665         if (rc)
7666                 return (rc);
7667
7668         if (!(sc->flags & FULL_INIT_DONE)) {
7669                 rc = EAGAIN;
7670                 goto done;
7671         }
7672
7673         /*
7674          * Translate the cxgbetool parameters into T4 firmware parameters.  (The
7675          * sub-command and type are in common locations.)
7676          */
7677         if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
7678                 fw_subcmd = FW_SCHED_SC_CONFIG;
7679         else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
7680                 fw_subcmd = FW_SCHED_SC_PARAMS;
7681         else {
7682                 rc = EINVAL;
7683                 goto done;
7684         }
7685         if (p->type == SCHED_CLASS_TYPE_PACKET)
7686                 fw_type = FW_SCHED_TYPE_PKTSCHED;
7687         else {
7688                 rc = EINVAL;
7689                 goto done;
7690         }
7691
7692         if (fw_subcmd == FW_SCHED_SC_CONFIG) {
7693                 /* Vet our parameters ..*/
7694                 if (p->u.config.minmax < 0) {
7695                         rc = EINVAL;
7696                         goto done;
7697                 }
7698
7699                 /* And pass the request to the firmware ...*/
7700                 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1);
7701                 goto done;
7702         }
7703
7704         if (fw_subcmd == FW_SCHED_SC_PARAMS) {
7705                 int fw_level;
7706                 int fw_mode;
7707                 int fw_rateunit;
7708                 int fw_ratemode;
7709
7710                 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
7711                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
7712                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
7713                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
7714                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
7715                         fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
7716                 else {
7717                         rc = EINVAL;
7718                         goto done;
7719                 }
7720
7721                 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
7722                         fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
7723                 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
7724                         fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
7725                 else {
7726                         rc = EINVAL;
7727                         goto done;
7728                 }
7729
7730                 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
7731                         fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
7732                 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
7733                         fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
7734                 else {
7735                         rc = EINVAL;
7736                         goto done;
7737                 }
7738
7739                 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
7740                         fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
7741                 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
7742                         fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
7743                 else {
7744                         rc = EINVAL;
7745                         goto done;
7746                 }
7747
7748                 /* Vet our parameters ... */
7749                 if (!in_range(p->u.params.channel, 0, 3) ||
7750                     !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
7751                     !in_range(p->u.params.minrate, 0, 10000000) ||
7752                     !in_range(p->u.params.maxrate, 0, 10000000) ||
7753                     !in_range(p->u.params.weight, 0, 100)) {
7754                         rc = ERANGE;
7755                         goto done;
7756                 }
7757
7758                 /*
7759                  * Translate any unset parameters into the firmware's
7760                  * nomenclature and/or fail the call if the parameters
7761                  * are required ...
7762                  */
7763                 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
7764                     p->u.params.channel < 0 || p->u.params.cl < 0) {
7765                         rc = EINVAL;
7766                         goto done;
7767                 }
7768                 if (p->u.params.minrate < 0)
7769                         p->u.params.minrate = 0;
7770                 if (p->u.params.maxrate < 0) {
7771                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7772                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7773                                 rc = EINVAL;
7774                                 goto done;
7775                         } else
7776                                 p->u.params.maxrate = 0;
7777                 }
7778                 if (p->u.params.weight < 0) {
7779                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
7780                                 rc = EINVAL;
7781                                 goto done;
7782                         } else
7783                                 p->u.params.weight = 0;
7784                 }
7785                 if (p->u.params.pktsize < 0) {
7786                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7787                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7788                                 rc = EINVAL;
7789                                 goto done;
7790                         } else
7791                                 p->u.params.pktsize = 0;
7792                 }
7793
7794                 /* See what the firmware thinks of the request ... */
7795                 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
7796                     fw_rateunit, fw_ratemode, p->u.params.channel,
7797                     p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
7798                     p->u.params.weight, p->u.params.pktsize, 1);
7799                 goto done;
7800         }
7801
7802         rc = EINVAL;
7803 done:
7804         end_synchronized_op(sc, 0);
7805         return (rc);
7806 }
7807
7808 static int
7809 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
7810 {
7811         struct port_info *pi = NULL;
7812         struct sge_txq *txq;
7813         uint32_t fw_mnem, fw_queue, fw_class;
7814         int i, rc;
7815
7816         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
7817         if (rc)
7818                 return (rc);
7819
7820         if (!(sc->flags & FULL_INIT_DONE)) {
7821                 rc = EAGAIN;
7822                 goto done;
7823         }
7824
7825         if (p->port >= sc->params.nports) {
7826                 rc = EINVAL;
7827                 goto done;
7828         }
7829
7830         pi = sc->port[p->port];
7831         if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
7832                 rc = EINVAL;
7833                 goto done;
7834         }
7835
7836         /*
7837          * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
7838          * Scheduling Class in this case).
7839          */
7840         fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
7841             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
7842         fw_class = p->cl < 0 ? 0xffffffff : p->cl;
7843
7844         /*
7845          * If op.queue is non-negative, then we're only changing the scheduling
7846          * on a single specified TX queue.
7847          */
7848         if (p->queue >= 0) {
7849                 txq = &sc->sge.txq[pi->first_txq + p->queue];
7850                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7851                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7852                     &fw_class);
7853                 goto done;
7854         }
7855
7856         /*
7857          * Change the scheduling on all the TX queues for the
7858          * interface.
7859          */
7860         for_each_txq(pi, i, txq) {
7861                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7862                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7863                     &fw_class);
7864                 if (rc)
7865                         goto done;
7866         }
7867
7868         rc = 0;
7869 done:
7870         end_synchronized_op(sc, 0);
7871         return (rc);
7872 }
7873
7874 int
7875 t4_os_find_pci_capability(struct adapter *sc, int cap)
7876 {
7877         int i;
7878
7879         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
7880 }
7881
7882 int
7883 t4_os_pci_save_state(struct adapter *sc)
7884 {
7885         device_t dev;
7886         struct pci_devinfo *dinfo;
7887
7888         dev = sc->dev;
7889         dinfo = device_get_ivars(dev);
7890
7891         pci_cfg_save(dev, dinfo, 0);
7892         return (0);
7893 }
7894
7895 int
7896 t4_os_pci_restore_state(struct adapter *sc)
7897 {
7898         device_t dev;
7899         struct pci_devinfo *dinfo;
7900
7901         dev = sc->dev;
7902         dinfo = device_get_ivars(dev);
7903
7904         pci_cfg_restore(dev, dinfo);
7905         return (0);
7906 }
7907
7908 void
7909 t4_os_portmod_changed(const struct adapter *sc, int idx)
7910 {
7911         struct port_info *pi = sc->port[idx];
7912         static const char *mod_str[] = {
7913                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
7914         };
7915
7916         build_medialist(pi, &pi->media);
7917 #ifdef DEV_NETMAP
7918         build_medialist(pi, &pi->nm_media);
7919 #endif
7920
7921         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
7922                 if_printf(pi->ifp, "transceiver unplugged.\n");
7923         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
7924                 if_printf(pi->ifp, "unknown transceiver inserted.\n");
7925         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
7926                 if_printf(pi->ifp, "unsupported transceiver inserted.\n");
7927         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
7928                 if_printf(pi->ifp, "%s transceiver inserted.\n",
7929                     mod_str[pi->mod_type]);
7930         } else {
7931                 if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
7932                     pi->mod_type);
7933         }
7934 }
7935
7936 void
7937 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
7938 {
7939         struct port_info *pi = sc->port[idx];
7940         struct ifnet *ifp = pi->ifp;
7941
7942         if (link_stat) {
7943                 pi->linkdnrc = -1;
7944                 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
7945                 if_link_state_change(ifp, LINK_STATE_UP);
7946         } else {
7947                 if (reason >= 0)
7948                         pi->linkdnrc = reason;
7949                 if_link_state_change(ifp, LINK_STATE_DOWN);
7950         }
7951 }
7952
7953 void
7954 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
7955 {
7956         struct adapter *sc;
7957
7958         sx_slock(&t4_list_lock);
7959         SLIST_FOREACH(sc, &t4_list, link) {
7960                 /*
7961                  * func should not make any assumptions about what state sc is
7962                  * in - the only guarantee is that sc->sc_lock is a valid lock.
7963                  */
7964                 func(sc, arg);
7965         }
7966         sx_sunlock(&t4_list_lock);
7967 }
7968
7969 static int
7970 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
7971 {
7972        return (0);
7973 }
7974
7975 static int
7976 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
7977 {
7978        return (0);
7979 }
7980
7981 static int
7982 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
7983     struct thread *td)
7984 {
7985         int rc;
7986         struct adapter *sc = dev->si_drv1;
7987
7988         rc = priv_check(td, PRIV_DRIVER);
7989         if (rc != 0)
7990                 return (rc);
7991
7992         switch (cmd) {
7993         case CHELSIO_T4_GETREG: {
7994                 struct t4_reg *edata = (struct t4_reg *)data;
7995
7996                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
7997                         return (EFAULT);
7998
7999                 if (edata->size == 4)
8000                         edata->val = t4_read_reg(sc, edata->addr);
8001                 else if (edata->size == 8)
8002                         edata->val = t4_read_reg64(sc, edata->addr);
8003                 else
8004                         return (EINVAL);
8005
8006                 break;
8007         }
8008         case CHELSIO_T4_SETREG: {
8009                 struct t4_reg *edata = (struct t4_reg *)data;
8010
8011                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8012                         return (EFAULT);
8013
8014                 if (edata->size == 4) {
8015                         if (edata->val & 0xffffffff00000000)
8016                                 return (EINVAL);
8017                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
8018                 } else if (edata->size == 8)
8019                         t4_write_reg64(sc, edata->addr, edata->val);
8020                 else
8021                         return (EINVAL);
8022                 break;
8023         }
8024         case CHELSIO_T4_REGDUMP: {
8025                 struct t4_regdump *regs = (struct t4_regdump *)data;
8026                 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
8027                 uint8_t *buf;
8028
8029                 if (regs->len < reglen) {
8030                         regs->len = reglen; /* hint to the caller */
8031                         return (ENOBUFS);
8032                 }
8033
8034                 regs->len = reglen;
8035                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
8036                 t4_get_regs(sc, regs, buf);
8037                 rc = copyout(buf, regs->data, reglen);
8038                 free(buf, M_CXGBE);
8039                 break;
8040         }
8041         case CHELSIO_T4_GET_FILTER_MODE:
8042                 rc = get_filter_mode(sc, (uint32_t *)data);
8043                 break;
8044         case CHELSIO_T4_SET_FILTER_MODE:
8045                 rc = set_filter_mode(sc, *(uint32_t *)data);
8046                 break;
8047         case CHELSIO_T4_GET_FILTER:
8048                 rc = get_filter(sc, (struct t4_filter *)data);
8049                 break;
8050         case CHELSIO_T4_SET_FILTER:
8051                 rc = set_filter(sc, (struct t4_filter *)data);
8052                 break;
8053         case CHELSIO_T4_DEL_FILTER:
8054                 rc = del_filter(sc, (struct t4_filter *)data);
8055                 break;
8056         case CHELSIO_T4_GET_SGE_CONTEXT:
8057                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
8058                 break;
8059         case CHELSIO_T4_LOAD_FW:
8060                 rc = load_fw(sc, (struct t4_data *)data);
8061                 break;
8062         case CHELSIO_T4_GET_MEM:
8063                 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
8064                 break;
8065         case CHELSIO_T4_GET_I2C:
8066                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
8067                 break;
8068         case CHELSIO_T4_CLEAR_STATS: {
8069                 int i;
8070                 u_int port_id = *(uint32_t *)data;
8071                 struct port_info *pi;
8072
8073                 if (port_id >= sc->params.nports)
8074                         return (EINVAL);
8075                 pi = sc->port[port_id];
8076
8077                 /* MAC stats */
8078                 t4_clr_port_stats(sc, pi->tx_chan);
8079
8080                 if (pi->flags & PORT_INIT_DONE) {
8081                         struct sge_rxq *rxq;
8082                         struct sge_txq *txq;
8083                         struct sge_wrq *wrq;
8084
8085                         for_each_rxq(pi, i, rxq) {
8086 #if defined(INET) || defined(INET6)
8087                                 rxq->lro.lro_queued = 0;
8088                                 rxq->lro.lro_flushed = 0;
8089 #endif
8090                                 rxq->rxcsum = 0;
8091                                 rxq->vlan_extraction = 0;
8092                         }
8093
8094                         for_each_txq(pi, i, txq) {
8095                                 txq->txcsum = 0;
8096                                 txq->tso_wrs = 0;
8097                                 txq->vlan_insertion = 0;
8098                                 txq->imm_wrs = 0;
8099                                 txq->sgl_wrs = 0;
8100                                 txq->txpkt_wrs = 0;
8101                                 txq->txpkts_wrs = 0;
8102                                 txq->txpkts_pkts = 0;
8103                                 txq->br->br_drops = 0;
8104                                 txq->no_dmamap = 0;
8105                                 txq->no_desc = 0;
8106                         }
8107
8108 #ifdef TCP_OFFLOAD
8109                         /* nothing to clear for each ofld_rxq */
8110
8111                         for_each_ofld_txq(pi, i, wrq) {
8112                                 wrq->tx_wrs = 0;
8113                                 wrq->no_desc = 0;
8114                         }
8115 #endif
8116                         wrq = &sc->sge.ctrlq[pi->port_id];
8117                         wrq->tx_wrs = 0;
8118                         wrq->no_desc = 0;
8119                 }
8120                 break;
8121         }
8122         case CHELSIO_T4_SCHED_CLASS:
8123                 rc = set_sched_class(sc, (struct t4_sched_params *)data);
8124                 break;
8125         case CHELSIO_T4_SCHED_QUEUE:
8126                 rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
8127                 break;
8128         case CHELSIO_T4_GET_TRACER:
8129                 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
8130                 break;
8131         case CHELSIO_T4_SET_TRACER:
8132                 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
8133                 break;
8134         default:
8135                 rc = EINVAL;
8136         }
8137
8138         return (rc);
8139 }
8140
8141 #ifdef TCP_OFFLOAD
8142 void
8143 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask,
8144     const unsigned int *pgsz_order)
8145 {
8146         struct port_info *pi = ifp->if_softc;
8147         struct adapter *sc = pi->adapter;
8148
8149         t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
8150         t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |
8151                 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) |
8152                 V_HPZ3(pgsz_order[3]));
8153 }
8154
8155 static int
8156 toe_capability(struct port_info *pi, int enable)
8157 {
8158         int rc;
8159         struct adapter *sc = pi->adapter;
8160
8161         ASSERT_SYNCHRONIZED_OP(sc);
8162
8163         if (!is_offload(sc))
8164                 return (ENODEV);
8165
8166         if (enable) {
8167                 /*
8168                  * We need the port's queues around so that we're able to send
8169                  * and receive CPLs to/from the TOE even if the ifnet for this
8170                  * port has never been UP'd administratively.
8171                  */
8172                 if (!(pi->flags & PORT_INIT_DONE)) {
8173                         rc = cxgbe_init_synchronized(pi);
8174                         if (rc)
8175                                 return (rc);
8176                 }
8177
8178                 if (isset(&sc->offload_map, pi->port_id))
8179                         return (0);
8180
8181                 if (!(sc->flags & TOM_INIT_DONE)) {
8182                         rc = t4_activate_uld(sc, ULD_TOM);
8183                         if (rc == EAGAIN) {
8184                                 log(LOG_WARNING,
8185                                     "You must kldload t4_tom.ko before trying "
8186                                     "to enable TOE on a cxgbe interface.\n");
8187                         }
8188                         if (rc != 0)
8189                                 return (rc);
8190                         KASSERT(sc->tom_softc != NULL,
8191                             ("%s: TOM activated but softc NULL", __func__));
8192                         KASSERT(sc->flags & TOM_INIT_DONE,
8193                             ("%s: TOM activated but flag not set", __func__));
8194                 }
8195
8196                 setbit(&sc->offload_map, pi->port_id);
8197         } else {
8198                 if (!isset(&sc->offload_map, pi->port_id))
8199                         return (0);
8200
8201                 KASSERT(sc->flags & TOM_INIT_DONE,
8202                     ("%s: TOM never initialized?", __func__));
8203                 clrbit(&sc->offload_map, pi->port_id);
8204         }
8205
8206         return (0);
8207 }
8208
8209 /*
8210  * Add an upper layer driver to the global list.
8211  */
8212 int
8213 t4_register_uld(struct uld_info *ui)
8214 {
8215         int rc = 0;
8216         struct uld_info *u;
8217
8218         sx_xlock(&t4_uld_list_lock);
8219         SLIST_FOREACH(u, &t4_uld_list, link) {
8220             if (u->uld_id == ui->uld_id) {
8221                     rc = EEXIST;
8222                     goto done;
8223             }
8224         }
8225
8226         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
8227         ui->refcount = 0;
8228 done:
8229         sx_xunlock(&t4_uld_list_lock);
8230         return (rc);
8231 }
8232
8233 int
8234 t4_unregister_uld(struct uld_info *ui)
8235 {
8236         int rc = EINVAL;
8237         struct uld_info *u;
8238
8239         sx_xlock(&t4_uld_list_lock);
8240
8241         SLIST_FOREACH(u, &t4_uld_list, link) {
8242             if (u == ui) {
8243                     if (ui->refcount > 0) {
8244                             rc = EBUSY;
8245                             goto done;
8246                     }
8247
8248                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
8249                     rc = 0;
8250                     goto done;
8251             }
8252         }
8253 done:
8254         sx_xunlock(&t4_uld_list_lock);
8255         return (rc);
8256 }
8257
8258 int
8259 t4_activate_uld(struct adapter *sc, int id)
8260 {
8261         int rc = EAGAIN;
8262         struct uld_info *ui;
8263
8264         ASSERT_SYNCHRONIZED_OP(sc);
8265
8266         sx_slock(&t4_uld_list_lock);
8267
8268         SLIST_FOREACH(ui, &t4_uld_list, link) {
8269                 if (ui->uld_id == id) {
8270                         rc = ui->activate(sc);
8271                         if (rc == 0)
8272                                 ui->refcount++;
8273                         goto done;
8274                 }
8275         }
8276 done:
8277         sx_sunlock(&t4_uld_list_lock);
8278
8279         return (rc);
8280 }
8281
8282 int
8283 t4_deactivate_uld(struct adapter *sc, int id)
8284 {
8285         int rc = EINVAL;
8286         struct uld_info *ui;
8287
8288         ASSERT_SYNCHRONIZED_OP(sc);
8289
8290         sx_slock(&t4_uld_list_lock);
8291
8292         SLIST_FOREACH(ui, &t4_uld_list, link) {
8293                 if (ui->uld_id == id) {
8294                         rc = ui->deactivate(sc);
8295                         if (rc == 0)
8296                                 ui->refcount--;
8297                         goto done;
8298                 }
8299         }
8300 done:
8301         sx_sunlock(&t4_uld_list_lock);
8302
8303         return (rc);
8304 }
8305 #endif
8306
8307 /*
8308  * Come up with reasonable defaults for some of the tunables, provided they're
8309  * not set by the user (in which case we'll use the values as is).
8310  */
8311 static void
8312 tweak_tunables(void)
8313 {
8314         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
8315
8316         if (t4_ntxq10g < 1)
8317                 t4_ntxq10g = min(nc, NTXQ_10G);
8318
8319         if (t4_ntxq1g < 1)
8320                 t4_ntxq1g = min(nc, NTXQ_1G);
8321
8322         if (t4_nrxq10g < 1)
8323                 t4_nrxq10g = min(nc, NRXQ_10G);
8324
8325         if (t4_nrxq1g < 1)
8326                 t4_nrxq1g = min(nc, NRXQ_1G);
8327
8328 #ifdef TCP_OFFLOAD
8329         if (t4_nofldtxq10g < 1)
8330                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
8331
8332         if (t4_nofldtxq1g < 1)
8333                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
8334
8335         if (t4_nofldrxq10g < 1)
8336                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
8337
8338         if (t4_nofldrxq1g < 1)
8339                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
8340
8341         if (t4_toecaps_allowed == -1)
8342                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
8343 #else
8344         if (t4_toecaps_allowed == -1)
8345                 t4_toecaps_allowed = 0;
8346 #endif
8347
8348 #ifdef DEV_NETMAP
8349         if (t4_nnmtxq10g < 1)
8350                 t4_nnmtxq10g = min(nc, NNMTXQ_10G);
8351
8352         if (t4_nnmtxq1g < 1)
8353                 t4_nnmtxq1g = min(nc, NNMTXQ_1G);
8354
8355         if (t4_nnmrxq10g < 1)
8356                 t4_nnmrxq10g = min(nc, NNMRXQ_10G);
8357
8358         if (t4_nnmrxq1g < 1)
8359                 t4_nnmrxq1g = min(nc, NNMRXQ_1G);
8360 #endif
8361
8362         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
8363                 t4_tmr_idx_10g = TMR_IDX_10G;
8364
8365         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
8366                 t4_pktc_idx_10g = PKTC_IDX_10G;
8367
8368         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
8369                 t4_tmr_idx_1g = TMR_IDX_1G;
8370
8371         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
8372                 t4_pktc_idx_1g = PKTC_IDX_1G;
8373
8374         if (t4_qsize_txq < 128)
8375                 t4_qsize_txq = 128;
8376
8377         if (t4_qsize_rxq < 128)
8378                 t4_qsize_rxq = 128;
8379         while (t4_qsize_rxq & 7)
8380                 t4_qsize_rxq++;
8381
8382         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
8383 }
8384
8385 static struct sx mlu;   /* mod load unload */
8386 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
8387
8388 static int
8389 mod_event(module_t mod, int cmd, void *arg)
8390 {
8391         int rc = 0;
8392         static int loaded = 0;
8393
8394         switch (cmd) {
8395         case MOD_LOAD:
8396                 sx_xlock(&mlu);
8397                 if (loaded++ == 0) {
8398                         t4_sge_modload();
8399                         sx_init(&t4_list_lock, "T4/T5 adapters");
8400                         SLIST_INIT(&t4_list);
8401 #ifdef TCP_OFFLOAD
8402                         sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
8403                         SLIST_INIT(&t4_uld_list);
8404 #endif
8405                         t4_tracer_modload();
8406                         tweak_tunables();
8407                 }
8408                 sx_xunlock(&mlu);
8409                 break;
8410
8411         case MOD_UNLOAD:
8412                 sx_xlock(&mlu);
8413                 if (--loaded == 0) {
8414                         int tries;
8415
8416                         sx_slock(&t4_list_lock);
8417                         if (!SLIST_EMPTY(&t4_list)) {
8418                                 rc = EBUSY;
8419                                 sx_sunlock(&t4_list_lock);
8420                                 goto done_unload;
8421                         }
8422 #ifdef TCP_OFFLOAD
8423                         sx_slock(&t4_uld_list_lock);
8424                         if (!SLIST_EMPTY(&t4_uld_list)) {
8425                                 rc = EBUSY;
8426                                 sx_sunlock(&t4_uld_list_lock);
8427                                 sx_sunlock(&t4_list_lock);
8428                                 goto done_unload;
8429                         }
8430 #endif
8431                         tries = 0;
8432                         while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
8433                                 uprintf("%ju clusters with custom free routine "
8434                                     "still is use.\n", t4_sge_extfree_refs());
8435                                 pause("t4unload", 2 * hz);
8436                         }
8437 #ifdef TCP_OFFLOAD
8438                         sx_sunlock(&t4_uld_list_lock);
8439 #endif
8440                         sx_sunlock(&t4_list_lock);
8441
8442                         if (t4_sge_extfree_refs() == 0) {
8443                                 t4_tracer_modunload();
8444 #ifdef TCP_OFFLOAD
8445                                 sx_destroy(&t4_uld_list_lock);
8446 #endif
8447                                 sx_destroy(&t4_list_lock);
8448                                 t4_sge_modunload();
8449                                 loaded = 0;
8450                         } else {
8451                                 rc = EBUSY;
8452                                 loaded++;       /* undo earlier decrement */
8453                         }
8454                 }
8455 done_unload:
8456                 sx_xunlock(&mlu);
8457                 break;
8458         }
8459
8460         return (rc);
8461 }
8462
8463 static devclass_t t4_devclass, t5_devclass;
8464 static devclass_t cxgbe_devclass, cxl_devclass;
8465
8466 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
8467 MODULE_VERSION(t4nex, 1);
8468 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
8469
8470 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
8471 MODULE_VERSION(t5nex, 1);
8472 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
8473
8474 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
8475 MODULE_VERSION(cxgbe, 1);
8476
8477 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
8478 MODULE_VERSION(cxl, 1);