sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/conf.h>
  36 #include <sys/priv.h>
  37 #include <sys/kernel.h>
  38 #include <sys/bus.h>
  39 #include <sys/systm.h>
  40 #include <sys/counter.h>
  41 #include <sys/module.h>
  42 #include <sys/malloc.h>
  43 #include <sys/queue.h>
  44 #include <sys/taskqueue.h>
  45 #include <sys/pciio.h>
  46 #include <dev/pci/pcireg.h>
  47 #include <dev/pci/pcivar.h>
  48 #include <dev/pci/pci_private.h>
  49 #include <sys/firmware.h>
  50 #include <sys/sbuf.h>
  51 #include <sys/smp.h>
  52 #include <sys/socket.h>
  53 #include <sys/sockio.h>
  54 #include <sys/sysctl.h>
  55 #include <net/ethernet.h>
  56 #include <net/if.h>
  57 #include <net/if_types.h>
  58 #include <net/if_dl.h>
  59 #include <net/if_vlan_var.h>
  60 #if defined(__i386__) || defined(__amd64__)
  61 #include <vm/vm.h>
  62 #include <vm/pmap.h>
  63 #endif
  64
  65 #include "common/common.h"
  66 #include "common/t4_msg.h"
  67 #include "common/t4_regs.h"
  68 #include "common/t4_regs_values.h"
  69 #include "t4_ioctl.h"
  70 #include "t4_l2t.h"
  71 #include "t4_mp_ring.h"
  72
  73 /* T4 bus driver interface */
  74 static int t4_probe(device_t);
  75 static int t4_attach(device_t);
  76 static int t4_detach(device_t);
  77 static device_method_t t4_methods[] = {
  78         DEVMETHOD(device_probe,         t4_probe),
  79         DEVMETHOD(device_attach,        t4_attach),
  80         DEVMETHOD(device_detach,        t4_detach),
  81
  82         DEVMETHOD_END
  83 };
  84 static driver_t t4_driver = {
  85         "t4nex",
  86         t4_methods,
  87         sizeof(struct adapter)
  88 };
  89
  90
  91 /* T4 port (cxgbe) interface */
  92 static int cxgbe_probe(device_t);
  93 static int cxgbe_attach(device_t);
  94 static int cxgbe_detach(device_t);
  95 static device_method_t cxgbe_methods[] = {
  96         DEVMETHOD(device_probe,         cxgbe_probe),
  97         DEVMETHOD(device_attach,        cxgbe_attach),
  98         DEVMETHOD(device_detach,        cxgbe_detach),
  99         { 0, 0 }
 100 };
 101 static driver_t cxgbe_driver = {
 102         "cxgbe",
 103         cxgbe_methods,
 104         sizeof(struct port_info)
 105 };
 106
 107 static d_ioctl_t t4_ioctl;
 108 static d_open_t t4_open;
 109 static d_close_t t4_close;
 110
 111 static struct cdevsw t4_cdevsw = {
 112        .d_version = D_VERSION,
 113        .d_flags = 0,
 114        .d_open = t4_open,
 115        .d_close = t4_close,
 116        .d_ioctl = t4_ioctl,
 117        .d_name = "t4nex",
 118 };
 119
 120 /* T5 bus driver interface */
 121 static int t5_probe(device_t);
 122 static device_method_t t5_methods[] = {
 123         DEVMETHOD(device_probe,         t5_probe),
 124         DEVMETHOD(device_attach,        t4_attach),
 125         DEVMETHOD(device_detach,        t4_detach),
 126
 127         DEVMETHOD_END
 128 };
 129 static driver_t t5_driver = {
 130         "t5nex",
 131         t5_methods,
 132         sizeof(struct adapter)
 133 };
 134
 135
 136 /* T5 port (cxl) interface */
 137 static driver_t cxl_driver = {
 138         "cxl",
 139         cxgbe_methods,
 140         sizeof(struct port_info)
 141 };
 142
 143 static struct cdevsw t5_cdevsw = {
 144        .d_version = D_VERSION,
 145        .d_flags = 0,
 146        .d_open = t4_open,
 147        .d_close = t4_close,
 148        .d_ioctl = t4_ioctl,
 149        .d_name = "t5nex",
 150 };
 151
 152 /* ifnet + media interface */
 153 static void cxgbe_init(void *);
 154 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 155 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 156 static void cxgbe_qflush(struct ifnet *);
 157 static int cxgbe_media_change(struct ifnet *);
 158 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 159
 160 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
 161
 162 /*
 163  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 164  * then ADAPTER_LOCK, then t4_uld_list_lock.
 165  */
 166 static struct sx t4_list_lock;
 167 SLIST_HEAD(, adapter) t4_list;
 168 #ifdef TCP_OFFLOAD
 169 static struct sx t4_uld_list_lock;
 170 SLIST_HEAD(, uld_info) t4_uld_list;
 171 #endif
 172
 173 /*
 174  * Tunables.  See tweak_tunables() too.
 175  *
 176  * Each tunable is set to a default value here if it's known at compile-time.
 177  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
 178  * provide a reasonable default when the driver is loaded.
 179  *
 180  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
 181  * T5 are under hw.cxl.
 182  */
 183
 184 /*
 185  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 186  */
 187 #define NTXQ_10G 16
 188 static int t4_ntxq10g = -1;
 189 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 190
 191 #define NRXQ_10G 8
 192 static int t4_nrxq10g = -1;
 193 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 194
 195 #define NTXQ_1G 4
 196 static int t4_ntxq1g = -1;
 197 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 198
 199 #define NRXQ_1G 2
 200 static int t4_nrxq1g = -1;
 201 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 202
 203 static int t4_rsrv_noflowq = 0;
 204 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
 205
 206 #ifdef TCP_OFFLOAD
 207 #define NOFLDTXQ_10G 8
 208 static int t4_nofldtxq10g = -1;
 209 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 210
 211 #define NOFLDRXQ_10G 2
 212 static int t4_nofldrxq10g = -1;
 213 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 214
 215 #define NOFLDTXQ_1G 2
 216 static int t4_nofldtxq1g = -1;
 217 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 218
 219 #define NOFLDRXQ_1G 1
 220 static int t4_nofldrxq1g = -1;
 221 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 222 #endif
 223
 224 #ifdef DEV_NETMAP
 225 #define NNMTXQ_10G 2
 226 static int t4_nnmtxq10g = -1;
 227 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g);
 228
 229 #define NNMRXQ_10G 2
 230 static int t4_nnmrxq10g = -1;
 231 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g);
 232
 233 #define NNMTXQ_1G 1
 234 static int t4_nnmtxq1g = -1;
 235 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g);
 236
 237 #define NNMRXQ_1G 1
 238 static int t4_nnmrxq1g = -1;
 239 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g);
 240 #endif
 241
 242 /*
 243  * Holdoff parameters for 10G and 1G ports.
 244  */
 245 #define TMR_IDX_10G 1
 246 static int t4_tmr_idx_10g = TMR_IDX_10G;
 247 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 248
 249 #define PKTC_IDX_10G (-1)
 250 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 251 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 252
 253 #define TMR_IDX_1G 1
 254 static int t4_tmr_idx_1g = TMR_IDX_1G;
 255 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 256
 257 #define PKTC_IDX_1G (-1)
 258 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 259 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 260
 261 /*
 262  * Size (# of entries) of each tx and rx queue.
 263  */
 264 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 265 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 266
 267 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 268 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 269
 270 /*
 271  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 272  */
 273 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 274 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 275
 276 /*
 277  * Configuration file.
 278  */
 279 #define DEFAULT_CF      "default"
 280 #define FLASH_CF        "flash"
 281 #define UWIRE_CF        "uwire"
 282 #define FPGA_CF         "fpga"
 283 static char t4_cfg_file[32] = DEFAULT_CF;
 284 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 285
 286 /*
 287  * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively).
 288  * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
 289  * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
 290  *            mark or when signalled to do so, 0 to never emit PAUSE.
 291  */
 292 static int t4_pause_settings = PAUSE_TX | PAUSE_RX;
 293 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings);
 294
 295 /*
 296  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
 297  * encouraged respectively).
 298  */
 299 static unsigned int t4_fw_install = 1;
 300 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 301
 302 /*
 303  * ASIC features that will be used.  Disable the ones you don't want so that the
 304  * chip resources aren't wasted on features that will not be used.
 305  */
 306 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 307 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 308
 309 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 310 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 311
 312 static int t4_toecaps_allowed = -1;
 313 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 314
 315 static int t4_rdmacaps_allowed = 0;
 316 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 317
 318 static int t4_iscsicaps_allowed = 0;
 319 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 320
 321 static int t4_fcoecaps_allowed = 0;
 322 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 323
 324 static int t5_write_combine = 0;
 325 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 326
 327 struct intrs_and_queues {
 328         uint16_t intr_type;     /* INTx, MSI, or MSI-X */
 329         uint16_t nirq;          /* Total # of vectors */
 330         uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
 331         uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */
 332         uint16_t ntxq10g;       /* # of NIC txq's for each 10G port */
 333         uint16_t nrxq10g;       /* # of NIC rxq's for each 10G port */
 334         uint16_t ntxq1g;        /* # of NIC txq's for each 1G port */
 335         uint16_t nrxq1g;        /* # of NIC rxq's for each 1G port */
 336         uint16_t rsrv_noflowq;  /* Flag whether to reserve queue 0 */
 337 #ifdef TCP_OFFLOAD
 338         uint16_t nofldtxq10g;   /* # of TOE txq's for each 10G port */
 339         uint16_t nofldrxq10g;   /* # of TOE rxq's for each 10G port */
 340         uint16_t nofldtxq1g;    /* # of TOE txq's for each 1G port */
 341         uint16_t nofldrxq1g;    /* # of TOE rxq's for each 1G port */
 342 #endif
 343 #ifdef DEV_NETMAP
 344         uint16_t nnmtxq10g;     /* # of netmap txq's for each 10G port */
 345         uint16_t nnmrxq10g;     /* # of netmap rxq's for each 10G port */
 346         uint16_t nnmtxq1g;      /* # of netmap txq's for each 1G port */
 347         uint16_t nnmrxq1g;      /* # of netmap rxq's for each 1G port */
 348 #endif
 349 };
 350
 351 struct filter_entry {
 352         uint32_t valid:1;       /* filter allocated and valid */
 353         uint32_t locked:1;      /* filter is administratively locked */
 354         uint32_t pending:1;     /* filter action is pending firmware reply */
 355         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 356         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 357
 358         struct t4_filter_specification fs;
 359 };
 360
 361 static int map_bars_0_and_4(struct adapter *);
 362 static int map_bar_2(struct adapter *);
 363 static void setup_memwin(struct adapter *);
 364 static int validate_mem_range(struct adapter *, uint32_t, int);
 365 static int fwmtype_to_hwmtype(int);
 366 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
 367     uint32_t *);
 368 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
 369 static uint32_t position_memwin(struct adapter *, int, uint32_t);
 370 static int cfg_itype_and_nqueues(struct adapter *, int, int,
 371     struct intrs_and_queues *);
 372 static int prep_firmware(struct adapter *);
 373 static int partition_resources(struct adapter *, const struct firmware *,
 374     const char *);
 375 static int get_params__pre_init(struct adapter *);
 376 static int get_params__post_init(struct adapter *);
 377 static int set_params__post_init(struct adapter *);
 378 static void t4_set_desc(struct adapter *);
 379 static void build_medialist(struct port_info *, struct ifmedia *);
 380 static int cxgbe_init_synchronized(struct port_info *);
 381 static int cxgbe_uninit_synchronized(struct port_info *);
 382 static int setup_intr_handlers(struct adapter *);
 383 static void quiesce_txq(struct adapter *, struct sge_txq *);
 384 static void quiesce_wrq(struct adapter *, struct sge_wrq *);
 385 static void quiesce_iq(struct adapter *, struct sge_iq *);
 386 static void quiesce_fl(struct adapter *, struct sge_fl *);
 387 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 388     driver_intr_t *, void *, char *);
 389 static int t4_free_irq(struct adapter *, struct irq *);
 390 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
 391     unsigned int);
 392 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 393 static void cxgbe_refresh_stats(struct adapter *, struct port_info *);
 394 static void cxgbe_tick(void *);
 395 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 396 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
 397     struct mbuf *);
 398 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 399 static int fw_msg_not_handled(struct adapter *, const __be64 *);
 400 static int t4_sysctls(struct adapter *);
 401 static int cxgbe_sysctls(struct port_info *);
 402 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 403 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 404 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
 405 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 406 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 407 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 408 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 409 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 410 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
 411 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 412 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 413 #ifdef SBUF_DRAIN
 414 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 415 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 416 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 417 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
 418 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
 419 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 420 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 421 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 422 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 423 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 424 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 425 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 426 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
 427 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 428 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
 429 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 430 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 431 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 432 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 433 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 434 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 435 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
 436 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 437 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 438 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 439 #endif
 440 static uint32_t fconf_to_mode(uint32_t);
 441 static uint32_t mode_to_fconf(uint32_t);
 442 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 443 static int get_filter_mode(struct adapter *, uint32_t *);
 444 static int set_filter_mode(struct adapter *, uint32_t);
 445 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 446 static int get_filter(struct adapter *, struct t4_filter *);
 447 static int set_filter(struct adapter *, struct t4_filter *);
 448 static int del_filter(struct adapter *, struct t4_filter *);
 449 static void clear_filter(struct filter_entry *);
 450 static int set_filter_wr(struct adapter *, int);
 451 static int del_filter_wr(struct adapter *, int);
 452 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 453 static int load_fw(struct adapter *, struct t4_data *);
 454 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 455 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 456 static int set_sched_class(struct adapter *, struct t4_sched_params *);
 457 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
 458 #ifdef TCP_OFFLOAD
 459 static int toe_capability(struct port_info *, int);
 460 #endif
 461 static int mod_event(module_t, int, void *);
 462
 463 struct {
 464         uint16_t device;
 465         char *desc;
 466 } t4_pciids[] = {
 467         {0xa000, "Chelsio Terminator 4 FPGA"},
 468         {0x4400, "Chelsio T440-dbg"},
 469         {0x4401, "Chelsio T420-CR"},
 470         {0x4402, "Chelsio T422-CR"},
 471         {0x4403, "Chelsio T440-CR"},
 472         {0x4404, "Chelsio T420-BCH"},
 473         {0x4405, "Chelsio T440-BCH"},
 474         {0x4406, "Chelsio T440-CH"},
 475         {0x4407, "Chelsio T420-SO"},
 476         {0x4408, "Chelsio T420-CX"},
 477         {0x4409, "Chelsio T420-BT"},
 478         {0x440a, "Chelsio T404-BT"},
 479         {0x440e, "Chelsio T440-LP-CR"},
 480 }, t5_pciids[] = {
 481         {0xb000, "Chelsio Terminator 5 FPGA"},
 482         {0x5400, "Chelsio T580-dbg"},
 483         {0x5401,  "Chelsio T520-CR"},           /* 2 x 10G */
 484         {0x5402,  "Chelsio T522-CR"},           /* 2 x 10G, 2 X 1G */
 485         {0x5403,  "Chelsio T540-CR"},           /* 4 x 10G */
 486         {0x5407,  "Chelsio T520-SO"},           /* 2 x 10G, nomem */
 487         {0x5409,  "Chelsio T520-BT"},           /* 2 x 10GBaseT */
 488         {0x540a,  "Chelsio T504-BT"},           /* 4 x 1G */
 489         {0x540d,  "Chelsio T580-CR"},           /* 2 x 40G */
 490         {0x540e,  "Chelsio T540-LP-CR"},        /* 4 x 10G */
 491         {0x5410,  "Chelsio T580-LP-CR"},        /* 2 x 40G */
 492         {0x5411,  "Chelsio T520-LL-CR"},        /* 2 x 10G */
 493         {0x5412,  "Chelsio T560-CR"},           /* 1 x 40G, 2 x 10G */
 494         {0x5414,  "Chelsio T580-LP-SO-CR"},     /* 2 x 40G, nomem */
 495         {0x5415,  "Chelsio T502-BT"},           /* 2 x 1G */
 496 #ifdef notyet
 497         {0x5404,  "Chelsio T520-BCH"},
 498         {0x5405,  "Chelsio T540-BCH"},
 499         {0x5406,  "Chelsio T540-CH"},
 500         {0x5408,  "Chelsio T520-CX"},
 501         {0x540b,  "Chelsio B520-SR"},
 502         {0x540c,  "Chelsio B504-BT"},
 503         {0x540f,  "Chelsio Amsterdam"},
 504         {0x5413,  "Chelsio T580-CHR"},
 505 #endif
 506 };
 507
 508 #ifdef TCP_OFFLOAD
 509 /*
 510  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 511  * exactly the same for both rxq and ofld_rxq.
 512  */
 513 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 514 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 515 #endif
 516
 517 /* No easy way to include t4_msg.h before adapter.h so we check this way */
 518 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
 519 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
 520
 521 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
 522
 523 static int
 524 t4_probe(device_t dev)
 525 {
 526         int i;
 527         uint16_t v = pci_get_vendor(dev);
 528         uint16_t d = pci_get_device(dev);
 529         uint8_t f = pci_get_function(dev);
 530
 531         if (v != PCI_VENDOR_ID_CHELSIO)
 532                 return (ENXIO);
 533
 534         /* Attach only to PF0 of the FPGA */
 535         if (d == 0xa000 && f != 0)
 536                 return (ENXIO);
 537
 538         for (i = 0; i < nitems(t4_pciids); i++) {
 539                 if (d == t4_pciids[i].device) {
 540                         device_set_desc(dev, t4_pciids[i].desc);
 541                         return (BUS_PROBE_DEFAULT);
 542                 }
 543         }
 544
 545         return (ENXIO);
 546 }
 547
 548 static int
 549 t5_probe(device_t dev)
 550 {
 551         int i;
 552         uint16_t v = pci_get_vendor(dev);
 553         uint16_t d = pci_get_device(dev);
 554         uint8_t f = pci_get_function(dev);
 555
 556         if (v != PCI_VENDOR_ID_CHELSIO)
 557                 return (ENXIO);
 558
 559         /* Attach only to PF0 of the FPGA */
 560         if (d == 0xb000 && f != 0)
 561                 return (ENXIO);
 562
 563         for (i = 0; i < nitems(t5_pciids); i++) {
 564                 if (d == t5_pciids[i].device) {
 565                         device_set_desc(dev, t5_pciids[i].desc);
 566                         return (BUS_PROBE_DEFAULT);
 567                 }
 568         }
 569
 570         return (ENXIO);
 571 }
 572
 573 static void
 574 t5_attribute_workaround(device_t dev)
 575 {
 576         device_t root_port;
 577         uint32_t v;
 578
 579         /*
 580          * The T5 chips do not properly echo the No Snoop and Relaxed
 581          * Ordering attributes when replying to a TLP from a Root
 582          * Port.  As a workaround, find the parent Root Port and
 583          * disable No Snoop and Relaxed Ordering.  Note that this
 584          * affects all devices under this root port.
 585          */
 586         root_port = pci_find_pcie_root_port(dev);
 587         if (root_port == NULL) {
 588                 device_printf(dev, "Unable to find parent root port\n");
 589                 return;
 590         }
 591
 592         v = pcie_adjust_config(root_port, PCIER_DEVICE_CTL,
 593             PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE, 0, 2);
 594         if ((v & (PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE)) !=
 595             0)
 596                 device_printf(dev, "Disabled No Snoop/Relaxed Ordering on %s\n",
 597                     device_get_nameunit(root_port));
 598 }
 599
 600 static int
 601 t4_attach(device_t dev)
 602 {
 603         struct adapter *sc;
 604         int rc = 0, i, n10g, n1g, rqidx, tqidx;
 605         struct intrs_and_queues iaq;
 606         struct sge *s;
 607 #ifdef TCP_OFFLOAD
 608         int ofld_rqidx, ofld_tqidx;
 609 #endif
 610 #ifdef DEV_NETMAP
 611         int nm_rqidx, nm_tqidx;
 612 #endif
 613
 614         sc = device_get_softc(dev);
 615         sc->dev = dev;
 616         TUNABLE_INT_FETCH("hw.cxgbe.debug_flags", &sc->debug_flags);
 617
 618         if ((pci_get_device(dev) & 0xff00) == 0x5400)
 619                 t5_attribute_workaround(dev);
 620         pci_enable_busmaster(dev);
 621         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 622                 uint32_t v;
 623
 624                 pci_set_max_read_req(dev, 4096);
 625                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 626                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 627                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 628
 629                 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
 630         }
 631
 632         sc->traceq = -1;
 633         mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
 634         snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
 635             device_get_nameunit(dev));
 636
 637         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 638             device_get_nameunit(dev));
 639         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 640         sx_xlock(&t4_list_lock);
 641         SLIST_INSERT_HEAD(&t4_list, sc, link);
 642         sx_xunlock(&t4_list_lock);
 643
 644         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 645         TAILQ_INIT(&sc->sfl);
 646         callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
 647
 648         mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF);
 649
 650         rc = map_bars_0_and_4(sc);
 651         if (rc != 0)
 652                 goto done; /* error message displayed already */
 653
 654         /*
 655          * This is the real PF# to which we're attaching.  Works from within PCI
 656          * passthrough environments too, where pci_get_function() could return a
 657          * different PF# depending on the passthrough configuration.  We need to
 658          * use the real PF# in all our communication with the firmware.
 659          */
 660         sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 661         sc->mbox = sc->pf;
 662
 663         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 664         sc->an_handler = an_not_handled;
 665         for (i = 0; i < nitems(sc->cpl_handler); i++)
 666                 sc->cpl_handler[i] = cpl_not_handled;
 667         for (i = 0; i < nitems(sc->fw_msg_handler); i++)
 668                 sc->fw_msg_handler[i] = fw_msg_not_handled;
 669         t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 670         t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
 671         t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt);
 672         t4_init_sge_cpl_handlers(sc);
 673
 674         /* Prepare the adapter for operation */
 675         rc = -t4_prep_adapter(sc);
 676         if (rc != 0) {
 677                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 678                 goto done;
 679         }
 680
 681         /*
 682          * Do this really early, with the memory windows set up even before the
 683          * character device.  The userland tool's register i/o and mem read
 684          * will work even in "recovery mode".
 685          */
 686         setup_memwin(sc);
 687         sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
 688             device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
 689             device_get_nameunit(dev));
 690         if (sc->cdev == NULL)
 691                 device_printf(dev, "failed to create nexus char device.\n");
 692         else
 693                 sc->cdev->si_drv1 = sc;
 694
 695         /* Go no further if recovery mode has been requested. */
 696         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 697                 device_printf(dev, "recovery mode.\n");
 698                 goto done;
 699         }
 700
 701 #if defined(__i386__)
 702         if ((cpu_feature & CPUID_CX8) == 0) {
 703                 device_printf(dev, "64 bit atomics not available.\n");
 704                 rc = ENOTSUP;
 705                 goto done;
 706         }
 707 #endif
 708
 709         /* Prepare the firmware for operation */
 710         rc = prep_firmware(sc);
 711         if (rc != 0)
 712                 goto done; /* error message displayed already */
 713
 714         rc = get_params__post_init(sc);
 715         if (rc != 0)
 716                 goto done; /* error message displayed already */
 717
 718         rc = set_params__post_init(sc);
 719         if (rc != 0)
 720                 goto done; /* error message displayed already */
 721
 722         rc = map_bar_2(sc);
 723         if (rc != 0)
 724                 goto done; /* error message displayed already */
 725
 726         rc = t4_create_dma_tag(sc);
 727         if (rc != 0)
 728                 goto done; /* error message displayed already */
 729
 730         /*
 731          * First pass over all the ports - allocate VIs and initialize some
 732          * basic parameters like mac address, port type, etc.  We also figure
 733          * out whether a port is 10G or 1G and use that information when
 734          * calculating how many interrupts to attempt to allocate.
 735          */
 736         n10g = n1g = 0;
 737         for_each_port(sc, i) {
 738                 struct port_info *pi;
 739
 740                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 741                 sc->port[i] = pi;
 742
 743                 /* These must be set before t4_port_init */
 744                 pi->adapter = sc;
 745                 pi->port_id = i;
 746
 747                 /* Allocate the vi and initialize parameters like mac addr */
 748                 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 749                 if (rc != 0) {
 750                         device_printf(dev, "unable to initialize port %d: %d\n",
 751                             i, rc);
 752                         free(pi, M_CXGBE);
 753                         sc->port[i] = NULL;
 754                         goto done;
 755                 }
 756
 757                 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX);
 758                 pi->link_cfg.requested_fc |= t4_pause_settings;
 759                 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX);
 760                 pi->link_cfg.fc |= t4_pause_settings;
 761
 762                 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
 763                 if (rc != 0) {
 764                         device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
 765                         free(pi, M_CXGBE);
 766                         sc->port[i] = NULL;
 767                         goto done;
 768                 }
 769
 770                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 771                     device_get_nameunit(dev), i);
 772                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 773                 sc->chan_map[pi->tx_chan] = i;
 774
 775                 if (is_10G_port(pi) || is_40G_port(pi)) {
 776                         n10g++;
 777                         pi->tmr_idx = t4_tmr_idx_10g;
 778                         pi->pktc_idx = t4_pktc_idx_10g;
 779                 } else {
 780                         n1g++;
 781                         pi->tmr_idx = t4_tmr_idx_1g;
 782                         pi->pktc_idx = t4_pktc_idx_1g;
 783                 }
 784
 785                 pi->xact_addr_filt = -1;
 786                 pi->linkdnrc = -1;
 787
 788                 pi->qsize_rxq = t4_qsize_rxq;
 789                 pi->qsize_txq = t4_qsize_txq;
 790
 791                 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
 792                 if (pi->dev == NULL) {
 793                         device_printf(dev,
 794                             "failed to add device for port %d.\n", i);
 795                         rc = ENXIO;
 796                         goto done;
 797                 }
 798                 device_set_softc(pi->dev, pi);
 799         }
 800
 801         /*
 802          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 803          */
 804         rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
 805         if (rc != 0)
 806                 goto done; /* error message displayed already */
 807
 808         sc->intr_type = iaq.intr_type;
 809         sc->intr_count = iaq.nirq;
 810
 811         s = &sc->sge;
 812         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 813         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 814         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
 815         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 816         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
 817 #ifdef TCP_OFFLOAD
 818         if (is_offload(sc)) {
 819                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 820                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 821                 s->neq += s->nofldtxq + s->nofldrxq;
 822                 s->niq += s->nofldrxq;
 823
 824                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 825                     M_CXGBE, M_ZERO | M_WAITOK);
 826                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 827                     M_CXGBE, M_ZERO | M_WAITOK);
 828         }
 829 #endif
 830 #ifdef DEV_NETMAP
 831         s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g;
 832         s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g;
 833         s->neq += s->nnmtxq + s->nnmrxq;
 834         s->niq += s->nnmrxq;
 835
 836         s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
 837             M_CXGBE, M_ZERO | M_WAITOK);
 838         s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
 839             M_CXGBE, M_ZERO | M_WAITOK);
 840 #endif
 841
 842         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 843             M_ZERO | M_WAITOK);
 844         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 845             M_ZERO | M_WAITOK);
 846         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 847             M_ZERO | M_WAITOK);
 848         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 849             M_ZERO | M_WAITOK);
 850         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 851             M_ZERO | M_WAITOK);
 852
 853         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 854             M_ZERO | M_WAITOK);
 855
 856         t4_init_l2t(sc, M_WAITOK);
 857
 858         /*
 859          * Second pass over the ports.  This time we know the number of rx and
 860          * tx queues that each port should get.
 861          */
 862         rqidx = tqidx = 0;
 863 #ifdef TCP_OFFLOAD
 864         ofld_rqidx = ofld_tqidx = 0;
 865 #endif
 866 #ifdef DEV_NETMAP
 867         nm_rqidx = nm_tqidx = 0;
 868 #endif
 869         for_each_port(sc, i) {
 870                 struct port_info *pi = sc->port[i];
 871
 872                 if (pi == NULL)
 873                         continue;
 874
 875                 pi->first_rxq = rqidx;
 876                 pi->first_txq = tqidx;
 877                 if (is_10G_port(pi) || is_40G_port(pi)) {
 878                         pi->flags |= iaq.intr_flags_10g;
 879                         pi->nrxq = iaq.nrxq10g;
 880                         pi->ntxq = iaq.ntxq10g;
 881                 } else {
 882                         pi->flags |= iaq.intr_flags_1g;
 883                         pi->nrxq = iaq.nrxq1g;
 884                         pi->ntxq = iaq.ntxq1g;
 885                 }
 886
 887                 if (pi->ntxq > 1)
 888                         pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
 889                 else
 890                         pi->rsrv_noflowq = 0;
 891
 892                 rqidx += pi->nrxq;
 893                 tqidx += pi->ntxq;
 894 #ifdef TCP_OFFLOAD
 895                 if (is_offload(sc)) {
 896                         pi->first_ofld_rxq = ofld_rqidx;
 897                         pi->first_ofld_txq = ofld_tqidx;
 898                         if (is_10G_port(pi) || is_40G_port(pi)) {
 899                                 pi->nofldrxq = iaq.nofldrxq10g;
 900                                 pi->nofldtxq = iaq.nofldtxq10g;
 901                         } else {
 902                                 pi->nofldrxq = iaq.nofldrxq1g;
 903                                 pi->nofldtxq = iaq.nofldtxq1g;
 904                         }
 905                         ofld_rqidx += pi->nofldrxq;
 906                         ofld_tqidx += pi->nofldtxq;
 907                 }
 908 #endif
 909 #ifdef DEV_NETMAP
 910                 pi->first_nm_rxq = nm_rqidx;
 911                 pi->first_nm_txq = nm_tqidx;
 912                 if (is_10G_port(pi) || is_40G_port(pi)) {
 913                         pi->nnmrxq = iaq.nnmrxq10g;
 914                         pi->nnmtxq = iaq.nnmtxq10g;
 915                 } else {
 916                         pi->nnmrxq = iaq.nnmrxq1g;
 917                         pi->nnmtxq = iaq.nnmtxq1g;
 918                 }
 919                 nm_rqidx += pi->nnmrxq;
 920                 nm_tqidx += pi->nnmtxq;
 921 #endif
 922         }
 923
 924         rc = setup_intr_handlers(sc);
 925         if (rc != 0) {
 926                 device_printf(dev,
 927                     "failed to setup interrupt handlers: %d\n", rc);
 928                 goto done;
 929         }
 930
 931         rc = bus_generic_attach(dev);
 932         if (rc != 0) {
 933                 device_printf(dev,
 934                     "failed to attach all child ports: %d\n", rc);
 935                 goto done;
 936         }
 937
 938         device_printf(dev,
 939             "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
 940             sc->params.pci.speed, sc->params.pci.width, sc->params.nports,
 941             sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" :
 942             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
 943             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
 944
 945         t4_set_desc(sc);
 946
 947 done:
 948         if (rc != 0 && sc->cdev) {
 949                 /* cdev was created and so cxgbetool works; recover that way. */
 950                 device_printf(dev,
 951                     "error during attach, adapter is now in recovery mode.\n");
 952                 rc = 0;
 953         }
 954
 955         if (rc != 0)
 956                 t4_detach(dev);
 957         else
 958                 t4_sysctls(sc);
 959
 960         return (rc);
 961 }
 962
 963 /*
 964  * Idempotent
 965  */
 966 static int
 967 t4_detach(device_t dev)
 968 {
 969         struct adapter *sc;
 970         struct port_info *pi;
 971         int i, rc;
 972
 973         sc = device_get_softc(dev);
 974
 975         if (sc->flags & FULL_INIT_DONE)
 976                 t4_intr_disable(sc);
 977
 978         if (sc->cdev) {
 979                 destroy_dev(sc->cdev);
 980                 sc->cdev = NULL;
 981         }
 982
 983         rc = bus_generic_detach(dev);
 984         if (rc) {
 985                 device_printf(dev,
 986                     "failed to detach child devices: %d\n", rc);
 987                 return (rc);
 988         }
 989
 990         for (i = 0; i < sc->intr_count; i++)
 991                 t4_free_irq(sc, &sc->irq[i]);
 992
 993         for (i = 0; i < MAX_NPORTS; i++) {
 994                 pi = sc->port[i];
 995                 if (pi) {
 996                         t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid);
 997                         if (pi->dev)
 998                                 device_delete_child(dev, pi->dev);
 999
1000                         mtx_destroy(&pi->pi_lock);
1001                         free(pi, M_CXGBE);
1002                 }
1003         }
1004
1005         if (sc->flags & FULL_INIT_DONE)
1006                 adapter_full_uninit(sc);
1007
1008         if (sc->flags & FW_OK)
1009                 t4_fw_bye(sc, sc->mbox);
1010
1011         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
1012                 pci_release_msi(dev);
1013
1014         if (sc->regs_res)
1015                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
1016                     sc->regs_res);
1017
1018         if (sc->udbs_res)
1019                 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
1020                     sc->udbs_res);
1021
1022         if (sc->msix_res)
1023                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
1024                     sc->msix_res);
1025
1026         if (sc->l2t)
1027                 t4_free_l2t(sc->l2t);
1028
1029 #ifdef TCP_OFFLOAD
1030         free(sc->sge.ofld_rxq, M_CXGBE);
1031         free(sc->sge.ofld_txq, M_CXGBE);
1032 #endif
1033 #ifdef DEV_NETMAP
1034         free(sc->sge.nm_rxq, M_CXGBE);
1035         free(sc->sge.nm_txq, M_CXGBE);
1036 #endif
1037         free(sc->irq, M_CXGBE);
1038         free(sc->sge.rxq, M_CXGBE);
1039         free(sc->sge.txq, M_CXGBE);
1040         free(sc->sge.ctrlq, M_CXGBE);
1041         free(sc->sge.iqmap, M_CXGBE);
1042         free(sc->sge.eqmap, M_CXGBE);
1043         free(sc->tids.ftid_tab, M_CXGBE);
1044         t4_destroy_dma_tag(sc);
1045         if (mtx_initialized(&sc->sc_lock)) {
1046                 sx_xlock(&t4_list_lock);
1047                 SLIST_REMOVE(&t4_list, sc, adapter, link);
1048                 sx_xunlock(&t4_list_lock);
1049                 mtx_destroy(&sc->sc_lock);
1050         }
1051
1052         if (mtx_initialized(&sc->tids.ftid_lock))
1053                 mtx_destroy(&sc->tids.ftid_lock);
1054         if (mtx_initialized(&sc->sfl_lock))
1055                 mtx_destroy(&sc->sfl_lock);
1056         if (mtx_initialized(&sc->ifp_lock))
1057                 mtx_destroy(&sc->ifp_lock);
1058         if (mtx_initialized(&sc->regwin_lock))
1059                 mtx_destroy(&sc->regwin_lock);
1060
1061         bzero(sc, sizeof(*sc));
1062
1063         return (0);
1064 }
1065
1066 static int
1067 cxgbe_probe(device_t dev)
1068 {
1069         char buf[128];
1070         struct port_info *pi = device_get_softc(dev);
1071
1072         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
1073         device_set_desc_copy(dev, buf);
1074
1075         return (BUS_PROBE_DEFAULT);
1076 }
1077
1078 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
1079     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1080     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
1081 #define T4_CAP_ENABLE (T4_CAP)
1082
1083 static int
1084 cxgbe_attach(device_t dev)
1085 {
1086         struct port_info *pi = device_get_softc(dev);
1087         struct ifnet *ifp;
1088         char *s;
1089         int n, o;
1090
1091         /* Allocate an ifnet and set it up */
1092         ifp = if_alloc(IFT_ETHER);
1093         if (ifp == NULL) {
1094                 device_printf(dev, "Cannot allocate ifnet\n");
1095                 return (ENOMEM);
1096         }
1097         pi->ifp = ifp;
1098         ifp->if_softc = pi;
1099
1100         callout_init(&pi->tick, CALLOUT_MPSAFE);
1101
1102         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1103         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1104
1105         ifp->if_init = cxgbe_init;
1106         ifp->if_ioctl = cxgbe_ioctl;
1107         ifp->if_transmit = cxgbe_transmit;
1108         ifp->if_qflush = cxgbe_qflush;
1109
1110         ifp->if_capabilities = T4_CAP;
1111 #ifdef TCP_OFFLOAD
1112         if (is_offload(pi->adapter))
1113                 ifp->if_capabilities |= IFCAP_TOE;
1114 #endif
1115         ifp->if_capenable = T4_CAP_ENABLE;
1116         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1117             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1118
1119         ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1120         ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS;
1121         ifp->if_hw_tsomaxsegsize = 65536;
1122
1123         /* Initialize ifmedia for this port */
1124         ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1125             cxgbe_media_status);
1126         build_medialist(pi, &pi->media);
1127
1128         pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1129             EVENTHANDLER_PRI_ANY);
1130
1131         ether_ifattach(ifp, pi->hw_addr);
1132
1133         n = 128;
1134         s = malloc(n, M_CXGBE, M_WAITOK);
1135         o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq);
1136         MPASS(n > o);
1137 #ifdef TCP_OFFLOAD
1138         if (is_offload(pi->adapter)) {
1139                 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)",
1140                     pi->nofldtxq, pi->nofldrxq);
1141                 MPASS(n > o);
1142         }
1143 #endif
1144 #ifdef DEV_NETMAP
1145         o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq,
1146             pi->nnmrxq);
1147         MPASS(n > o);
1148 #endif
1149         device_printf(dev, "%s\n", s);
1150         free(s, M_CXGBE);
1151
1152 #ifdef DEV_NETMAP
1153         /* nm_media handled here to keep implementation private to this file */
1154         ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change,
1155             cxgbe_media_status);
1156         build_medialist(pi, &pi->nm_media);
1157         create_netmap_ifnet(pi);        /* logs errors it something fails */
1158 #endif
1159         cxgbe_sysctls(pi);
1160
1161         return (0);
1162 }
1163
1164 static int
1165 cxgbe_detach(device_t dev)
1166 {
1167         struct port_info *pi = device_get_softc(dev);
1168         struct adapter *sc = pi->adapter;
1169         struct ifnet *ifp = pi->ifp;
1170
1171         /* Tell if_ioctl and if_init that the port is going away */
1172         ADAPTER_LOCK(sc);
1173         SET_DOOMED(pi);
1174         wakeup(&sc->flags);
1175         while (IS_BUSY(sc))
1176                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
1177         SET_BUSY(sc);
1178 #ifdef INVARIANTS
1179         sc->last_op = "t4detach";
1180         sc->last_op_thr = curthread;
1181         sc->last_op_flags = 0;
1182 #endif
1183         ADAPTER_UNLOCK(sc);
1184
1185         if (pi->flags & HAS_TRACEQ) {
1186                 sc->traceq = -1;        /* cloner should not create ifnet */
1187                 t4_tracer_port_detach(sc);
1188         }
1189
1190         if (pi->vlan_c)
1191                 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
1192
1193         PORT_LOCK(pi);
1194         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1195         callout_stop(&pi->tick);
1196         PORT_UNLOCK(pi);
1197         callout_drain(&pi->tick);
1198
1199         /* Let detach proceed even if these fail. */
1200         cxgbe_uninit_synchronized(pi);
1201         port_full_uninit(pi);
1202
1203         ifmedia_removeall(&pi->media);
1204         ether_ifdetach(pi->ifp);
1205         if_free(pi->ifp);
1206
1207 #ifdef DEV_NETMAP
1208         /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */
1209         destroy_netmap_ifnet(pi);
1210 #endif
1211
1212         ADAPTER_LOCK(sc);
1213         CLR_BUSY(sc);
1214         wakeup(&sc->flags);
1215         ADAPTER_UNLOCK(sc);
1216
1217         return (0);
1218 }
1219
1220 static void
1221 cxgbe_init(void *arg)
1222 {
1223         struct port_info *pi = arg;
1224         struct adapter *sc = pi->adapter;
1225
1226         if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
1227                 return;
1228         cxgbe_init_synchronized(pi);
1229         end_synchronized_op(sc, 0);
1230 }
1231
1232 static int
1233 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1234 {
1235         int rc = 0, mtu, flags, can_sleep;
1236         struct port_info *pi = ifp->if_softc;
1237         struct adapter *sc = pi->adapter;
1238         struct ifreq *ifr = (struct ifreq *)data;
1239         uint32_t mask;
1240
1241         switch (cmd) {
1242         case SIOCSIFMTU:
1243                 mtu = ifr->ifr_mtu;
1244                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1245                         return (EINVAL);
1246
1247                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
1248                 if (rc)
1249                         return (rc);
1250                 ifp->if_mtu = mtu;
1251                 if (pi->flags & PORT_INIT_DONE) {
1252                         t4_update_fl_bufsize(ifp);
1253                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1254                                 rc = update_mac_settings(ifp, XGMAC_MTU);
1255                 }
1256                 end_synchronized_op(sc, 0);
1257                 break;
1258
1259         case SIOCSIFFLAGS:
1260                 can_sleep = 0;
1261 redo_sifflags:
1262                 rc = begin_synchronized_op(sc, pi,
1263                     can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg");
1264                 if (rc)
1265                         return (rc);
1266
1267                 if (ifp->if_flags & IFF_UP) {
1268                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1269                                 flags = pi->if_flags;
1270                                 if ((ifp->if_flags ^ flags) &
1271                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1272                                         if (can_sleep == 1) {
1273                                                 end_synchronized_op(sc, 0);
1274                                                 can_sleep = 0;
1275                                                 goto redo_sifflags;
1276                                         }
1277                                         rc = update_mac_settings(ifp,
1278                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1279                                 }
1280                         } else {
1281                                 if (can_sleep == 0) {
1282                                         end_synchronized_op(sc, LOCK_HELD);
1283                                         can_sleep = 1;
1284                                         goto redo_sifflags;
1285                                 }
1286                                 rc = cxgbe_init_synchronized(pi);
1287                         }
1288                         pi->if_flags = ifp->if_flags;
1289                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1290                         if (can_sleep == 0) {
1291                                 end_synchronized_op(sc, LOCK_HELD);
1292                                 can_sleep = 1;
1293                                 goto redo_sifflags;
1294                         }
1295                         rc = cxgbe_uninit_synchronized(pi);
1296                 }
1297                 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD);
1298                 break;
1299
1300         case SIOCADDMULTI:
1301         case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1302                 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
1303                 if (rc)
1304                         return (rc);
1305                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1306                         rc = update_mac_settings(ifp, XGMAC_MCADDRS);
1307                 end_synchronized_op(sc, LOCK_HELD);
1308                 break;
1309
1310         case SIOCSIFCAP:
1311                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
1312                 if (rc)
1313                         return (rc);
1314
1315                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1316                 if (mask & IFCAP_TXCSUM) {
1317                         ifp->if_capenable ^= IFCAP_TXCSUM;
1318                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1319
1320                         if (IFCAP_TSO4 & ifp->if_capenable &&
1321                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1322                                 ifp->if_capenable &= ~IFCAP_TSO4;
1323                                 if_printf(ifp,
1324                                     "tso4 disabled due to -txcsum.\n");
1325                         }
1326                 }
1327                 if (mask & IFCAP_TXCSUM_IPV6) {
1328                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1329                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1330
1331                         if (IFCAP_TSO6 & ifp->if_capenable &&
1332                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1333                                 ifp->if_capenable &= ~IFCAP_TSO6;
1334                                 if_printf(ifp,
1335                                     "tso6 disabled due to -txcsum6.\n");
1336                         }
1337                 }
1338                 if (mask & IFCAP_RXCSUM)
1339                         ifp->if_capenable ^= IFCAP_RXCSUM;
1340                 if (mask & IFCAP_RXCSUM_IPV6)
1341                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1342
1343                 /*
1344                  * Note that we leave CSUM_TSO alone (it is always set).  The
1345                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1346                  * sending a TSO request our way, so it's sufficient to toggle
1347                  * IFCAP_TSOx only.
1348                  */
1349                 if (mask & IFCAP_TSO4) {
1350                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1351                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1352                                 if_printf(ifp, "enable txcsum first.\n");
1353                                 rc = EAGAIN;
1354                                 goto fail;
1355                         }
1356                         ifp->if_capenable ^= IFCAP_TSO4;
1357                 }
1358                 if (mask & IFCAP_TSO6) {
1359                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1360                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1361                                 if_printf(ifp, "enable txcsum6 first.\n");
1362                                 rc = EAGAIN;
1363                                 goto fail;
1364                         }
1365                         ifp->if_capenable ^= IFCAP_TSO6;
1366                 }
1367                 if (mask & IFCAP_LRO) {
1368 #if defined(INET) || defined(INET6)
1369                         int i;
1370                         struct sge_rxq *rxq;
1371
1372                         ifp->if_capenable ^= IFCAP_LRO;
1373                         for_each_rxq(pi, i, rxq) {
1374                                 if (ifp->if_capenable & IFCAP_LRO)
1375                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1376                                 else
1377                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1378                         }
1379 #endif
1380                 }
1381 #ifdef TCP_OFFLOAD
1382                 if (mask & IFCAP_TOE) {
1383                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1384
1385                         rc = toe_capability(pi, enable);
1386                         if (rc != 0)
1387                                 goto fail;
1388
1389                         ifp->if_capenable ^= mask;
1390                 }
1391 #endif
1392                 if (mask & IFCAP_VLAN_HWTAGGING) {
1393                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1394                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1395                                 rc = update_mac_settings(ifp, XGMAC_VLANEX);
1396                 }
1397                 if (mask & IFCAP_VLAN_MTU) {
1398                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1399
1400                         /* Need to find out how to disable auto-mtu-inflation */
1401                 }
1402                 if (mask & IFCAP_VLAN_HWTSO)
1403                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1404                 if (mask & IFCAP_VLAN_HWCSUM)
1405                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1406
1407 #ifdef VLAN_CAPABILITIES
1408                 VLAN_CAPABILITIES(ifp);
1409 #endif
1410 fail:
1411                 end_synchronized_op(sc, 0);
1412                 break;
1413
1414         case SIOCSIFMEDIA:
1415         case SIOCGIFMEDIA:
1416                 ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1417                 break;
1418
1419         case SIOCGI2C: {
1420                 struct ifi2creq i2c;
1421
1422                 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
1423                 if (rc != 0)
1424                         break;
1425                 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
1426                         rc = EPERM;
1427                         break;
1428                 }
1429                 if (i2c.len > sizeof(i2c.data)) {
1430                         rc = EINVAL;
1431                         break;
1432                 }
1433                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4i2c");
1434                 if (rc)
1435                         return (rc);
1436                 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr,
1437                     i2c.offset, i2c.len, &i2c.data[0]);
1438                 end_synchronized_op(sc, 0);
1439                 if (rc == 0)
1440                         rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
1441                 break;
1442         }
1443
1444         default:
1445                 rc = ether_ioctl(ifp, cmd, data);
1446         }
1447
1448         return (rc);
1449 }
1450
1451 static int
1452 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1453 {
1454         struct port_info *pi = ifp->if_softc;
1455         struct adapter *sc = pi->adapter;
1456         struct sge_txq *txq;
1457         void *items[1];
1458         int rc;
1459
1460         M_ASSERTPKTHDR(m);
1461         MPASS(m->m_nextpkt == NULL);    /* not quite ready for this yet */
1462
1463         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1464                 m_freem(m);
1465                 return (ENETDOWN);
1466         }
1467
1468         rc = parse_pkt(&m);
1469         if (__predict_false(rc != 0)) {
1470                 MPASS(m == NULL);                       /* was freed already */
1471                 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */
1472                 return (rc);
1473         }
1474
1475         /* Select a txq. */
1476         txq = &sc->sge.txq[pi->first_txq];
1477         if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
1478                 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) +
1479                     pi->rsrv_noflowq);
1480
1481         items[0] = m;
1482         rc = mp_ring_enqueue(txq->r, items, 1, 4096);
1483         if (__predict_false(rc != 0))
1484                 m_freem(m);
1485
1486         return (rc);
1487 }
1488
1489 static void
1490 cxgbe_qflush(struct ifnet *ifp)
1491 {
1492         struct port_info *pi = ifp->if_softc;
1493         struct sge_txq *txq;
1494         int i;
1495
1496         /* queues do not exist if !PORT_INIT_DONE. */
1497         if (pi->flags & PORT_INIT_DONE) {
1498                 for_each_txq(pi, i, txq) {
1499                         TXQ_LOCK(txq);
1500                         txq->eq.flags &= ~EQ_ENABLED;
1501                         TXQ_UNLOCK(txq);
1502                         while (!mp_ring_is_idle(txq->r)) {
1503                                 mp_ring_check_drainage(txq->r, 0);
1504                                 pause("qflush", 1);
1505                         }
1506                 }
1507         }
1508         if_qflush(ifp);
1509 }
1510
1511 static int
1512 cxgbe_media_change(struct ifnet *ifp)
1513 {
1514         struct port_info *pi = ifp->if_softc;
1515
1516         device_printf(pi->dev, "%s unimplemented.\n", __func__);
1517
1518         return (EOPNOTSUPP);
1519 }
1520
1521 static void
1522 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1523 {
1524         struct port_info *pi = ifp->if_softc;
1525         struct ifmedia *media = NULL;
1526         struct ifmedia_entry *cur;
1527         int speed = pi->link_cfg.speed;
1528
1529         if (ifp == pi->ifp)
1530                 media = &pi->media;
1531 #ifdef DEV_NETMAP
1532         else if (ifp == pi->nm_ifp)
1533                 media = &pi->nm_media;
1534 #endif
1535         MPASS(media != NULL);
1536
1537         cur = media->ifm_cur;
1538
1539         ifmr->ifm_status = IFM_AVALID;
1540         if (!pi->link_cfg.link_ok)
1541                 return;
1542
1543         ifmr->ifm_status |= IFM_ACTIVE;
1544
1545         /* active and current will differ iff current media is autoselect. */
1546         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1547                 return;
1548
1549         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1550         if (speed == SPEED_10000)
1551                 ifmr->ifm_active |= IFM_10G_T;
1552         else if (speed == SPEED_1000)
1553                 ifmr->ifm_active |= IFM_1000_T;
1554         else if (speed == SPEED_100)
1555                 ifmr->ifm_active |= IFM_100_TX;
1556         else if (speed == SPEED_10)
1557                 ifmr->ifm_active |= IFM_10_T;
1558         else
1559                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1560                             speed));
1561 }
1562
1563 void
1564 t4_fatal_err(struct adapter *sc)
1565 {
1566         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1567         t4_intr_disable(sc);
1568         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1569             device_get_nameunit(sc->dev));
1570 }
1571
1572 static int
1573 map_bars_0_and_4(struct adapter *sc)
1574 {
1575         sc->regs_rid = PCIR_BAR(0);
1576         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1577             &sc->regs_rid, RF_ACTIVE);
1578         if (sc->regs_res == NULL) {
1579                 device_printf(sc->dev, "cannot map registers.\n");
1580                 return (ENXIO);
1581         }
1582         sc->bt = rman_get_bustag(sc->regs_res);
1583         sc->bh = rman_get_bushandle(sc->regs_res);
1584         sc->mmio_len = rman_get_size(sc->regs_res);
1585         setbit(&sc->doorbells, DOORBELL_KDB);
1586
1587         sc->msix_rid = PCIR_BAR(4);
1588         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1589             &sc->msix_rid, RF_ACTIVE);
1590         if (sc->msix_res == NULL) {
1591                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1592                 return (ENXIO);
1593         }
1594
1595         return (0);
1596 }
1597
1598 static int
1599 map_bar_2(struct adapter *sc)
1600 {
1601
1602         /*
1603          * T4: only iWARP driver uses the userspace doorbells.  There is no need
1604          * to map it if RDMA is disabled.
1605          */
1606         if (is_t4(sc) && sc->rdmacaps == 0)
1607                 return (0);
1608
1609         sc->udbs_rid = PCIR_BAR(2);
1610         sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1611             &sc->udbs_rid, RF_ACTIVE);
1612         if (sc->udbs_res == NULL) {
1613                 device_printf(sc->dev, "cannot map doorbell BAR.\n");
1614                 return (ENXIO);
1615         }
1616         sc->udbs_base = rman_get_virtual(sc->udbs_res);
1617
1618         if (is_t5(sc)) {
1619                 setbit(&sc->doorbells, DOORBELL_UDB);
1620 #if defined(__i386__) || defined(__amd64__)
1621                 if (t5_write_combine) {
1622                         int rc;
1623
1624                         /*
1625                          * Enable write combining on BAR2.  This is the
1626                          * userspace doorbell BAR and is split into 128B
1627                          * (UDBS_SEG_SIZE) doorbell regions, each associated
1628                          * with an egress queue.  The first 64B has the doorbell
1629                          * and the second 64B can be used to submit a tx work
1630                          * request with an implicit doorbell.
1631                          */
1632
1633                         rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
1634                             rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
1635                         if (rc == 0) {
1636                                 clrbit(&sc->doorbells, DOORBELL_UDB);
1637                                 setbit(&sc->doorbells, DOORBELL_WCWR);
1638                                 setbit(&sc->doorbells, DOORBELL_UDBWC);
1639                         } else {
1640                                 device_printf(sc->dev,
1641                                     "couldn't enable write combining: %d\n",
1642                                     rc);
1643                         }
1644
1645                         t4_write_reg(sc, A_SGE_STAT_CFG,
1646                             V_STATSOURCE_T5(7) | V_STATMODE(0));
1647                 }
1648 #endif
1649         }
1650
1651         return (0);
1652 }
1653
1654 static const struct memwin t4_memwin[] = {
1655         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1656         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1657         { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
1658 };
1659
1660 static const struct memwin t5_memwin[] = {
1661         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1662         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1663         { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1664 };
1665
1666 static void
1667 setup_memwin(struct adapter *sc)
1668 {
1669         const struct memwin *mw;
1670         int i, n;
1671         uint32_t bar0;
1672
1673         if (is_t4(sc)) {
1674                 /*
1675                  * Read low 32b of bar0 indirectly via the hardware backdoor
1676                  * mechanism.  Works from within PCI passthrough environments
1677                  * too, where rman_get_start() can return a different value.  We
1678                  * need to program the T4 memory window decoders with the actual
1679                  * addresses that will be coming across the PCIe link.
1680                  */
1681                 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1682                 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1683
1684                 mw = &t4_memwin[0];
1685                 n = nitems(t4_memwin);
1686         } else {
1687                 /* T5 uses the relative offset inside the PCIe BAR */
1688                 bar0 = 0;
1689
1690                 mw = &t5_memwin[0];
1691                 n = nitems(t5_memwin);
1692         }
1693
1694         for (i = 0; i < n; i++, mw++) {
1695                 t4_write_reg(sc,
1696                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
1697                     (mw->base + bar0) | V_BIR(0) |
1698                     V_WINDOW(ilog2(mw->aperture) - 10));
1699         }
1700
1701         /* flush */
1702         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1703 }
1704
1705 /*
1706  * Verify that the memory range specified by the addr/len pair is valid and lies
1707  * entirely within a single region (EDCx or MCx).
1708  */
1709 static int
1710 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
1711 {
1712         uint32_t em, addr_len, maddr, mlen;
1713
1714         /* Memory can only be accessed in naturally aligned 4 byte units */
1715         if (addr & 3 || len & 3 || len == 0)
1716                 return (EINVAL);
1717
1718         /* Enabled memories */
1719         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1720         if (em & F_EDRAM0_ENABLE) {
1721                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1722                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1723                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1724                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1725                     addr + len <= maddr + mlen)
1726                         return (0);
1727         }
1728         if (em & F_EDRAM1_ENABLE) {
1729                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1730                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1731                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1732                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1733                     addr + len <= maddr + mlen)
1734                         return (0);
1735         }
1736         if (em & F_EXT_MEM_ENABLE) {
1737                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1738                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1739                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1740                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1741                     addr + len <= maddr + mlen)
1742                         return (0);
1743         }
1744         if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
1745                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1746                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1747                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1748                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1749                     addr + len <= maddr + mlen)
1750                         return (0);
1751         }
1752
1753         return (EFAULT);
1754 }
1755
1756 static int
1757 fwmtype_to_hwmtype(int mtype)
1758 {
1759
1760         switch (mtype) {
1761         case FW_MEMTYPE_EDC0:
1762                 return (MEM_EDC0);
1763         case FW_MEMTYPE_EDC1:
1764                 return (MEM_EDC1);
1765         case FW_MEMTYPE_EXTMEM:
1766                 return (MEM_MC0);
1767         case FW_MEMTYPE_EXTMEM1:
1768                 return (MEM_MC1);
1769         default:
1770                 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
1771         }
1772 }
1773
1774 /*
1775  * Verify that the memory range specified by the memtype/offset/len pair is
1776  * valid and lies entirely within the memtype specified.  The global address of
1777  * the start of the range is returned in addr.
1778  */
1779 static int
1780 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1781     uint32_t *addr)
1782 {
1783         uint32_t em, addr_len, maddr, mlen;
1784
1785         /* Memory can only be accessed in naturally aligned 4 byte units */
1786         if (off & 3 || len & 3 || len == 0)
1787                 return (EINVAL);
1788
1789         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1790         switch (fwmtype_to_hwmtype(mtype)) {
1791         case MEM_EDC0:
1792                 if (!(em & F_EDRAM0_ENABLE))
1793                         return (EINVAL);
1794                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1795                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1796                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1797                 break;
1798         case MEM_EDC1:
1799                 if (!(em & F_EDRAM1_ENABLE))
1800                         return (EINVAL);
1801                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1802                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1803                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1804                 break;
1805         case MEM_MC:
1806                 if (!(em & F_EXT_MEM_ENABLE))
1807                         return (EINVAL);
1808                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1809                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1810                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1811                 break;
1812         case MEM_MC1:
1813                 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
1814                         return (EINVAL);
1815                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1816                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1817                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1818                 break;
1819         default:
1820                 return (EINVAL);
1821         }
1822
1823         if (mlen > 0 && off < mlen && off + len <= mlen) {
1824                 *addr = maddr + off;    /* global address */
1825                 return (0);
1826         }
1827
1828         return (EFAULT);
1829 }
1830
1831 static void
1832 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1833 {
1834         const struct memwin *mw;
1835
1836         if (is_t4(sc)) {
1837                 KASSERT(win >= 0 && win < nitems(t4_memwin),
1838                     ("%s: incorrect memwin# (%d)", __func__, win));
1839                 mw = &t4_memwin[win];
1840         } else {
1841                 KASSERT(win >= 0 && win < nitems(t5_memwin),
1842                     ("%s: incorrect memwin# (%d)", __func__, win));
1843                 mw = &t5_memwin[win];
1844         }
1845
1846         if (base != NULL)
1847                 *base = mw->base;
1848         if (aperture != NULL)
1849                 *aperture = mw->aperture;
1850 }
1851
1852 /*
1853  * Positions the memory window such that it can be used to access the specified
1854  * address in the chip's address space.  The return value is the offset of addr
1855  * from the start of the window.
1856  */
1857 static uint32_t
1858 position_memwin(struct adapter *sc, int n, uint32_t addr)
1859 {
1860         uint32_t start, pf;
1861         uint32_t reg;
1862
1863         KASSERT(n >= 0 && n <= 3,
1864             ("%s: invalid window %d.", __func__, n));
1865         KASSERT((addr & 3) == 0,
1866             ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
1867
1868         if (is_t4(sc)) {
1869                 pf = 0;
1870                 start = addr & ~0xf;    /* start must be 16B aligned */
1871         } else {
1872                 pf = V_PFNUM(sc->pf);
1873                 start = addr & ~0x7f;   /* start must be 128B aligned */
1874         }
1875         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1876
1877         t4_write_reg(sc, reg, start | pf);
1878         t4_read_reg(sc, reg);
1879
1880         return (addr - start);
1881 }
1882
1883 static int
1884 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1885     struct intrs_and_queues *iaq)
1886 {
1887         int rc, itype, navail, nrxq10g, nrxq1g, n;
1888         int nofldrxq10g = 0, nofldrxq1g = 0;
1889         int nnmrxq10g = 0, nnmrxq1g = 0;
1890
1891         bzero(iaq, sizeof(*iaq));
1892
1893         iaq->ntxq10g = t4_ntxq10g;
1894         iaq->ntxq1g = t4_ntxq1g;
1895         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1896         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1897         iaq->rsrv_noflowq = t4_rsrv_noflowq;
1898 #ifdef TCP_OFFLOAD
1899         if (is_offload(sc)) {
1900                 iaq->nofldtxq10g = t4_nofldtxq10g;
1901                 iaq->nofldtxq1g = t4_nofldtxq1g;
1902                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1903                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1904         }
1905 #endif
1906 #ifdef DEV_NETMAP
1907         iaq->nnmtxq10g = t4_nnmtxq10g;
1908         iaq->nnmtxq1g = t4_nnmtxq1g;
1909         iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g;
1910         iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g;
1911 #endif
1912
1913         for (itype = INTR_MSIX; itype; itype >>= 1) {
1914
1915                 if ((itype & t4_intr_types) == 0)
1916                         continue;       /* not allowed */
1917
1918                 if (itype == INTR_MSIX)
1919                         navail = pci_msix_count(sc->dev);
1920                 else if (itype == INTR_MSI)
1921                         navail = pci_msi_count(sc->dev);
1922                 else
1923                         navail = 1;
1924 restart:
1925                 if (navail == 0)
1926                         continue;
1927
1928                 iaq->intr_type = itype;
1929                 iaq->intr_flags_10g = 0;
1930                 iaq->intr_flags_1g = 0;
1931
1932                 /*
1933                  * Best option: an interrupt vector for errors, one for the
1934                  * firmware event queue, and one for every rxq (NIC, TOE, and
1935                  * netmap).
1936                  */
1937                 iaq->nirq = T4_EXTRA_INTR;
1938                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g);
1939                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g);
1940                 if (iaq->nirq <= navail &&
1941                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
1942                         iaq->intr_flags_10g = INTR_ALL;
1943                         iaq->intr_flags_1g = INTR_ALL;
1944                         goto allocate;
1945                 }
1946
1947                 /*
1948                  * Second best option: a vector for errors, one for the firmware
1949                  * event queue, and vectors for either all the NIC rx queues or
1950                  * all the TOE rx queues.  The queues that don't get vectors
1951                  * will forward their interrupts to those that do.
1952                  *
1953                  * Note: netmap rx queues cannot be created early and so they
1954                  * can't be setup to receive forwarded interrupts for others.
1955                  */
1956                 iaq->nirq = T4_EXTRA_INTR;
1957                 if (nrxq10g >= nofldrxq10g) {
1958                         iaq->intr_flags_10g = INTR_RXQ;
1959                         iaq->nirq += n10g * nrxq10g;
1960 #ifdef DEV_NETMAP
1961                         iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g);
1962 #endif
1963                 } else {
1964                         iaq->intr_flags_10g = INTR_OFLD_RXQ;
1965                         iaq->nirq += n10g * nofldrxq10g;
1966 #ifdef DEV_NETMAP
1967                         iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g);
1968 #endif
1969                 }
1970                 if (nrxq1g >= nofldrxq1g) {
1971                         iaq->intr_flags_1g = INTR_RXQ;
1972                         iaq->nirq += n1g * nrxq1g;
1973 #ifdef DEV_NETMAP
1974                         iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g);
1975 #endif
1976                 } else {
1977                         iaq->intr_flags_1g = INTR_OFLD_RXQ;
1978                         iaq->nirq += n1g * nofldrxq1g;
1979 #ifdef DEV_NETMAP
1980                         iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g);
1981 #endif
1982                 }
1983                 if (iaq->nirq <= navail &&
1984                     (itype != INTR_MSI || powerof2(iaq->nirq)))
1985                         goto allocate;
1986
1987                 /*
1988                  * Next best option: an interrupt vector for errors, one for the
1989                  * firmware event queue, and at least one per port.  At this
1990                  * point we know we'll have to downsize nrxq and/or nofldrxq
1991                  * and/or nnmrxq to fit what's available to us.
1992                  */
1993                 iaq->nirq = T4_EXTRA_INTR;
1994                 iaq->nirq += n10g + n1g;
1995                 if (iaq->nirq <= navail) {
1996                         int leftover = navail - iaq->nirq;
1997
1998                         if (n10g > 0) {
1999                                 int target = max(nrxq10g, nofldrxq10g);
2000
2001                                 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
2002                                     INTR_RXQ : INTR_OFLD_RXQ;
2003
2004                                 n = 1;
2005                                 while (n < target && leftover >= n10g) {
2006                                         leftover -= n10g;
2007                                         iaq->nirq += n10g;
2008                                         n++;
2009                                 }
2010                                 iaq->nrxq10g = min(n, nrxq10g);
2011 #ifdef TCP_OFFLOAD
2012                                 iaq->nofldrxq10g = min(n, nofldrxq10g);
2013 #endif
2014 #ifdef DEV_NETMAP
2015                                 iaq->nnmrxq10g = min(n, nnmrxq10g);
2016 #endif
2017                         }
2018
2019                         if (n1g > 0) {
2020                                 int target = max(nrxq1g, nofldrxq1g);
2021
2022                                 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
2023                                     INTR_RXQ : INTR_OFLD_RXQ;
2024
2025                                 n = 1;
2026                                 while (n < target && leftover >= n1g) {
2027                                         leftover -= n1g;
2028                                         iaq->nirq += n1g;
2029                                         n++;
2030                                 }
2031                                 iaq->nrxq1g = min(n, nrxq1g);
2032 #ifdef TCP_OFFLOAD
2033                                 iaq->nofldrxq1g = min(n, nofldrxq1g);
2034 #endif
2035 #ifdef DEV_NETMAP
2036                                 iaq->nnmrxq1g = min(n, nnmrxq1g);
2037 #endif
2038                         }
2039
2040                         if (itype != INTR_MSI || powerof2(iaq->nirq))
2041                                 goto allocate;
2042                 }
2043
2044                 /*
2045                  * Least desirable option: one interrupt vector for everything.
2046                  */
2047                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
2048                 iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
2049 #ifdef TCP_OFFLOAD
2050                 if (is_offload(sc))
2051                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2052 #endif
2053 #ifdef DEV_NETMAP
2054                 iaq->nnmrxq10g = iaq->nnmrxq1g = 1;
2055 #endif
2056
2057 allocate:
2058                 navail = iaq->nirq;
2059                 rc = 0;
2060                 if (itype == INTR_MSIX)
2061                         rc = pci_alloc_msix(sc->dev, &navail);
2062                 else if (itype == INTR_MSI)
2063                         rc = pci_alloc_msi(sc->dev, &navail);
2064
2065                 if (rc == 0) {
2066                         if (navail == iaq->nirq)
2067                                 return (0);
2068
2069                         /*
2070                          * Didn't get the number requested.  Use whatever number
2071                          * the kernel is willing to allocate (it's in navail).
2072                          */
2073                         device_printf(sc->dev, "fewer vectors than requested, "
2074                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
2075                             itype, iaq->nirq, navail);
2076                         pci_release_msi(sc->dev);
2077                         goto restart;
2078                 }
2079
2080                 device_printf(sc->dev,
2081                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
2082                     itype, rc, iaq->nirq, navail);
2083         }
2084
2085         device_printf(sc->dev,
2086             "failed to find a usable interrupt type.  "
2087             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
2088             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
2089
2090         return (ENXIO);
2091 }
2092
2093 #define FW_VERSION(chip) ( \
2094     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
2095     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
2096     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
2097     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
2098 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
2099
2100 struct fw_info {
2101         uint8_t chip;
2102         char *kld_name;
2103         char *fw_mod_name;
2104         struct fw_hdr fw_hdr;   /* XXX: waste of space, need a sparse struct */
2105 } fw_info[] = {
2106         {
2107                 .chip = CHELSIO_T4,
2108                 .kld_name = "t4fw_cfg",
2109                 .fw_mod_name = "t4fw",
2110                 .fw_hdr = {
2111                         .chip = FW_HDR_CHIP_T4,
2112                         .fw_ver = htobe32_const(FW_VERSION(T4)),
2113                         .intfver_nic = FW_INTFVER(T4, NIC),
2114                         .intfver_vnic = FW_INTFVER(T4, VNIC),
2115                         .intfver_ofld = FW_INTFVER(T4, OFLD),
2116                         .intfver_ri = FW_INTFVER(T4, RI),
2117                         .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
2118                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
2119                         .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
2120                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
2121                 },
2122         }, {
2123                 .chip = CHELSIO_T5,
2124                 .kld_name = "t5fw_cfg",
2125                 .fw_mod_name = "t5fw",
2126                 .fw_hdr = {
2127                         .chip = FW_HDR_CHIP_T5,
2128                         .fw_ver = htobe32_const(FW_VERSION(T5)),
2129                         .intfver_nic = FW_INTFVER(T5, NIC),
2130                         .intfver_vnic = FW_INTFVER(T5, VNIC),
2131                         .intfver_ofld = FW_INTFVER(T5, OFLD),
2132                         .intfver_ri = FW_INTFVER(T5, RI),
2133                         .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
2134                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
2135                         .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
2136                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
2137                 },
2138         }
2139 };
2140
2141 static struct fw_info *
2142 find_fw_info(int chip)
2143 {
2144         int i;
2145
2146         for (i = 0; i < nitems(fw_info); i++) {
2147                 if (fw_info[i].chip == chip)
2148                         return (&fw_info[i]);
2149         }
2150         return (NULL);
2151 }
2152
2153 /*
2154  * Is the given firmware API compatible with the one the driver was compiled
2155  * with?
2156  */
2157 static int
2158 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
2159 {
2160
2161         /* short circuit if it's the exact same firmware version */
2162         if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
2163                 return (1);
2164
2165         /*
2166          * XXX: Is this too conservative?  Perhaps I should limit this to the
2167          * features that are supported in the driver.
2168          */
2169 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
2170         if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
2171             SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
2172             SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
2173                 return (1);
2174 #undef SAME_INTF
2175
2176         return (0);
2177 }
2178
2179 /*
2180  * The firmware in the KLD is usable, but should it be installed?  This routine
2181  * explains itself in detail if it indicates the KLD firmware should be
2182  * installed.
2183  */
2184 static int
2185 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
2186 {
2187         const char *reason;
2188
2189         if (!card_fw_usable) {
2190                 reason = "incompatible or unusable";
2191                 goto install;
2192         }
2193
2194         if (k > c) {
2195                 reason = "older than the version bundled with this driver";
2196                 goto install;
2197         }
2198
2199         if (t4_fw_install == 2 && k != c) {
2200                 reason = "different than the version bundled with this driver";
2201                 goto install;
2202         }
2203
2204         return (0);
2205
2206 install:
2207         if (t4_fw_install == 0) {
2208                 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2209                     "but the driver is prohibited from installing a different "
2210                     "firmware on the card.\n",
2211                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2212                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2213
2214                 return (0);
2215         }
2216
2217         device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2218             "installing firmware %u.%u.%u.%u on card.\n",
2219             G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2220             G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2221             G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2222             G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2223
2224         return (1);
2225 }
2226 /*
2227  * Establish contact with the firmware and determine if we are the master driver
2228  * or not, and whether we are responsible for chip initialization.
2229  */
2230 static int
2231 prep_firmware(struct adapter *sc)
2232 {
2233         const struct firmware *fw = NULL, *default_cfg;
2234         int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2235         enum dev_state state;
2236         struct fw_info *fw_info;
2237         struct fw_hdr *card_fw;         /* fw on the card */
2238         const struct fw_hdr *kld_fw;    /* fw in the KLD */
2239         const struct fw_hdr *drv_fw;    /* fw header the driver was compiled
2240                                            against */
2241
2242         /* Contact firmware. */
2243         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2244         if (rc < 0 || state == DEV_STATE_ERR) {
2245                 rc = -rc;
2246                 device_printf(sc->dev,
2247                     "failed to connect to the firmware: %d, %d.\n", rc, state);
2248                 return (rc);
2249         }
2250         pf = rc;
2251         if (pf == sc->mbox)
2252                 sc->flags |= MASTER_PF;
2253         else if (state == DEV_STATE_UNINIT) {
2254                 /*
2255                  * We didn't get to be the master so we definitely won't be
2256                  * configuring the chip.  It's a bug if someone else hasn't
2257                  * configured it already.
2258                  */
2259                 device_printf(sc->dev, "couldn't be master(%d), "
2260                     "device not already initialized either(%d).\n", rc, state);
2261                 return (EDOOFUS);
2262         }
2263
2264         /* This is the firmware whose headers the driver was compiled against */
2265         fw_info = find_fw_info(chip_id(sc));
2266         if (fw_info == NULL) {
2267                 device_printf(sc->dev,
2268                     "unable to look up firmware information for chip %d.\n",
2269                     chip_id(sc));
2270                 return (EINVAL);
2271         }
2272         drv_fw = &fw_info->fw_hdr;
2273
2274         /*
2275          * The firmware KLD contains many modules.  The KLD name is also the
2276          * name of the module that contains the default config file.
2277          */
2278         default_cfg = firmware_get(fw_info->kld_name);
2279
2280         /* Read the header of the firmware on the card */
2281         card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2282         rc = -t4_read_flash(sc, FLASH_FW_START,
2283             sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2284         if (rc == 0)
2285                 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2286         else {
2287                 device_printf(sc->dev,
2288                     "Unable to read card's firmware header: %d\n", rc);
2289                 card_fw_usable = 0;
2290         }
2291
2292         /* This is the firmware in the KLD */
2293         fw = firmware_get(fw_info->fw_mod_name);
2294         if (fw != NULL) {
2295                 kld_fw = (const void *)fw->data;
2296                 kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2297         } else {
2298                 kld_fw = NULL;
2299                 kld_fw_usable = 0;
2300         }
2301
2302         if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2303             (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2304                 /*
2305                  * Common case: the firmware on the card is an exact match and
2306                  * the KLD is an exact match too, or the KLD is
2307                  * absent/incompatible.  Note that t4_fw_install = 2 is ignored
2308                  * here -- use cxgbetool loadfw if you want to reinstall the
2309                  * same firmware as the one on the card.
2310                  */
2311         } else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2312             should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2313             be32toh(card_fw->fw_ver))) {
2314
2315                 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
2316                 if (rc != 0) {
2317                         device_printf(sc->dev,
2318                             "failed to install firmware: %d\n", rc);
2319                         goto done;
2320                 }
2321
2322                 /* Installed successfully, update the cached header too. */
2323                 memcpy(card_fw, kld_fw, sizeof(*card_fw));
2324                 card_fw_usable = 1;
2325                 need_fw_reset = 0;      /* already reset as part of load_fw */
2326         }
2327
2328         if (!card_fw_usable) {
2329                 uint32_t d, c, k;
2330
2331                 d = ntohl(drv_fw->fw_ver);
2332                 c = ntohl(card_fw->fw_ver);
2333                 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
2334
2335                 device_printf(sc->dev, "Cannot find a usable firmware: "
2336                     "fw_install %d, chip state %d, "
2337                     "driver compiled with %d.%d.%d.%d, "
2338                     "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
2339                     t4_fw_install, state,
2340                     G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
2341                     G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
2342                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2343                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
2344                     G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2345                     G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2346                 rc = EINVAL;
2347                 goto done;
2348         }
2349
2350         /* We're using whatever's on the card and it's known to be good. */
2351         sc->params.fw_vers = ntohl(card_fw->fw_ver);
2352         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
2353             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2354             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2355             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2356             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2357         t4_get_tp_version(sc, &sc->params.tp_vers);
2358
2359         /* Reset device */
2360         if (need_fw_reset &&
2361             (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
2362                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
2363                 if (rc != ETIMEDOUT && rc != EIO)
2364                         t4_fw_bye(sc, sc->mbox);
2365                 goto done;
2366         }
2367         sc->flags |= FW_OK;
2368
2369         rc = get_params__pre_init(sc);
2370         if (rc != 0)
2371                 goto done; /* error message displayed already */
2372
2373         /* Partition adapter resources as specified in the config file. */
2374         if (state == DEV_STATE_UNINIT) {
2375
2376                 KASSERT(sc->flags & MASTER_PF,
2377                     ("%s: trying to change chip settings when not master.",
2378                     __func__));
2379
2380                 rc = partition_resources(sc, default_cfg, fw_info->kld_name);
2381                 if (rc != 0)
2382                         goto done;      /* error message displayed already */
2383
2384                 t4_tweak_chip_settings(sc);
2385
2386                 /* get basic stuff going */
2387                 rc = -t4_fw_initialize(sc, sc->mbox);
2388                 if (rc != 0) {
2389                         device_printf(sc->dev, "fw init failed: %d.\n", rc);
2390                         goto done;
2391                 }
2392         } else {
2393                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
2394                 sc->cfcsum = 0;
2395         }
2396
2397 done:
2398         free(card_fw, M_CXGBE);
2399         if (fw != NULL)
2400                 firmware_put(fw, FIRMWARE_UNLOAD);
2401         if (default_cfg != NULL)
2402                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
2403
2404         return (rc);
2405 }
2406
2407 #define FW_PARAM_DEV(param) \
2408         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2409          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2410 #define FW_PARAM_PFVF(param) \
2411         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2412          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2413
2414 /*
2415  * Partition chip resources for use between various PFs, VFs, etc.
2416  */
2417 static int
2418 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
2419     const char *name_prefix)
2420 {
2421         const struct firmware *cfg = NULL;
2422         int rc = 0;
2423         struct fw_caps_config_cmd caps;
2424         uint32_t mtype, moff, finicsum, cfcsum;
2425
2426         /*
2427          * Figure out what configuration file to use.  Pick the default config
2428          * file for the card if the user hasn't specified one explicitly.
2429          */
2430         snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
2431         if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
2432                 /* Card specific overrides go here. */
2433                 if (pci_get_device(sc->dev) == 0x440a)
2434                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
2435                 if (is_fpga(sc))
2436                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
2437         }
2438
2439         /*
2440          * We need to load another module if the profile is anything except
2441          * "default" or "flash".
2442          */
2443         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
2444             strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2445                 char s[32];
2446
2447                 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
2448                 cfg = firmware_get(s);
2449                 if (cfg == NULL) {
2450                         if (default_cfg != NULL) {
2451                                 device_printf(sc->dev,
2452                                     "unable to load module \"%s\" for "
2453                                     "configuration profile \"%s\", will use "
2454                                     "the default config file instead.\n",
2455                                     s, sc->cfg_file);
2456                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2457                                     "%s", DEFAULT_CF);
2458                         } else {
2459                                 device_printf(sc->dev,
2460                                     "unable to load module \"%s\" for "
2461                                     "configuration profile \"%s\", will use "
2462                                     "the config file on the card's flash "
2463                                     "instead.\n", s, sc->cfg_file);
2464                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2465                                     "%s", FLASH_CF);
2466                         }
2467                 }
2468         }
2469
2470         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
2471             default_cfg == NULL) {
2472                 device_printf(sc->dev,
2473                     "default config file not available, will use the config "
2474                     "file on the card's flash instead.\n");
2475                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
2476         }
2477
2478         if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2479                 u_int cflen, i, n;
2480                 const uint32_t *cfdata;
2481                 uint32_t param, val, addr, off, mw_base, mw_aperture;
2482
2483                 KASSERT(cfg != NULL || default_cfg != NULL,
2484                     ("%s: no config to upload", __func__));
2485
2486                 /*
2487                  * Ask the firmware where it wants us to upload the config file.
2488                  */
2489                 param = FW_PARAM_DEV(CF);
2490                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2491                 if (rc != 0) {
2492                         /* No support for config file?  Shouldn't happen. */
2493                         device_printf(sc->dev,
2494                             "failed to query config file location: %d.\n", rc);
2495                         goto done;
2496                 }
2497                 mtype = G_FW_PARAMS_PARAM_Y(val);
2498                 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
2499
2500                 /*
2501                  * XXX: sheer laziness.  We deliberately added 4 bytes of
2502                  * useless stuffing/comments at the end of the config file so
2503                  * it's ok to simply throw away the last remaining bytes when
2504                  * the config file is not an exact multiple of 4.  This also
2505                  * helps with the validate_mt_off_len check.
2506                  */
2507                 if (cfg != NULL) {
2508                         cflen = cfg->datasize & ~3;
2509                         cfdata = cfg->data;
2510                 } else {
2511                         cflen = default_cfg->datasize & ~3;
2512                         cfdata = default_cfg->data;
2513                 }
2514
2515                 if (cflen > FLASH_CFG_MAX_SIZE) {
2516                         device_printf(sc->dev,
2517                             "config file too long (%d, max allowed is %d).  "
2518                             "Will try to use the config on the card, if any.\n",
2519                             cflen, FLASH_CFG_MAX_SIZE);
2520                         goto use_config_on_flash;
2521                 }
2522
2523                 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
2524                 if (rc != 0) {
2525                         device_printf(sc->dev,
2526                             "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
2527                             "Will try to use the config on the card, if any.\n",
2528                             __func__, mtype, moff, cflen, rc);
2529                         goto use_config_on_flash;
2530                 }
2531
2532                 memwin_info(sc, 2, &mw_base, &mw_aperture);
2533                 while (cflen) {
2534                         off = position_memwin(sc, 2, addr);
2535                         n = min(cflen, mw_aperture - off);
2536                         for (i = 0; i < n; i += 4)
2537                                 t4_write_reg(sc, mw_base + off + i, *cfdata++);
2538                         cflen -= n;
2539                         addr += n;
2540                 }
2541         } else {
2542 use_config_on_flash:
2543                 mtype = FW_MEMTYPE_FLASH;
2544                 moff = t4_flash_cfg_addr(sc);
2545         }
2546
2547         bzero(&caps, sizeof(caps));
2548         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2549             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2550         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
2551             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
2552             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
2553         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2554         if (rc != 0) {
2555                 device_printf(sc->dev,
2556                     "failed to pre-process config file: %d "
2557                     "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
2558                 goto done;
2559         }
2560
2561         finicsum = be32toh(caps.finicsum);
2562         cfcsum = be32toh(caps.cfcsum);
2563         if (finicsum != cfcsum) {
2564                 device_printf(sc->dev,
2565                     "WARNING: config file checksum mismatch: %08x %08x\n",
2566                     finicsum, cfcsum);
2567         }
2568         sc->cfcsum = cfcsum;
2569
2570 #define LIMIT_CAPS(x) do { \
2571         caps.x &= htobe16(t4_##x##_allowed); \
2572 } while (0)
2573
2574         /*
2575          * Let the firmware know what features will (not) be used so it can tune
2576          * things accordingly.
2577          */
2578         LIMIT_CAPS(linkcaps);
2579         LIMIT_CAPS(niccaps);
2580         LIMIT_CAPS(toecaps);
2581         LIMIT_CAPS(rdmacaps);
2582         LIMIT_CAPS(iscsicaps);
2583         LIMIT_CAPS(fcoecaps);
2584 #undef LIMIT_CAPS
2585
2586         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2587             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
2588         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2589         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
2590         if (rc != 0) {
2591                 device_printf(sc->dev,
2592                     "failed to process config file: %d.\n", rc);
2593         }
2594 done:
2595         if (cfg != NULL)
2596                 firmware_put(cfg, FIRMWARE_UNLOAD);
2597         return (rc);
2598 }
2599
2600 /*
2601  * Retrieve parameters that are needed (or nice to have) very early.
2602  */
2603 static int
2604 get_params__pre_init(struct adapter *sc)
2605 {
2606         int rc;
2607         uint32_t param[2], val[2];
2608         struct fw_devlog_cmd cmd;
2609         struct devlog_params *dlog = &sc->params.devlog;
2610
2611         param[0] = FW_PARAM_DEV(PORTVEC);
2612         param[1] = FW_PARAM_DEV(CCLK);
2613         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2614         if (rc != 0) {
2615                 device_printf(sc->dev,
2616                     "failed to query parameters (pre_init): %d.\n", rc);
2617                 return (rc);
2618         }
2619
2620         sc->params.portvec = val[0];
2621         sc->params.nports = bitcount32(val[0]);
2622         sc->params.vpd.cclk = val[1];
2623
2624         /* Read device log parameters. */
2625         bzero(&cmd, sizeof(cmd));
2626         cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
2627             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2628         cmd.retval_len16 = htobe32(FW_LEN16(cmd));
2629         rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
2630         if (rc != 0) {
2631                 device_printf(sc->dev,
2632                     "failed to get devlog parameters: %d.\n", rc);
2633                 bzero(dlog, sizeof (*dlog));
2634                 rc = 0; /* devlog isn't critical for device operation */
2635         } else {
2636                 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
2637                 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
2638                 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
2639                 dlog->size = be32toh(cmd.memsize_devlog);
2640         }
2641
2642         return (rc);
2643 }
2644
2645 /*
2646  * Retrieve various parameters that are of interest to the driver.  The device
2647  * has been initialized by the firmware at this point.
2648  */
2649 static int
2650 get_params__post_init(struct adapter *sc)
2651 {
2652         int rc;
2653         uint32_t param[7], val[7];
2654         struct fw_caps_config_cmd caps;
2655
2656         param[0] = FW_PARAM_PFVF(IQFLINT_START);
2657         param[1] = FW_PARAM_PFVF(EQ_START);
2658         param[2] = FW_PARAM_PFVF(FILTER_START);
2659         param[3] = FW_PARAM_PFVF(FILTER_END);
2660         param[4] = FW_PARAM_PFVF(L2T_START);
2661         param[5] = FW_PARAM_PFVF(L2T_END);
2662         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2663         if (rc != 0) {
2664                 device_printf(sc->dev,
2665                     "failed to query parameters (post_init): %d.\n", rc);
2666                 return (rc);
2667         }
2668
2669         sc->sge.iq_start = val[0];
2670         sc->sge.eq_start = val[1];
2671         sc->tids.ftid_base = val[2];
2672         sc->tids.nftids = val[3] - val[2] + 1;
2673         sc->params.ftid_min = val[2];
2674         sc->params.ftid_max = val[3];
2675         sc->vres.l2t.start = val[4];
2676         sc->vres.l2t.size = val[5] - val[4] + 1;
2677         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
2678             ("%s: L2 table size (%u) larger than expected (%u)",
2679             __func__, sc->vres.l2t.size, L2T_SIZE));
2680
2681         /* get capabilites */
2682         bzero(&caps, sizeof(caps));
2683         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2684             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2685         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2686         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2687         if (rc != 0) {
2688                 device_printf(sc->dev,
2689                     "failed to get card capabilities: %d.\n", rc);
2690                 return (rc);
2691         }
2692
2693 #define READ_CAPS(x) do { \
2694         sc->x = htobe16(caps.x); \
2695 } while (0)
2696         READ_CAPS(linkcaps);
2697         READ_CAPS(niccaps);
2698         READ_CAPS(toecaps);
2699         READ_CAPS(rdmacaps);
2700         READ_CAPS(iscsicaps);
2701         READ_CAPS(fcoecaps);
2702
2703         if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
2704                 param[0] = FW_PARAM_PFVF(ETHOFLD_START);
2705                 param[1] = FW_PARAM_PFVF(ETHOFLD_END);
2706                 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2707                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
2708                 if (rc != 0) {
2709                         device_printf(sc->dev,
2710                             "failed to query NIC parameters: %d.\n", rc);
2711                         return (rc);
2712                 }
2713                 sc->tids.etid_base = val[0];
2714                 sc->params.etid_min = val[0];
2715                 sc->tids.netids = val[1] - val[0] + 1;
2716                 sc->params.netids = sc->tids.netids;
2717                 sc->params.eo_wr_cred = val[2];
2718                 sc->params.ethoffload = 1;
2719         }
2720
2721         if (sc->toecaps) {
2722                 /* query offload-related parameters */
2723                 param[0] = FW_PARAM_DEV(NTID);
2724                 param[1] = FW_PARAM_PFVF(SERVER_START);
2725                 param[2] = FW_PARAM_PFVF(SERVER_END);
2726                 param[3] = FW_PARAM_PFVF(TDDP_START);
2727                 param[4] = FW_PARAM_PFVF(TDDP_END);
2728                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2729                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2730                 if (rc != 0) {
2731                         device_printf(sc->dev,
2732                             "failed to query TOE parameters: %d.\n", rc);
2733                         return (rc);
2734                 }
2735                 sc->tids.ntids = val[0];
2736                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
2737                 sc->tids.stid_base = val[1];
2738                 sc->tids.nstids = val[2] - val[1] + 1;
2739                 sc->vres.ddp.start = val[3];
2740                 sc->vres.ddp.size = val[4] - val[3] + 1;
2741                 sc->params.ofldq_wr_cred = val[5];
2742                 sc->params.offload = 1;
2743         }
2744         if (sc->rdmacaps) {
2745                 param[0] = FW_PARAM_PFVF(STAG_START);
2746                 param[1] = FW_PARAM_PFVF(STAG_END);
2747                 param[2] = FW_PARAM_PFVF(RQ_START);
2748                 param[3] = FW_PARAM_PFVF(RQ_END);
2749                 param[4] = FW_PARAM_PFVF(PBL_START);
2750                 param[5] = FW_PARAM_PFVF(PBL_END);
2751                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2752                 if (rc != 0) {
2753                         device_printf(sc->dev,
2754                             "failed to query RDMA parameters(1): %d.\n", rc);
2755                         return (rc);
2756                 }
2757                 sc->vres.stag.start = val[0];
2758                 sc->vres.stag.size = val[1] - val[0] + 1;
2759                 sc->vres.rq.start = val[2];
2760                 sc->vres.rq.size = val[3] - val[2] + 1;
2761                 sc->vres.pbl.start = val[4];
2762                 sc->vres.pbl.size = val[5] - val[4] + 1;
2763
2764                 param[0] = FW_PARAM_PFVF(SQRQ_START);
2765                 param[1] = FW_PARAM_PFVF(SQRQ_END);
2766                 param[2] = FW_PARAM_PFVF(CQ_START);
2767                 param[3] = FW_PARAM_PFVF(CQ_END);
2768                 param[4] = FW_PARAM_PFVF(OCQ_START);
2769                 param[5] = FW_PARAM_PFVF(OCQ_END);
2770                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2771                 if (rc != 0) {
2772                         device_printf(sc->dev,
2773                             "failed to query RDMA parameters(2): %d.\n", rc);
2774                         return (rc);
2775                 }
2776                 sc->vres.qp.start = val[0];
2777                 sc->vres.qp.size = val[1] - val[0] + 1;
2778                 sc->vres.cq.start = val[2];
2779                 sc->vres.cq.size = val[3] - val[2] + 1;
2780                 sc->vres.ocq.start = val[4];
2781                 sc->vres.ocq.size = val[5] - val[4] + 1;
2782         }
2783         if (sc->iscsicaps) {
2784                 param[0] = FW_PARAM_PFVF(ISCSI_START);
2785                 param[1] = FW_PARAM_PFVF(ISCSI_END);
2786                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2787                 if (rc != 0) {
2788                         device_printf(sc->dev,
2789                             "failed to query iSCSI parameters: %d.\n", rc);
2790                         return (rc);
2791                 }
2792                 sc->vres.iscsi.start = val[0];
2793                 sc->vres.iscsi.size = val[1] - val[0] + 1;
2794         }
2795
2796         /*
2797          * We've got the params we wanted to query via the firmware.  Now grab
2798          * some others directly from the chip.
2799          */
2800         rc = t4_read_chip_settings(sc);
2801
2802         return (rc);
2803 }
2804
2805 static int
2806 set_params__post_init(struct adapter *sc)
2807 {
2808         uint32_t param, val;
2809
2810         /* ask for encapsulated CPLs */
2811         param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
2812         val = 1;
2813         (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2814
2815         return (0);
2816 }
2817
2818 #undef FW_PARAM_PFVF
2819 #undef FW_PARAM_DEV
2820
2821 static void
2822 t4_set_desc(struct adapter *sc)
2823 {
2824         char buf[128];
2825         struct adapter_params *p = &sc->params;
2826
2827         snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
2828             "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
2829             chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
2830
2831         device_set_desc_copy(sc->dev, buf);
2832 }
2833
2834 static void
2835 build_medialist(struct port_info *pi, struct ifmedia *media)
2836 {
2837         int m;
2838
2839         PORT_LOCK(pi);
2840
2841         ifmedia_removeall(media);
2842
2843         m = IFM_ETHER | IFM_FDX;
2844
2845         switch(pi->port_type) {
2846         case FW_PORT_TYPE_BT_XFI:
2847         case FW_PORT_TYPE_BT_XAUI:
2848                 ifmedia_add(media, m | IFM_10G_T, 0, NULL);
2849                 /* fall through */
2850
2851         case FW_PORT_TYPE_BT_SGMII:
2852                 ifmedia_add(media, m | IFM_1000_T, 0, NULL);
2853                 ifmedia_add(media, m | IFM_100_TX, 0, NULL);
2854                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, 0, NULL);
2855                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2856                 break;
2857
2858         case FW_PORT_TYPE_CX4:
2859                 ifmedia_add(media, m | IFM_10G_CX4, 0, NULL);
2860                 ifmedia_set(media, m | IFM_10G_CX4);
2861                 break;
2862
2863         case FW_PORT_TYPE_QSFP_10G:
2864         case FW_PORT_TYPE_SFP:
2865         case FW_PORT_TYPE_FIBER_XFI:
2866         case FW_PORT_TYPE_FIBER_XAUI:
2867                 switch (pi->mod_type) {
2868
2869                 case FW_PORT_MOD_TYPE_LR:
2870                         ifmedia_add(media, m | IFM_10G_LR, 0, NULL);
2871                         ifmedia_set(media, m | IFM_10G_LR);
2872                         break;
2873
2874                 case FW_PORT_MOD_TYPE_SR:
2875                         ifmedia_add(media, m | IFM_10G_SR, 0, NULL);
2876                         ifmedia_set(media, m | IFM_10G_SR);
2877                         break;
2878
2879                 case FW_PORT_MOD_TYPE_LRM:
2880                         ifmedia_add(media, m | IFM_10G_LRM, 0, NULL);
2881                         ifmedia_set(media, m | IFM_10G_LRM);
2882                         break;
2883
2884                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2885                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2886                         ifmedia_add(media, m | IFM_10G_TWINAX, 0, NULL);
2887                         ifmedia_set(media, m | IFM_10G_TWINAX);
2888                         break;
2889
2890                 case FW_PORT_MOD_TYPE_NONE:
2891                         m &= ~IFM_FDX;
2892                         ifmedia_add(media, m | IFM_NONE, 0, NULL);
2893                         ifmedia_set(media, m | IFM_NONE);
2894                         break;
2895
2896                 case FW_PORT_MOD_TYPE_NA:
2897                 case FW_PORT_MOD_TYPE_ER:
2898                 default:
2899                         device_printf(pi->dev,
2900                             "unknown port_type (%d), mod_type (%d)\n",
2901                             pi->port_type, pi->mod_type);
2902                         ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
2903                         ifmedia_set(media, m | IFM_UNKNOWN);
2904                         break;
2905                 }
2906                 break;
2907
2908         case FW_PORT_TYPE_QSFP:
2909                 switch (pi->mod_type) {
2910
2911                 case FW_PORT_MOD_TYPE_LR:
2912                         ifmedia_add(media, m | IFM_40G_LR4, 0, NULL);
2913                         ifmedia_set(media, m | IFM_40G_LR4);
2914                         break;
2915
2916                 case FW_PORT_MOD_TYPE_SR:
2917                         ifmedia_add(media, m | IFM_40G_SR4, 0, NULL);
2918                         ifmedia_set(media, m | IFM_40G_SR4);
2919                         break;
2920
2921                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2922                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2923                         ifmedia_add(media, m | IFM_40G_CR4, 0, NULL);
2924                         ifmedia_set(media, m | IFM_40G_CR4);
2925                         break;
2926
2927                 case FW_PORT_MOD_TYPE_NONE:
2928                         m &= ~IFM_FDX;
2929                         ifmedia_add(media, m | IFM_NONE, 0, NULL);
2930                         ifmedia_set(media, m | IFM_NONE);
2931                         break;
2932
2933                 default:
2934                         device_printf(pi->dev,
2935                             "unknown port_type (%d), mod_type (%d)\n",
2936                             pi->port_type, pi->mod_type);
2937                         ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
2938                         ifmedia_set(media, m | IFM_UNKNOWN);
2939                         break;
2940                 }
2941                 break;
2942
2943         default:
2944                 device_printf(pi->dev,
2945                     "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
2946                     pi->mod_type);
2947                 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL);
2948                 ifmedia_set(media, m | IFM_UNKNOWN);
2949                 break;
2950         }
2951
2952         PORT_UNLOCK(pi);
2953 }
2954
2955 #define FW_MAC_EXACT_CHUNK      7
2956
2957 /*
2958  * Program the port's XGMAC based on parameters in ifnet.  The caller also
2959  * indicates which parameters should be programmed (the rest are left alone).
2960  */
2961 int
2962 update_mac_settings(struct ifnet *ifp, int flags)
2963 {
2964         int rc = 0;
2965         struct port_info *pi = ifp->if_softc;
2966         struct adapter *sc = pi->adapter;
2967         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
2968         uint16_t viid = 0xffff;
2969         int16_t *xact_addr_filt = NULL;
2970
2971         ASSERT_SYNCHRONIZED_OP(sc);
2972         KASSERT(flags, ("%s: not told what to update.", __func__));
2973
2974         if (ifp == pi->ifp) {
2975                 viid = pi->viid;
2976                 xact_addr_filt = &pi->xact_addr_filt;
2977         }
2978 #ifdef DEV_NETMAP
2979         else if (ifp == pi->nm_ifp) {
2980                 viid = pi->nm_viid;
2981                 xact_addr_filt = &pi->nm_xact_addr_filt;
2982         }
2983 #endif
2984         if (flags & XGMAC_MTU)
2985                 mtu = ifp->if_mtu;
2986
2987         if (flags & XGMAC_PROMISC)
2988                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
2989
2990         if (flags & XGMAC_ALLMULTI)
2991                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
2992
2993         if (flags & XGMAC_VLANEX)
2994                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
2995
2996         if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
2997                 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti,
2998                     1, vlanex, false);
2999                 if (rc) {
3000                         if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
3001                             rc);
3002                         return (rc);
3003                 }
3004         }
3005
3006         if (flags & XGMAC_UCADDR) {
3007                 uint8_t ucaddr[ETHER_ADDR_LEN];
3008
3009                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
3010                 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr,
3011                     true, true);
3012                 if (rc < 0) {
3013                         rc = -rc;
3014                         if_printf(ifp, "change_mac failed: %d\n", rc);
3015                         return (rc);
3016                 } else {
3017                         *xact_addr_filt = rc;
3018                         rc = 0;
3019                 }
3020         }
3021
3022         if (flags & XGMAC_MCADDRS) {
3023                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
3024                 int del = 1;
3025                 uint64_t hash = 0;
3026                 struct ifmultiaddr *ifma;
3027                 int i = 0, j;
3028
3029                 if_maddr_rlock(ifp);
3030                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
3031                         if (ifma->ifma_addr->sa_family != AF_LINK)
3032                                 continue;
3033                         mcaddr[i] =
3034                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
3035                         MPASS(ETHER_IS_MULTICAST(mcaddr[i]));
3036                         i++;
3037
3038                         if (i == FW_MAC_EXACT_CHUNK) {
3039                                 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del,
3040                                     i, mcaddr, NULL, &hash, 0);
3041                                 if (rc < 0) {
3042                                         rc = -rc;
3043                                         for (j = 0; j < i; j++) {
3044                                                 if_printf(ifp,
3045                                                     "failed to add mc address"
3046                                                     " %02x:%02x:%02x:"
3047                                                     "%02x:%02x:%02x rc=%d\n",
3048                                                     mcaddr[j][0], mcaddr[j][1],
3049                                                     mcaddr[j][2], mcaddr[j][3],
3050                                                     mcaddr[j][4], mcaddr[j][5],
3051                                                     rc);
3052                                         }
3053                                         goto mcfail;
3054                                 }
3055                                 del = 0;
3056                                 i = 0;
3057                         }
3058                 }
3059                 if (i > 0) {
3060                         rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i,
3061                             mcaddr, NULL, &hash, 0);
3062                         if (rc < 0) {
3063                                 rc = -rc;
3064                                 for (j = 0; j < i; j++) {
3065                                         if_printf(ifp,
3066                                             "failed to add mc address"
3067                                             " %02x:%02x:%02x:"
3068                                             "%02x:%02x:%02x rc=%d\n",
3069                                             mcaddr[j][0], mcaddr[j][1],
3070                                             mcaddr[j][2], mcaddr[j][3],
3071                                             mcaddr[j][4], mcaddr[j][5],
3072                                             rc);
3073                                 }
3074                                 goto mcfail;
3075                         }
3076                 }
3077
3078                 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0);
3079                 if (rc != 0)
3080                         if_printf(ifp, "failed to set mc address hash: %d", rc);
3081 mcfail:
3082                 if_maddr_runlock(ifp);
3083         }
3084
3085         return (rc);
3086 }
3087
3088 /*
3089  * {begin|end}_synchronized_op must be called from the same thread.
3090  */
3091 int
3092 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
3093     char *wmesg)
3094 {
3095         int rc, pri;
3096
3097 #ifdef WITNESS
3098         /* the caller thinks it's ok to sleep, but is it really? */
3099         if (flags & SLEEP_OK)
3100                 pause("t4slptst", 1);
3101 #endif
3102
3103         if (INTR_OK)
3104                 pri = PCATCH;
3105         else
3106                 pri = 0;
3107
3108         ADAPTER_LOCK(sc);
3109         for (;;) {
3110
3111                 if (pi && IS_DOOMED(pi)) {
3112                         rc = ENXIO;
3113                         goto done;
3114                 }
3115
3116                 if (!IS_BUSY(sc)) {
3117                         rc = 0;
3118                         break;
3119                 }
3120
3121                 if (!(flags & SLEEP_OK)) {
3122                         rc = EBUSY;
3123                         goto done;
3124                 }
3125
3126                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
3127                         rc = EINTR;
3128                         goto done;
3129                 }
3130         }
3131
3132         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
3133         SET_BUSY(sc);
3134 #ifdef INVARIANTS
3135         sc->last_op = wmesg;
3136         sc->last_op_thr = curthread;
3137         sc->last_op_flags = flags;
3138 #endif
3139
3140 done:
3141         if (!(flags & HOLD_LOCK) || rc)
3142                 ADAPTER_UNLOCK(sc);
3143
3144         return (rc);
3145 }
3146
3147 /*
3148  * {begin|end}_synchronized_op must be called from the same thread.
3149  */
3150 void
3151 end_synchronized_op(struct adapter *sc, int flags)
3152 {
3153
3154         if (flags & LOCK_HELD)
3155                 ADAPTER_LOCK_ASSERT_OWNED(sc);
3156         else
3157                 ADAPTER_LOCK(sc);
3158
3159         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
3160         CLR_BUSY(sc);
3161         wakeup(&sc->flags);
3162         ADAPTER_UNLOCK(sc);
3163 }
3164
3165 static int
3166 cxgbe_init_synchronized(struct port_info *pi)
3167 {
3168         struct adapter *sc = pi->adapter;
3169         struct ifnet *ifp = pi->ifp;
3170         int rc = 0, i;
3171         struct sge_txq *txq;
3172
3173         ASSERT_SYNCHRONIZED_OP(sc);
3174
3175         if (isset(&sc->open_device_map, pi->port_id)) {
3176                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
3177                     ("mismatch between open_device_map and if_drv_flags"));
3178                 return (0);     /* already running */
3179         }
3180
3181         if (!(sc->flags & FULL_INIT_DONE) &&
3182             ((rc = adapter_full_init(sc)) != 0))
3183                 return (rc);    /* error message displayed already */
3184
3185         if (!(pi->flags & PORT_INIT_DONE) &&
3186             ((rc = port_full_init(pi)) != 0))
3187                 return (rc); /* error message displayed already */
3188
3189         rc = update_mac_settings(ifp, XGMAC_ALL);
3190         if (rc)
3191                 goto done;      /* error message displayed already */
3192
3193         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
3194         if (rc != 0) {
3195                 if_printf(ifp, "enable_vi failed: %d\n", rc);
3196                 goto done;
3197         }
3198
3199         /*
3200          * Can't fail from this point onwards.  Review cxgbe_uninit_synchronized
3201          * if this changes.
3202          */
3203
3204         for_each_txq(pi, i, txq) {
3205                 TXQ_LOCK(txq);
3206                 txq->eq.flags |= EQ_ENABLED;
3207                 TXQ_UNLOCK(txq);
3208         }
3209
3210         /*
3211          * The first iq of the first port to come up is used for tracing.
3212          */
3213         if (sc->traceq < 0) {
3214                 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
3215                 t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
3216                     A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
3217                     V_QUEUENUMBER(sc->traceq));
3218                 pi->flags |= HAS_TRACEQ;
3219         }
3220
3221         /* all ok */
3222         setbit(&sc->open_device_map, pi->port_id);
3223         PORT_LOCK(pi);
3224         ifp->if_drv_flags |= IFF_DRV_RUNNING;
3225         PORT_UNLOCK(pi);
3226
3227         callout_reset(&pi->tick, hz, cxgbe_tick, pi);
3228 done:
3229         if (rc != 0)
3230                 cxgbe_uninit_synchronized(pi);
3231
3232         return (rc);
3233 }
3234
3235 /*
3236  * Idempotent.
3237  */
3238 static int
3239 cxgbe_uninit_synchronized(struct port_info *pi)
3240 {
3241         struct adapter *sc = pi->adapter;
3242         struct ifnet *ifp = pi->ifp;
3243         int rc, i;
3244         struct sge_txq *txq;
3245
3246         ASSERT_SYNCHRONIZED_OP(sc);
3247
3248         if (!(pi->flags & PORT_INIT_DONE)) {
3249                 KASSERT(!(ifp->if_drv_flags & IFF_DRV_RUNNING),
3250                     ("uninited port is running"));
3251                 return (0);
3252         }
3253
3254         /*
3255          * Disable the VI so that all its data in either direction is discarded
3256          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
3257          * tick) intact as the TP can deliver negative advice or data that it's
3258          * holding in its RAM (for an offloaded connection) even after the VI is
3259          * disabled.
3260          */
3261         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
3262         if (rc) {
3263                 if_printf(ifp, "disable_vi failed: %d\n", rc);
3264                 return (rc);
3265         }
3266
3267         for_each_txq(pi, i, txq) {
3268                 TXQ_LOCK(txq);
3269                 txq->eq.flags &= ~EQ_ENABLED;
3270                 TXQ_UNLOCK(txq);
3271         }
3272
3273         clrbit(&sc->open_device_map, pi->port_id);
3274         PORT_LOCK(pi);
3275         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3276         PORT_UNLOCK(pi);
3277
3278         pi->link_cfg.link_ok = 0;
3279         pi->link_cfg.speed = 0;
3280         pi->linkdnrc = -1;
3281         t4_os_link_changed(sc, pi->port_id, 0, -1);
3282
3283         return (0);
3284 }
3285
3286 /*
3287  * It is ok for this function to fail midway and return right away.  t4_detach
3288  * will walk the entire sc->irq list and clean up whatever is valid.
3289  */
3290 static int
3291 setup_intr_handlers(struct adapter *sc)
3292 {
3293         int rc, rid, p, q;
3294         char s[8];
3295         struct irq *irq;
3296         struct port_info *pi;
3297         struct sge_rxq *rxq;
3298 #ifdef TCP_OFFLOAD
3299         struct sge_ofld_rxq *ofld_rxq;
3300 #endif
3301 #ifdef DEV_NETMAP
3302         struct sge_nm_rxq *nm_rxq;
3303 #endif
3304
3305         /*
3306          * Setup interrupts.
3307          */
3308         irq = &sc->irq[0];
3309         rid = sc->intr_type == INTR_INTX ? 0 : 1;
3310         if (sc->intr_count == 1)
3311                 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
3312
3313         /* Multiple interrupts. */
3314         KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
3315             ("%s: too few intr.", __func__));
3316
3317         /* The first one is always error intr */
3318         rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
3319         if (rc != 0)
3320                 return (rc);
3321         irq++;
3322         rid++;
3323
3324         /* The second one is always the firmware event queue */
3325         rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
3326         if (rc != 0)
3327                 return (rc);
3328         irq++;
3329         rid++;
3330
3331         for_each_port(sc, p) {
3332                 pi = sc->port[p];
3333
3334                 if (pi->flags & INTR_RXQ) {
3335                         for_each_rxq(pi, q, rxq) {
3336                                 snprintf(s, sizeof(s), "%d.%d", p, q);
3337                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
3338                                     s);
3339                                 if (rc != 0)
3340                                         return (rc);
3341                                 irq++;
3342                                 rid++;
3343                         }
3344                 }
3345 #ifdef TCP_OFFLOAD
3346                 if (pi->flags & INTR_OFLD_RXQ) {
3347                         for_each_ofld_rxq(pi, q, ofld_rxq) {
3348                                 snprintf(s, sizeof(s), "%d,%d", p, q);
3349                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr,
3350                                     ofld_rxq, s);
3351                                 if (rc != 0)
3352                                         return (rc);
3353                                 irq++;
3354                                 rid++;
3355                         }
3356                 }
3357 #endif
3358 #ifdef DEV_NETMAP
3359                 if (pi->flags & INTR_NM_RXQ) {
3360                         for_each_nm_rxq(pi, q, nm_rxq) {
3361                                 snprintf(s, sizeof(s), "%d-%d", p, q);
3362                                 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr,
3363                                     nm_rxq, s);
3364                                 if (rc != 0)
3365                                         return (rc);
3366                                 irq++;
3367                                 rid++;
3368                         }
3369                 }
3370 #endif
3371         }
3372         MPASS(irq == &sc->irq[sc->intr_count]);
3373
3374         return (0);
3375 }
3376
3377 int
3378 adapter_full_init(struct adapter *sc)
3379 {
3380         int rc, i;
3381
3382         ASSERT_SYNCHRONIZED_OP(sc);
3383         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3384         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
3385             ("%s: FULL_INIT_DONE already", __func__));
3386
3387         /*
3388          * queues that belong to the adapter (not any particular port).
3389          */
3390         rc = t4_setup_adapter_queues(sc);
3391         if (rc != 0)
3392                 goto done;
3393
3394         for (i = 0; i < nitems(sc->tq); i++) {
3395                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
3396                     taskqueue_thread_enqueue, &sc->tq[i]);
3397                 if (sc->tq[i] == NULL) {
3398                         device_printf(sc->dev,
3399                             "failed to allocate task queue %d\n", i);
3400                         rc = ENOMEM;
3401                         goto done;
3402                 }
3403                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
3404                     device_get_nameunit(sc->dev), i);
3405         }
3406
3407         t4_intr_enable(sc);
3408         sc->flags |= FULL_INIT_DONE;
3409 done:
3410         if (rc != 0)
3411                 adapter_full_uninit(sc);
3412
3413         return (rc);
3414 }
3415
3416 int
3417 adapter_full_uninit(struct adapter *sc)
3418 {
3419         int i;
3420
3421         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3422
3423         t4_teardown_adapter_queues(sc);
3424
3425         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
3426                 taskqueue_free(sc->tq[i]);
3427                 sc->tq[i] = NULL;
3428         }
3429
3430         sc->flags &= ~FULL_INIT_DONE;
3431
3432         return (0);
3433 }
3434
3435 int
3436 port_full_init(struct port_info *pi)
3437 {
3438         struct adapter *sc = pi->adapter;
3439         struct ifnet *ifp = pi->ifp;
3440         uint16_t *rss;
3441         struct sge_rxq *rxq;
3442         int rc, i, j;
3443
3444         ASSERT_SYNCHRONIZED_OP(sc);
3445         KASSERT((pi->flags & PORT_INIT_DONE) == 0,
3446             ("%s: PORT_INIT_DONE already", __func__));
3447
3448         sysctl_ctx_init(&pi->ctx);
3449         pi->flags |= PORT_SYSCTL_CTX;
3450
3451         /*
3452          * Allocate tx/rx/fl queues for this port.
3453          */
3454         rc = t4_setup_port_queues(pi);
3455         if (rc != 0)
3456                 goto done;      /* error message displayed already */
3457
3458         /*
3459          * Setup RSS for this port.  Save a copy of the RSS table for later use.
3460          */
3461         rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
3462         for (i = 0; i < pi->rss_size;) {
3463                 for_each_rxq(pi, j, rxq) {
3464                         rss[i++] = rxq->iq.abs_id;
3465                         if (i == pi->rss_size)
3466                                 break;
3467                 }
3468         }
3469
3470         rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
3471             pi->rss_size);
3472         if (rc != 0) {
3473                 if_printf(ifp, "rss_config failed: %d\n", rc);
3474                 goto done;
3475         }
3476
3477         pi->rss = rss;
3478         pi->flags |= PORT_INIT_DONE;
3479 done:
3480         if (rc != 0)
3481                 port_full_uninit(pi);
3482
3483         return (rc);
3484 }
3485
3486 /*
3487  * Idempotent.
3488  */
3489 int
3490 port_full_uninit(struct port_info *pi)
3491 {
3492         struct adapter *sc = pi->adapter;
3493         int i;
3494         struct sge_rxq *rxq;
3495         struct sge_txq *txq;
3496 #ifdef TCP_OFFLOAD
3497         struct sge_ofld_rxq *ofld_rxq;
3498         struct sge_wrq *ofld_txq;
3499 #endif
3500
3501         if (pi->flags & PORT_INIT_DONE) {
3502
3503                 /* Need to quiesce queues.  */
3504
3505                 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
3506
3507                 for_each_txq(pi, i, txq) {
3508                         quiesce_txq(sc, txq);
3509                 }
3510
3511 #ifdef TCP_OFFLOAD
3512                 for_each_ofld_txq(pi, i, ofld_txq) {
3513                         quiesce_wrq(sc, ofld_txq);
3514                 }
3515 #endif
3516
3517                 for_each_rxq(pi, i, rxq) {
3518                         quiesce_iq(sc, &rxq->iq);
3519                         quiesce_fl(sc, &rxq->fl);
3520                 }
3521
3522 #ifdef TCP_OFFLOAD
3523                 for_each_ofld_rxq(pi, i, ofld_rxq) {
3524                         quiesce_iq(sc, &ofld_rxq->iq);
3525                         quiesce_fl(sc, &ofld_rxq->fl);
3526                 }
3527 #endif
3528                 free(pi->rss, M_CXGBE);
3529         }
3530
3531         t4_teardown_port_queues(pi);
3532         pi->flags &= ~PORT_INIT_DONE;
3533
3534         return (0);
3535 }
3536
3537 static void
3538 quiesce_txq(struct adapter *sc, struct sge_txq *txq)
3539 {
3540         struct sge_eq *eq = &txq->eq;
3541         struct sge_qstat *spg = (void *)&eq->desc[eq->sidx];
3542
3543         (void) sc;      /* unused */
3544
3545 #ifdef INVARIANTS
3546         TXQ_LOCK(txq);
3547         MPASS((eq->flags & EQ_ENABLED) == 0);
3548         TXQ_UNLOCK(txq);
3549 #endif
3550
3551         /* Wait for the mp_ring to empty. */
3552         while (!mp_ring_is_idle(txq->r)) {
3553                 mp_ring_check_drainage(txq->r, 0);
3554                 pause("rquiesce", 1);
3555         }
3556
3557         /* Then wait for the hardware to finish. */
3558         while (spg->cidx != htobe16(eq->pidx))
3559                 pause("equiesce", 1);
3560
3561         /* Finally, wait for the driver to reclaim all descriptors. */
3562         while (eq->cidx != eq->pidx)
3563                 pause("dquiesce", 1);
3564 }
3565
3566 static void
3567 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq)
3568 {
3569
3570         /* XXXTX */
3571 }
3572
3573 static void
3574 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
3575 {
3576         (void) sc;      /* unused */
3577
3578         /* Synchronize with the interrupt handler */
3579         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
3580                 pause("iqfree", 1);
3581 }
3582
3583 static void
3584 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
3585 {
3586         mtx_lock(&sc->sfl_lock);
3587         FL_LOCK(fl);
3588         fl->flags |= FL_DOOMED;
3589         FL_UNLOCK(fl);
3590         mtx_unlock(&sc->sfl_lock);
3591
3592         callout_drain(&sc->sfl_callout);
3593         KASSERT((fl->flags & FL_STARVING) == 0,
3594             ("%s: still starving", __func__));
3595 }
3596
3597 static int
3598 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
3599     driver_intr_t *handler, void *arg, char *name)
3600 {
3601         int rc;
3602
3603         irq->rid = rid;
3604         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
3605             RF_SHAREABLE | RF_ACTIVE);
3606         if (irq->res == NULL) {
3607                 device_printf(sc->dev,
3608                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
3609                 return (ENOMEM);
3610         }
3611
3612         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
3613             NULL, handler, arg, &irq->tag);
3614         if (rc != 0) {
3615                 device_printf(sc->dev,
3616                     "failed to setup interrupt for rid %d, name %s: %d\n",
3617                     rid, name, rc);
3618         } else if (name)
3619                 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
3620
3621         return (rc);
3622 }
3623
3624 static int
3625 t4_free_irq(struct adapter *sc, struct irq *irq)
3626 {
3627         if (irq->tag)
3628                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
3629         if (irq->res)
3630                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
3631
3632         bzero(irq, sizeof(*irq));
3633
3634         return (0);
3635 }
3636
3637 static void
3638 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
3639     unsigned int end)
3640 {
3641         uint32_t *p = (uint32_t *)(buf + start);
3642
3643         for ( ; start <= end; start += sizeof(uint32_t))
3644                 *p++ = t4_read_reg(sc, start);
3645 }
3646
3647 static void
3648 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
3649 {
3650         int i, n;
3651         const unsigned int *reg_ranges;
3652         static const unsigned int t4_reg_ranges[] = {
3653                 0x1008, 0x1108,
3654                 0x1180, 0x11b4,
3655                 0x11fc, 0x123c,
3656                 0x1300, 0x173c,
3657                 0x1800, 0x18fc,
3658                 0x3000, 0x30d8,
3659                 0x30e0, 0x5924,
3660                 0x5960, 0x59d4,
3661                 0x5a00, 0x5af8,
3662                 0x6000, 0x6098,
3663                 0x6100, 0x6150,
3664                 0x6200, 0x6208,
3665                 0x6240, 0x6248,
3666                 0x6280, 0x6338,
3667                 0x6370, 0x638c,
3668                 0x6400, 0x643c,
3669                 0x6500, 0x6524,
3670                 0x6a00, 0x6a38,
3671                 0x6a60, 0x6a78,
3672                 0x6b00, 0x6b84,
3673                 0x6bf0, 0x6c84,
3674                 0x6cf0, 0x6d84,
3675                 0x6df0, 0x6e84,
3676                 0x6ef0, 0x6f84,
3677                 0x6ff0, 0x7084,
3678                 0x70f0, 0x7184,
3679                 0x71f0, 0x7284,
3680                 0x72f0, 0x7384,
3681                 0x73f0, 0x7450,
3682                 0x7500, 0x7530,
3683                 0x7600, 0x761c,
3684                 0x7680, 0x76cc,
3685                 0x7700, 0x7798,
3686                 0x77c0, 0x77fc,
3687                 0x7900, 0x79fc,
3688                 0x7b00, 0x7c38,
3689                 0x7d00, 0x7efc,
3690                 0x8dc0, 0x8e1c,
3691                 0x8e30, 0x8e78,
3692                 0x8ea0, 0x8f6c,
3693                 0x8fc0, 0x9074,
3694                 0x90fc, 0x90fc,
3695                 0x9400, 0x9458,
3696                 0x9600, 0x96bc,
3697                 0x9800, 0x9808,
3698                 0x9820, 0x983c,
3699                 0x9850, 0x9864,
3700                 0x9c00, 0x9c6c,
3701                 0x9c80, 0x9cec,
3702                 0x9d00, 0x9d6c,
3703                 0x9d80, 0x9dec,
3704                 0x9e00, 0x9e6c,
3705                 0x9e80, 0x9eec,
3706                 0x9f00, 0x9f6c,
3707                 0x9f80, 0x9fec,
3708                 0xd004, 0xd03c,
3709                 0xdfc0, 0xdfe0,
3710                 0xe000, 0xea7c,
3711                 0xf000, 0x11110,
3712                 0x11118, 0x11190,
3713                 0x19040, 0x1906c,
3714                 0x19078, 0x19080,
3715                 0x1908c, 0x19124,
3716                 0x19150, 0x191b0,
3717                 0x191d0, 0x191e8,
3718                 0x19238, 0x1924c,
3719                 0x193f8, 0x19474,
3720                 0x19490, 0x194f8,
3721                 0x19800, 0x19f30,
3722                 0x1a000, 0x1a06c,
3723                 0x1a0b0, 0x1a120,
3724                 0x1a128, 0x1a138,
3725                 0x1a190, 0x1a1c4,
3726                 0x1a1fc, 0x1a1fc,
3727                 0x1e040, 0x1e04c,
3728                 0x1e284, 0x1e28c,
3729                 0x1e2c0, 0x1e2c0,
3730                 0x1e2e0, 0x1e2e0,
3731                 0x1e300, 0x1e384,
3732                 0x1e3c0, 0x1e3c8,
3733                 0x1e440, 0x1e44c,
3734                 0x1e684, 0x1e68c,
3735                 0x1e6c0, 0x1e6c0,
3736                 0x1e6e0, 0x1e6e0,
3737                 0x1e700, 0x1e784,
3738                 0x1e7c0, 0x1e7c8,
3739                 0x1e840, 0x1e84c,
3740                 0x1ea84, 0x1ea8c,
3741                 0x1eac0, 0x1eac0,
3742                 0x1eae0, 0x1eae0,
3743                 0x1eb00, 0x1eb84,
3744                 0x1ebc0, 0x1ebc8,
3745                 0x1ec40, 0x1ec4c,
3746                 0x1ee84, 0x1ee8c,
3747                 0x1eec0, 0x1eec0,
3748                 0x1eee0, 0x1eee0,
3749                 0x1ef00, 0x1ef84,
3750                 0x1efc0, 0x1efc8,
3751                 0x1f040, 0x1f04c,
3752                 0x1f284, 0x1f28c,
3753                 0x1f2c0, 0x1f2c0,
3754                 0x1f2e0, 0x1f2e0,
3755                 0x1f300, 0x1f384,
3756                 0x1f3c0, 0x1f3c8,
3757                 0x1f440, 0x1f44c,
3758                 0x1f684, 0x1f68c,
3759                 0x1f6c0, 0x1f6c0,
3760                 0x1f6e0, 0x1f6e0,
3761                 0x1f700, 0x1f784,
3762                 0x1f7c0, 0x1f7c8,
3763                 0x1f840, 0x1f84c,
3764                 0x1fa84, 0x1fa8c,
3765                 0x1fac0, 0x1fac0,
3766                 0x1fae0, 0x1fae0,
3767                 0x1fb00, 0x1fb84,
3768                 0x1fbc0, 0x1fbc8,
3769                 0x1fc40, 0x1fc4c,
3770                 0x1fe84, 0x1fe8c,
3771                 0x1fec0, 0x1fec0,
3772                 0x1fee0, 0x1fee0,
3773                 0x1ff00, 0x1ff84,
3774                 0x1ffc0, 0x1ffc8,
3775                 0x20000, 0x2002c,
3776                 0x20100, 0x2013c,
3777                 0x20190, 0x201c8,
3778                 0x20200, 0x20318,
3779                 0x20400, 0x20528,
3780                 0x20540, 0x20614,
3781                 0x21000, 0x21040,
3782                 0x2104c, 0x21060,
3783                 0x210c0, 0x210ec,
3784                 0x21200, 0x21268,
3785                 0x21270, 0x21284,
3786                 0x212fc, 0x21388,
3787                 0x21400, 0x21404,
3788                 0x21500, 0x21518,
3789                 0x2152c, 0x2153c,
3790                 0x21550, 0x21554,
3791                 0x21600, 0x21600,
3792                 0x21608, 0x21628,
3793                 0x21630, 0x2163c,
3794                 0x21700, 0x2171c,
3795                 0x21780, 0x2178c,
3796                 0x21800, 0x21c38,
3797                 0x21c80, 0x21d7c,
3798                 0x21e00, 0x21e04,
3799                 0x22000, 0x2202c,
3800                 0x22100, 0x2213c,
3801                 0x22190, 0x221c8,
3802                 0x22200, 0x22318,
3803                 0x22400, 0x22528,
3804                 0x22540, 0x22614,
3805                 0x23000, 0x23040,
3806                 0x2304c, 0x23060,
3807                 0x230c0, 0x230ec,
3808                 0x23200, 0x23268,
3809                 0x23270, 0x23284,
3810                 0x232fc, 0x23388,
3811                 0x23400, 0x23404,
3812                 0x23500, 0x23518,
3813                 0x2352c, 0x2353c,
3814                 0x23550, 0x23554,
3815                 0x23600, 0x23600,
3816                 0x23608, 0x23628,
3817                 0x23630, 0x2363c,
3818                 0x23700, 0x2371c,
3819                 0x23780, 0x2378c,
3820                 0x23800, 0x23c38,
3821                 0x23c80, 0x23d7c,
3822                 0x23e00, 0x23e04,
3823                 0x24000, 0x2402c,
3824                 0x24100, 0x2413c,
3825                 0x24190, 0x241c8,
3826                 0x24200, 0x24318,
3827                 0x24400, 0x24528,
3828                 0x24540, 0x24614,
3829                 0x25000, 0x25040,
3830                 0x2504c, 0x25060,
3831                 0x250c0, 0x250ec,
3832                 0x25200, 0x25268,
3833                 0x25270, 0x25284,
3834                 0x252fc, 0x25388,
3835                 0x25400, 0x25404,
3836                 0x25500, 0x25518,
3837                 0x2552c, 0x2553c,
3838                 0x25550, 0x25554,
3839                 0x25600, 0x25600,
3840                 0x25608, 0x25628,
3841                 0x25630, 0x2563c,
3842                 0x25700, 0x2571c,
3843                 0x25780, 0x2578c,
3844                 0x25800, 0x25c38,
3845                 0x25c80, 0x25d7c,
3846                 0x25e00, 0x25e04,
3847                 0x26000, 0x2602c,
3848                 0x26100, 0x2613c,
3849                 0x26190, 0x261c8,
3850                 0x26200, 0x26318,
3851                 0x26400, 0x26528,
3852                 0x26540, 0x26614,
3853                 0x27000, 0x27040,
3854                 0x2704c, 0x27060,
3855                 0x270c0, 0x270ec,
3856                 0x27200, 0x27268,
3857                 0x27270, 0x27284,
3858                 0x272fc, 0x27388,
3859                 0x27400, 0x27404,
3860                 0x27500, 0x27518,
3861                 0x2752c, 0x2753c,
3862                 0x27550, 0x27554,
3863                 0x27600, 0x27600,
3864                 0x27608, 0x27628,
3865                 0x27630, 0x2763c,
3866                 0x27700, 0x2771c,
3867                 0x27780, 0x2778c,
3868                 0x27800, 0x27c38,
3869                 0x27c80, 0x27d7c,
3870                 0x27e00, 0x27e04
3871         };
3872         static const unsigned int t5_reg_ranges[] = {
3873                 0x1008, 0x1148,
3874                 0x1180, 0x11b4,
3875                 0x11fc, 0x123c,
3876                 0x1280, 0x173c,
3877                 0x1800, 0x18fc,
3878                 0x3000, 0x3028,
3879                 0x3060, 0x30d8,
3880                 0x30e0, 0x30fc,
3881                 0x3140, 0x357c,
3882                 0x35a8, 0x35cc,
3883                 0x35ec, 0x35ec,
3884                 0x3600, 0x5624,
3885                 0x56cc, 0x575c,
3886                 0x580c, 0x5814,
3887                 0x5890, 0x58bc,
3888                 0x5940, 0x59dc,
3889                 0x59fc, 0x5a18,
3890                 0x5a60, 0x5a9c,
3891                 0x5b94, 0x5bfc,
3892                 0x6000, 0x6040,
3893                 0x6058, 0x614c,
3894                 0x7700, 0x7798,
3895                 0x77c0, 0x78fc,
3896                 0x7b00, 0x7c54,
3897                 0x7d00, 0x7efc,
3898                 0x8dc0, 0x8de0,
3899                 0x8df8, 0x8e84,
3900                 0x8ea0, 0x8f84,
3901                 0x8fc0, 0x90f8,
3902                 0x9400, 0x9470,
3903                 0x9600, 0x96f4,
3904                 0x9800, 0x9808,
3905                 0x9820, 0x983c,
3906                 0x9850, 0x9864,
3907                 0x9c00, 0x9c6c,
3908                 0x9c80, 0x9cec,
3909                 0x9d00, 0x9d6c,
3910                 0x9d80, 0x9dec,
3911                 0x9e00, 0x9e6c,
3912                 0x9e80, 0x9eec,
3913                 0x9f00, 0x9f6c,
3914                 0x9f80, 0xa020,
3915                 0xd004, 0xd03c,
3916                 0xdfc0, 0xdfe0,
3917                 0xe000, 0x11088,
3918                 0x1109c, 0x11110,
3919                 0x11118, 0x1117c,
3920                 0x11190, 0x11204,
3921                 0x19040, 0x1906c,
3922                 0x19078, 0x19080,
3923                 0x1908c, 0x19124,
3924                 0x19150, 0x191b0,
3925                 0x191d0, 0x191e8,
3926                 0x19238, 0x19290,
3927                 0x193f8, 0x19474,
3928                 0x19490, 0x194cc,
3929                 0x194f0, 0x194f8,
3930                 0x19c00, 0x19c60,
3931                 0x19c94, 0x19e10,
3932                 0x19e50, 0x19f34,
3933                 0x19f40, 0x19f50,
3934                 0x19f90, 0x19fe4,
3935                 0x1a000, 0x1a06c,
3936                 0x1a0b0, 0x1a120,
3937                 0x1a128, 0x1a138,
3938                 0x1a190, 0x1a1c4,
3939                 0x1a1fc, 0x1a1fc,
3940                 0x1e008, 0x1e00c,
3941                 0x1e040, 0x1e04c,
3942                 0x1e284, 0x1e290,
3943                 0x1e2c0, 0x1e2c0,
3944                 0x1e2e0, 0x1e2e0,
3945                 0x1e300, 0x1e384,
3946                 0x1e3c0, 0x1e3c8,
3947                 0x1e408, 0x1e40c,
3948                 0x1e440, 0x1e44c,
3949                 0x1e684, 0x1e690,
3950                 0x1e6c0, 0x1e6c0,
3951                 0x1e6e0, 0x1e6e0,
3952                 0x1e700, 0x1e784,
3953                 0x1e7c0, 0x1e7c8,
3954                 0x1e808, 0x1e80c,
3955                 0x1e840, 0x1e84c,
3956                 0x1ea84, 0x1ea90,
3957                 0x1eac0, 0x1eac0,
3958                 0x1eae0, 0x1eae0,
3959                 0x1eb00, 0x1eb84,
3960                 0x1ebc0, 0x1ebc8,
3961                 0x1ec08, 0x1ec0c,
3962                 0x1ec40, 0x1ec4c,
3963                 0x1ee84, 0x1ee90,
3964                 0x1eec0, 0x1eec0,
3965                 0x1eee0, 0x1eee0,
3966                 0x1ef00, 0x1ef84,
3967                 0x1efc0, 0x1efc8,
3968                 0x1f008, 0x1f00c,
3969                 0x1f040, 0x1f04c,
3970                 0x1f284, 0x1f290,
3971                 0x1f2c0, 0x1f2c0,
3972                 0x1f2e0, 0x1f2e0,
3973                 0x1f300, 0x1f384,
3974                 0x1f3c0, 0x1f3c8,
3975                 0x1f408, 0x1f40c,
3976                 0x1f440, 0x1f44c,
3977                 0x1f684, 0x1f690,
3978                 0x1f6c0, 0x1f6c0,
3979                 0x1f6e0, 0x1f6e0,
3980                 0x1f700, 0x1f784,
3981                 0x1f7c0, 0x1f7c8,
3982                 0x1f808, 0x1f80c,
3983                 0x1f840, 0x1f84c,
3984                 0x1fa84, 0x1fa90,
3985                 0x1fac0, 0x1fac0,
3986                 0x1fae0, 0x1fae0,
3987                 0x1fb00, 0x1fb84,
3988                 0x1fbc0, 0x1fbc8,
3989                 0x1fc08, 0x1fc0c,
3990                 0x1fc40, 0x1fc4c,
3991                 0x1fe84, 0x1fe90,
3992                 0x1fec0, 0x1fec0,
3993                 0x1fee0, 0x1fee0,
3994                 0x1ff00, 0x1ff84,
3995                 0x1ffc0, 0x1ffc8,
3996                 0x30000, 0x30030,
3997                 0x30100, 0x30144,
3998                 0x30190, 0x301d0,
3999                 0x30200, 0x30318,
4000                 0x30400, 0x3052c,
4001                 0x30540, 0x3061c,
4002                 0x30800, 0x30834,
4003                 0x308c0, 0x30908,
4004                 0x30910, 0x309ac,
4005                 0x30a00, 0x30a2c,
4006                 0x30a44, 0x30a50,
4007                 0x30a74, 0x30c24,
4008                 0x30d00, 0x30d00,
4009                 0x30d08, 0x30d14,
4010                 0x30d1c, 0x30d20,
4011                 0x30d3c, 0x30d50,
4012                 0x31200, 0x3120c,
4013                 0x31220, 0x31220,
4014                 0x31240, 0x31240,
4015                 0x31600, 0x3160c,
4016                 0x31a00, 0x31a1c,
4017                 0x31e00, 0x31e20,
4018                 0x31e38, 0x31e3c,
4019                 0x31e80, 0x31e80,
4020                 0x31e88, 0x31ea8,
4021                 0x31eb0, 0x31eb4,
4022                 0x31ec8, 0x31ed4,
4023                 0x31fb8, 0x32004,
4024                 0x32200, 0x32200,
4025                 0x32208, 0x32240,
4026                 0x32248, 0x32280,
4027                 0x32288, 0x322c0,
4028                 0x322c8, 0x322fc,
4029                 0x32600, 0x32630,
4030                 0x32a00, 0x32abc,
4031                 0x32b00, 0x32b70,
4032                 0x33000, 0x33048,
4033                 0x33060, 0x3309c,
4034                 0x330f0, 0x33148,
4035                 0x33160, 0x3319c,
4036                 0x331f0, 0x332e4,
4037                 0x332f8, 0x333e4,
4038                 0x333f8, 0x33448,
4039                 0x33460, 0x3349c,
4040                 0x334f0, 0x33548,
4041                 0x33560, 0x3359c,
4042                 0x335f0, 0x336e4,
4043                 0x336f8, 0x337e4,
4044                 0x337f8, 0x337fc,
4045                 0x33814, 0x33814,
4046                 0x3382c, 0x3382c,
4047                 0x33880, 0x3388c,
4048                 0x338e8, 0x338ec,
4049                 0x33900, 0x33948,
4050                 0x33960, 0x3399c,
4051                 0x339f0, 0x33ae4,
4052                 0x33af8, 0x33b10,
4053                 0x33b28, 0x33b28,
4054                 0x33b3c, 0x33b50,
4055                 0x33bf0, 0x33c10,
4056                 0x33c28, 0x33c28,
4057                 0x33c3c, 0x33c50,
4058                 0x33cf0, 0x33cfc,
4059                 0x34000, 0x34030,
4060                 0x34100, 0x34144,
4061                 0x34190, 0x341d0,
4062                 0x34200, 0x34318,
4063                 0x34400, 0x3452c,
4064                 0x34540, 0x3461c,
4065                 0x34800, 0x34834,
4066                 0x348c0, 0x34908,
4067                 0x34910, 0x349ac,
4068                 0x34a00, 0x34a2c,
4069                 0x34a44, 0x34a50,
4070                 0x34a74, 0x34c24,
4071                 0x34d00, 0x34d00,
4072                 0x34d08, 0x34d14,
4073                 0x34d1c, 0x34d20,
4074                 0x34d3c, 0x34d50,
4075                 0x35200, 0x3520c,
4076                 0x35220, 0x35220,
4077                 0x35240, 0x35240,
4078                 0x35600, 0x3560c,
4079                 0x35a00, 0x35a1c,
4080                 0x35e00, 0x35e20,
4081                 0x35e38, 0x35e3c,
4082                 0x35e80, 0x35e80,
4083                 0x35e88, 0x35ea8,
4084                 0x35eb0, 0x35eb4,
4085                 0x35ec8, 0x35ed4,
4086                 0x35fb8, 0x36004,
4087                 0x36200, 0x36200,
4088                 0x36208, 0x36240,
4089                 0x36248, 0x36280,
4090                 0x36288, 0x362c0,
4091                 0x362c8, 0x362fc,
4092                 0x36600, 0x36630,
4093                 0x36a00, 0x36abc,
4094                 0x36b00, 0x36b70,
4095                 0x37000, 0x37048,
4096                 0x37060, 0x3709c,
4097                 0x370f0, 0x37148,
4098                 0x37160, 0x3719c,
4099                 0x371f0, 0x372e4,
4100                 0x372f8, 0x373e4,
4101                 0x373f8, 0x37448,
4102                 0x37460, 0x3749c,
4103                 0x374f0, 0x37548,
4104                 0x37560, 0x3759c,
4105                 0x375f0, 0x376e4,
4106                 0x376f8, 0x377e4,
4107                 0x377f8, 0x377fc,
4108                 0x37814, 0x37814,
4109                 0x3782c, 0x3782c,
4110                 0x37880, 0x3788c,
4111                 0x378e8, 0x378ec,
4112                 0x37900, 0x37948,
4113                 0x37960, 0x3799c,
4114                 0x379f0, 0x37ae4,
4115                 0x37af8, 0x37b10,
4116                 0x37b28, 0x37b28,
4117                 0x37b3c, 0x37b50,
4118                 0x37bf0, 0x37c10,
4119                 0x37c28, 0x37c28,
4120                 0x37c3c, 0x37c50,
4121                 0x37cf0, 0x37cfc,
4122                 0x38000, 0x38030,
4123                 0x38100, 0x38144,
4124                 0x38190, 0x381d0,
4125                 0x38200, 0x38318,
4126                 0x38400, 0x3852c,
4127                 0x38540, 0x3861c,
4128                 0x38800, 0x38834,
4129                 0x388c0, 0x38908,
4130                 0x38910, 0x389ac,
4131                 0x38a00, 0x38a2c,
4132                 0x38a44, 0x38a50,
4133                 0x38a74, 0x38c24,
4134                 0x38d00, 0x38d00,
4135                 0x38d08, 0x38d14,
4136                 0x38d1c, 0x38d20,
4137                 0x38d3c, 0x38d50,
4138                 0x39200, 0x3920c,
4139                 0x39220, 0x39220,
4140                 0x39240, 0x39240,
4141                 0x39600, 0x3960c,
4142                 0x39a00, 0x39a1c,
4143                 0x39e00, 0x39e20,
4144                 0x39e38, 0x39e3c,
4145                 0x39e80, 0x39e80,
4146                 0x39e88, 0x39ea8,
4147                 0x39eb0, 0x39eb4,
4148                 0x39ec8, 0x39ed4,
4149                 0x39fb8, 0x3a004,
4150                 0x3a200, 0x3a200,
4151                 0x3a208, 0x3a240,
4152                 0x3a248, 0x3a280,
4153                 0x3a288, 0x3a2c0,
4154                 0x3a2c8, 0x3a2fc,
4155                 0x3a600, 0x3a630,
4156                 0x3aa00, 0x3aabc,
4157                 0x3ab00, 0x3ab70,
4158                 0x3b000, 0x3b048,
4159                 0x3b060, 0x3b09c,
4160                 0x3b0f0, 0x3b148,
4161                 0x3b160, 0x3b19c,
4162                 0x3b1f0, 0x3b2e4,
4163                 0x3b2f8, 0x3b3e4,
4164                 0x3b3f8, 0x3b448,
4165                 0x3b460, 0x3b49c,
4166                 0x3b4f0, 0x3b548,
4167                 0x3b560, 0x3b59c,
4168                 0x3b5f0, 0x3b6e4,
4169                 0x3b6f8, 0x3b7e4,
4170                 0x3b7f8, 0x3b7fc,
4171                 0x3b814, 0x3b814,
4172                 0x3b82c, 0x3b82c,
4173                 0x3b880, 0x3b88c,
4174                 0x3b8e8, 0x3b8ec,
4175                 0x3b900, 0x3b948,
4176                 0x3b960, 0x3b99c,
4177                 0x3b9f0, 0x3bae4,
4178                 0x3baf8, 0x3bb10,
4179                 0x3bb28, 0x3bb28,
4180                 0x3bb3c, 0x3bb50,
4181                 0x3bbf0, 0x3bc10,
4182                 0x3bc28, 0x3bc28,
4183                 0x3bc3c, 0x3bc50,
4184                 0x3bcf0, 0x3bcfc,
4185                 0x3c000, 0x3c030,
4186                 0x3c100, 0x3c144,
4187                 0x3c190, 0x3c1d0,
4188                 0x3c200, 0x3c318,
4189                 0x3c400, 0x3c52c,
4190                 0x3c540, 0x3c61c,
4191                 0x3c800, 0x3c834,
4192                 0x3c8c0, 0x3c908,
4193                 0x3c910, 0x3c9ac,
4194                 0x3ca00, 0x3ca2c,
4195                 0x3ca44, 0x3ca50,
4196                 0x3ca74, 0x3cc24,
4197                 0x3cd00, 0x3cd00,
4198                 0x3cd08, 0x3cd14,
4199                 0x3cd1c, 0x3cd20,
4200                 0x3cd3c, 0x3cd50,
4201                 0x3d200, 0x3d20c,
4202                 0x3d220, 0x3d220,
4203                 0x3d240, 0x3d240,
4204                 0x3d600, 0x3d60c,
4205                 0x3da00, 0x3da1c,
4206                 0x3de00, 0x3de20,
4207                 0x3de38, 0x3de3c,
4208                 0x3de80, 0x3de80,
4209                 0x3de88, 0x3dea8,
4210                 0x3deb0, 0x3deb4,
4211                 0x3dec8, 0x3ded4,
4212                 0x3dfb8, 0x3e004,
4213                 0x3e200, 0x3e200,
4214                 0x3e208, 0x3e240,
4215                 0x3e248, 0x3e280,
4216                 0x3e288, 0x3e2c0,
4217                 0x3e2c8, 0x3e2fc,
4218                 0x3e600, 0x3e630,
4219                 0x3ea00, 0x3eabc,
4220                 0x3eb00, 0x3eb70,
4221                 0x3f000, 0x3f048,
4222                 0x3f060, 0x3f09c,
4223                 0x3f0f0, 0x3f148,
4224                 0x3f160, 0x3f19c,
4225                 0x3f1f0, 0x3f2e4,
4226                 0x3f2f8, 0x3f3e4,
4227                 0x3f3f8, 0x3f448,
4228                 0x3f460, 0x3f49c,
4229                 0x3f4f0, 0x3f548,
4230                 0x3f560, 0x3f59c,
4231                 0x3f5f0, 0x3f6e4,
4232                 0x3f6f8, 0x3f7e4,
4233                 0x3f7f8, 0x3f7fc,
4234                 0x3f814, 0x3f814,
4235                 0x3f82c, 0x3f82c,
4236                 0x3f880, 0x3f88c,
4237                 0x3f8e8, 0x3f8ec,
4238                 0x3f900, 0x3f948,
4239                 0x3f960, 0x3f99c,
4240                 0x3f9f0, 0x3fae4,
4241                 0x3faf8, 0x3fb10,
4242                 0x3fb28, 0x3fb28,
4243                 0x3fb3c, 0x3fb50,
4244                 0x3fbf0, 0x3fc10,
4245                 0x3fc28, 0x3fc28,
4246                 0x3fc3c, 0x3fc50,
4247                 0x3fcf0, 0x3fcfc,
4248                 0x40000, 0x4000c,
4249                 0x40040, 0x40068,
4250                 0x4007c, 0x40144,
4251                 0x40180, 0x4018c,
4252                 0x40200, 0x40298,
4253                 0x402ac, 0x4033c,
4254                 0x403f8, 0x403fc,
4255                 0x41304, 0x413c4,
4256                 0x41400, 0x4141c,
4257                 0x41480, 0x414d0,
4258                 0x44000, 0x44078,
4259                 0x440c0, 0x44278,
4260                 0x442c0, 0x44478,
4261                 0x444c0, 0x44678,
4262                 0x446c0, 0x44878,
4263                 0x448c0, 0x449fc,
4264                 0x45000, 0x45068,
4265                 0x45080, 0x45084,
4266                 0x450a0, 0x450b0,
4267                 0x45200, 0x45268,
4268                 0x45280, 0x45284,
4269                 0x452a0, 0x452b0,
4270                 0x460c0, 0x460e4,
4271                 0x47000, 0x4708c,
4272                 0x47200, 0x47250,
4273                 0x47400, 0x47420,
4274                 0x47600, 0x47618,
4275                 0x47800, 0x47814,
4276                 0x48000, 0x4800c,
4277                 0x48040, 0x48068,
4278                 0x4807c, 0x48144,
4279                 0x48180, 0x4818c,
4280                 0x48200, 0x48298,
4281                 0x482ac, 0x4833c,
4282                 0x483f8, 0x483fc,
4283                 0x49304, 0x493c4,
4284                 0x49400, 0x4941c,
4285                 0x49480, 0x494d0,
4286                 0x4c000, 0x4c078,
4287                 0x4c0c0, 0x4c278,
4288                 0x4c2c0, 0x4c478,
4289                 0x4c4c0, 0x4c678,
4290                 0x4c6c0, 0x4c878,
4291                 0x4c8c0, 0x4c9fc,
4292                 0x4d000, 0x4d068,
4293                 0x4d080, 0x4d084,
4294                 0x4d0a0, 0x4d0b0,
4295                 0x4d200, 0x4d268,
4296                 0x4d280, 0x4d284,
4297                 0x4d2a0, 0x4d2b0,
4298                 0x4e0c0, 0x4e0e4,
4299                 0x4f000, 0x4f08c,
4300                 0x4f200, 0x4f250,
4301                 0x4f400, 0x4f420,
4302                 0x4f600, 0x4f618,
4303                 0x4f800, 0x4f814,
4304                 0x50000, 0x500cc,
4305                 0x50400, 0x50400,
4306                 0x50800, 0x508cc,
4307                 0x50c00, 0x50c00,
4308                 0x51000, 0x5101c,
4309                 0x51300, 0x51308,
4310         };
4311
4312         if (is_t4(sc)) {
4313                 reg_ranges = &t4_reg_ranges[0];
4314                 n = nitems(t4_reg_ranges);
4315         } else {
4316                 reg_ranges = &t5_reg_ranges[0];
4317                 n = nitems(t5_reg_ranges);
4318         }
4319
4320         regs->version = chip_id(sc) | chip_rev(sc) << 10;
4321         for (i = 0; i < n; i += 2)
4322                 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
4323 }
4324
4325 static void
4326 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi)
4327 {
4328         struct ifnet *ifp = pi->ifp;
4329         struct sge_txq *txq;
4330         int i, drops;
4331         struct port_stats *s = &pi->stats;
4332         struct timeval tv;
4333         const struct timeval interval = {0, 250000};    /* 250ms */
4334
4335         getmicrotime(&tv);
4336         timevalsub(&tv, &interval);
4337         if (timevalcmp(&tv, &pi->last_refreshed, <))
4338                 return;
4339
4340         t4_get_port_stats(sc, pi->tx_chan, s);
4341
4342         ifp->if_opackets = s->tx_frames - s->tx_pause;
4343         ifp->if_ipackets = s->rx_frames - s->rx_pause;
4344         ifp->if_obytes = s->tx_octets - s->tx_pause * 64;
4345         ifp->if_ibytes = s->rx_octets - s->rx_pause * 64;
4346         ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause;
4347         ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause;
4348         ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
4349             s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
4350             s->rx_trunc3;
4351         for (i = 0; i < NCHAN; i++) {
4352                 if (pi->rx_chan_map & (1 << i)) {
4353                         uint32_t v;
4354
4355                         mtx_lock(&sc->regwin_lock);
4356                         t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
4357                             1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4358                         mtx_unlock(&sc->regwin_lock);
4359                         ifp->if_iqdrops += v;
4360                 }
4361         }
4362
4363         drops = s->tx_drop;
4364         for_each_txq(pi, i, txq)
4365                 drops += counter_u64_fetch(txq->r->drops);
4366         ifp->if_snd.ifq_drops = drops;
4367
4368         ifp->if_oerrors = s->tx_error_frames;
4369         ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long +
4370             s->rx_fcs_err + s->rx_len_err;
4371
4372         getmicrotime(&pi->last_refreshed);
4373 }
4374
4375 static void
4376 cxgbe_tick(void *arg)
4377 {
4378         struct port_info *pi = arg;
4379         struct adapter *sc = pi->adapter;
4380         struct ifnet *ifp = pi->ifp;
4381
4382         PORT_LOCK(pi);
4383         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4384                 PORT_UNLOCK(pi);
4385                 return; /* without scheduling another callout */
4386         }
4387
4388         cxgbe_refresh_stats(sc, pi);
4389
4390         callout_schedule(&pi->tick, hz);
4391         PORT_UNLOCK(pi);
4392 }
4393
4394 static void
4395 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4396 {
4397         struct ifnet *vlan;
4398
4399         if (arg != ifp || ifp->if_type != IFT_ETHER)
4400                 return;
4401
4402         vlan = VLAN_DEVAT(ifp, vid);
4403         VLAN_SETCOOKIE(vlan, ifp);
4404 }
4405
4406 static int
4407 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4408 {
4409
4410 #ifdef INVARIANTS
4411         panic("%s: opcode 0x%02x on iq %p with payload %p",
4412             __func__, rss->opcode, iq, m);
4413 #else
4414         log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
4415             __func__, rss->opcode, iq, m);
4416         m_freem(m);
4417 #endif
4418         return (EDOOFUS);
4419 }
4420
4421 int
4422 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
4423 {
4424         uintptr_t *loc, new;
4425
4426         if (opcode >= nitems(sc->cpl_handler))
4427                 return (EINVAL);
4428
4429         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
4430         loc = (uintptr_t *) &sc->cpl_handler[opcode];
4431         atomic_store_rel_ptr(loc, new);
4432
4433         return (0);
4434 }
4435
4436 static int
4437 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
4438 {
4439
4440 #ifdef INVARIANTS
4441         panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
4442 #else
4443         log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
4444             __func__, iq, ctrl);
4445 #endif
4446         return (EDOOFUS);
4447 }
4448
4449 int
4450 t4_register_an_handler(struct adapter *sc, an_handler_t h)
4451 {
4452         uintptr_t *loc, new;
4453
4454         new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
4455         loc = (uintptr_t *) &sc->an_handler;
4456         atomic_store_rel_ptr(loc, new);
4457
4458         return (0);
4459 }
4460
4461 static int
4462 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
4463 {
4464         const struct cpl_fw6_msg *cpl =
4465             __containerof(rpl, struct cpl_fw6_msg, data[0]);
4466
4467 #ifdef INVARIANTS
4468         panic("%s: fw_msg type %d", __func__, cpl->type);
4469 #else
4470         log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
4471 #endif
4472         return (EDOOFUS);
4473 }
4474
4475 int
4476 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
4477 {
4478         uintptr_t *loc, new;
4479
4480         if (type >= nitems(sc->fw_msg_handler))
4481                 return (EINVAL);
4482
4483         /*
4484          * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
4485          * handler dispatch table.  Reject any attempt to install a handler for
4486          * this subtype.
4487          */
4488         if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
4489                 return (EINVAL);
4490
4491         new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
4492         loc = (uintptr_t *) &sc->fw_msg_handler[type];
4493         atomic_store_rel_ptr(loc, new);
4494
4495         return (0);
4496 }
4497
4498 static int
4499 t4_sysctls(struct adapter *sc)
4500 {
4501         struct sysctl_ctx_list *ctx;
4502         struct sysctl_oid *oid;
4503         struct sysctl_oid_list *children, *c0;
4504         static char *caps[] = {
4505                 "\20\1PPP\2QFC\3DCBX",                  /* caps[0] linkcaps */
4506                 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL"        /* caps[1] niccaps */
4507                     "\6HASHFILTER\7ETHOFLD",
4508                 "\20\1TOE",                             /* caps[2] toecaps */
4509                 "\20\1RDDP\2RDMAC",                     /* caps[3] rdmacaps */
4510                 "\20\1INITIATOR_PDU\2TARGET_PDU"        /* caps[4] iscsicaps */
4511                     "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
4512                     "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
4513                 "\20\1INITIATOR\2TARGET\3CTRL_OFLD"     /* caps[5] fcoecaps */
4514                     "\4PO_INITIAOR\5PO_TARGET"
4515         };
4516         static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
4517
4518         ctx = device_get_sysctl_ctx(sc->dev);
4519
4520         /*
4521          * dev.t4nex.X.
4522          */
4523         oid = device_get_sysctl_tree(sc->dev);
4524         c0 = children = SYSCTL_CHILDREN(oid);
4525
4526         sc->sc_do_rxcopy = 1;
4527         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
4528             &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
4529
4530         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
4531             sc->params.nports, "# of ports");
4532
4533         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
4534             NULL, chip_rev(sc), "chip hardware revision");
4535
4536         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
4537             CTLFLAG_RD, sc->fw_version, 0, "firmware version");
4538
4539         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
4540             CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
4541
4542         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
4543             sc->cfcsum, "config file checksum");
4544
4545         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
4546             CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
4547             sysctl_bitfield, "A", "available doorbells");
4548
4549         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
4550             CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
4551             sysctl_bitfield, "A", "available link capabilities");
4552
4553         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
4554             CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
4555             sysctl_bitfield, "A", "available NIC capabilities");
4556
4557         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
4558             CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
4559             sysctl_bitfield, "A", "available TCP offload capabilities");
4560
4561         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
4562             CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
4563             sysctl_bitfield, "A", "available RDMA capabilities");
4564
4565         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
4566             CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
4567             sysctl_bitfield, "A", "available iSCSI capabilities");
4568
4569         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
4570             CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
4571             sysctl_bitfield, "A", "available FCoE capabilities");
4572
4573         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
4574             sc->params.vpd.cclk, "core clock frequency (in KHz)");
4575
4576         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
4577             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
4578             sizeof(sc->sge.timer_val), sysctl_int_array, "A",
4579             "interrupt holdoff timer values (us)");
4580
4581         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
4582             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
4583             sizeof(sc->sge.counter_val), sysctl_int_array, "A",
4584             "interrupt holdoff packet counter values");
4585
4586         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
4587             NULL, sc->tids.nftids, "number of filters");
4588
4589         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
4590             CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
4591             "chip temperature (in Celsius)");
4592
4593         t4_sge_sysctls(sc, ctx, children);
4594
4595         sc->lro_timeout = 100;
4596         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
4597             &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
4598
4599         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "debug_flags", CTLFLAG_RW,
4600             &sc->debug_flags, 0, "flags to enable runtime debugging");
4601
4602 #ifdef SBUF_DRAIN
4603         /*
4604          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
4605          */
4606         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
4607             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
4608             "logs and miscellaneous information");
4609         children = SYSCTL_CHILDREN(oid);
4610
4611         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
4612             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4613             sysctl_cctrl, "A", "congestion control");
4614
4615         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
4616             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4617             sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
4618
4619         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
4620             CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
4621             sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
4622
4623         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
4624             CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
4625             sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
4626
4627         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
4628             CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
4629             sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
4630
4631         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
4632             CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
4633             sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
4634
4635         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
4636             CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
4637             sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
4638
4639         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
4640             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4641             sysctl_cim_la, "A", "CIM logic analyzer");
4642
4643         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
4644             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4645             sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
4646
4647         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
4648             CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
4649             sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
4650
4651         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
4652             CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
4653             sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
4654
4655         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
4656             CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
4657             sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
4658
4659         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
4660             CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
4661             sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
4662
4663         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
4664             CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
4665             sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
4666
4667         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
4668             CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
4669             sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
4670
4671         if (is_t5(sc)) {
4672                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
4673                     CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
4674                     sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
4675
4676                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
4677                     CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
4678                     sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
4679         }
4680
4681         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
4682             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4683             sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
4684
4685         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
4686             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4687             sysctl_cim_qcfg, "A", "CIM queue configuration");
4688
4689         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
4690             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4691             sysctl_cpl_stats, "A", "CPL statistics");
4692
4693         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
4694             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4695             sysctl_ddp_stats, "A", "non-TCP DDP statistics");
4696
4697         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
4698             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4699             sysctl_devlog, "A", "firmware's device log");
4700
4701         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
4702             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4703             sysctl_fcoe_stats, "A", "FCoE statistics");
4704
4705         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
4706             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4707             sysctl_hw_sched, "A", "hardware scheduler ");
4708
4709         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
4710             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4711             sysctl_l2t, "A", "hardware L2 table");
4712
4713         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
4714             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4715             sysctl_lb_stats, "A", "loopback statistics");
4716
4717         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
4718             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4719             sysctl_meminfo, "A", "memory regions");
4720
4721         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
4722             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4723             sysctl_mps_tcam, "A", "MPS TCAM entries");
4724
4725         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
4726             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4727             sysctl_path_mtus, "A", "path MTUs");
4728
4729         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
4730             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4731             sysctl_pm_stats, "A", "PM statistics");
4732
4733         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
4734             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4735             sysctl_rdma_stats, "A", "RDMA statistics");
4736
4737         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
4738             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4739             sysctl_tcp_stats, "A", "TCP statistics");
4740
4741         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
4742             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4743             sysctl_tids, "A", "TID information");
4744
4745         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
4746             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4747             sysctl_tp_err_stats, "A", "TP error statistics");
4748
4749         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
4750             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4751             sysctl_tp_la, "A", "TP logic analyzer");
4752
4753         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
4754             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4755             sysctl_tx_rate, "A", "Tx rate");
4756
4757         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
4758             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4759             sysctl_ulprx_la, "A", "ULPRX logic analyzer");
4760
4761         if (is_t5(sc)) {
4762                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
4763                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4764                     sysctl_wcwr_stats, "A", "write combined work requests");
4765         }
4766 #endif
4767
4768 #ifdef TCP_OFFLOAD
4769         if (is_offload(sc)) {
4770                 /*
4771                  * dev.t4nex.X.toe.
4772                  */
4773                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
4774                     NULL, "TOE parameters");
4775                 children = SYSCTL_CHILDREN(oid);
4776
4777                 sc->tt.sndbuf = 256 * 1024;
4778                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
4779                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
4780
4781                 sc->tt.ddp = 0;
4782                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
4783                     &sc->tt.ddp, 0, "DDP allowed");
4784
4785                 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
4786                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
4787                     &sc->tt.indsz, 0, "DDP max indicate size allowed");
4788
4789                 sc->tt.ddp_thres =
4790                     G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
4791                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
4792                     &sc->tt.ddp_thres, 0, "DDP threshold");
4793
4794                 sc->tt.rx_coalesce = 1;
4795                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
4796                     CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
4797
4798                 sc->tt.tx_align = 1;
4799                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align",
4800                     CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload");
4801         }
4802 #endif
4803
4804
4805         return (0);
4806 }
4807
4808 static int
4809 cxgbe_sysctls(struct port_info *pi)
4810 {
4811         struct sysctl_ctx_list *ctx;
4812         struct sysctl_oid *oid;
4813         struct sysctl_oid_list *children;
4814         struct adapter *sc = pi->adapter;
4815
4816         ctx = device_get_sysctl_ctx(pi->dev);
4817
4818         /*
4819          * dev.cxgbe.X.
4820          */
4821         oid = device_get_sysctl_tree(pi->dev);
4822         children = SYSCTL_CHILDREN(oid);
4823
4824         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
4825            CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
4826         if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
4827                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
4828                     CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
4829                     "PHY temperature (in Celsius)");
4830                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
4831                     CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
4832                     "PHY firmware version");
4833         }
4834         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
4835             &pi->nrxq, 0, "# of rx queues");
4836         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
4837             &pi->ntxq, 0, "# of tx queues");
4838         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
4839             &pi->first_rxq, 0, "index of first rx queue");
4840         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
4841             &pi->first_txq, 0, "index of first tx queue");
4842         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
4843             CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU",
4844             "Reserve queue 0 for non-flowid packets");
4845
4846 #ifdef TCP_OFFLOAD
4847         if (is_offload(sc)) {
4848                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
4849                     &pi->nofldrxq, 0,
4850                     "# of rx queues for offloaded TCP connections");
4851                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
4852                     &pi->nofldtxq, 0,
4853                     "# of tx queues for offloaded TCP connections");
4854                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
4855                     CTLFLAG_RD, &pi->first_ofld_rxq, 0,
4856                     "index of first TOE rx queue");
4857                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
4858                     CTLFLAG_RD, &pi->first_ofld_txq, 0,
4859                     "index of first TOE tx queue");
4860         }
4861 #endif
4862 #ifdef DEV_NETMAP
4863         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
4864             &pi->nnmrxq, 0, "# of rx queues for netmap");
4865         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
4866             &pi->nnmtxq, 0, "# of tx queues for netmap");
4867         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
4868             CTLFLAG_RD, &pi->first_nm_rxq, 0,
4869             "index of first netmap rx queue");
4870         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
4871             CTLFLAG_RD, &pi->first_nm_txq, 0,
4872             "index of first netmap tx queue");
4873 #endif
4874
4875         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
4876             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
4877             "holdoff timer index");
4878         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
4879             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
4880             "holdoff packet counter index");
4881
4882         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
4883             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
4884             "rx queue size");
4885         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
4886             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
4887             "tx queue size");
4888
4889         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
4890             CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings,
4891             "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
4892
4893         /*
4894          * dev.cxgbe.X.stats.
4895          */
4896         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
4897             NULL, "port statistics");
4898         children = SYSCTL_CHILDREN(oid);
4899         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD,
4900             &pi->tx_parse_error, 0,
4901             "# of tx packets with invalid length or # of segments");
4902
4903 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
4904         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
4905             CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
4906             sysctl_handle_t4_reg64, "QU", desc)
4907
4908         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
4909             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
4910         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
4911             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
4912         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
4913             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
4914         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
4915             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
4916         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
4917             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
4918         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
4919             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
4920         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
4921             "# of tx frames in this range",
4922             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
4923         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
4924             "# of tx frames in this range",
4925             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
4926         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
4927             "# of tx frames in this range",
4928             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
4929         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
4930             "# of tx frames in this range",
4931             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
4932         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
4933             "# of tx frames in this range",
4934             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
4935         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
4936             "# of tx frames in this range",
4937             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
4938         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
4939             "# of tx frames in this range",
4940             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
4941         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
4942             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
4943         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
4944             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
4945         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
4946             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
4947         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
4948             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
4949         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
4950             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
4951         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
4952             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
4953         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
4954             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
4955         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
4956             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
4957         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
4958             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
4959         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
4960             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
4961
4962         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
4963             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
4964         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
4965             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
4966         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
4967             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
4968         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
4969             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
4970         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
4971             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
4972         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
4973             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
4974         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
4975             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
4976         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
4977             "# of frames received with bad FCS",
4978             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
4979         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
4980             "# of frames received with length error",
4981             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
4982         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
4983             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
4984         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
4985             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
4986         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
4987             "# of rx frames in this range",
4988             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
4989         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
4990             "# of rx frames in this range",
4991             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
4992         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
4993             "# of rx frames in this range",
4994             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
4995         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
4996             "# of rx frames in this range",
4997             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
4998         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
4999             "# of rx frames in this range",
5000             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
5001         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
5002             "# of rx frames in this range",
5003             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
5004         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
5005             "# of rx frames in this range",
5006             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
5007         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
5008             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
5009         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
5010             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
5011         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
5012             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
5013         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
5014             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
5015         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
5016             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
5017         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
5018             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
5019         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
5020             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
5021         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
5022             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
5023         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
5024             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
5025
5026 #undef SYSCTL_ADD_T4_REG64
5027
5028 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
5029         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
5030             &pi->stats.name, desc)
5031
5032         /* We get these from port_stats and they may be stale by upto 1s */
5033         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
5034             "# drops due to buffer-group 0 overflows");
5035         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
5036             "# drops due to buffer-group 1 overflows");
5037         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
5038             "# drops due to buffer-group 2 overflows");
5039         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
5040             "# drops due to buffer-group 3 overflows");
5041         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
5042             "# of buffer-group 0 truncated packets");
5043         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
5044             "# of buffer-group 1 truncated packets");
5045         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
5046             "# of buffer-group 2 truncated packets");
5047         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
5048             "# of buffer-group 3 truncated packets");
5049
5050 #undef SYSCTL_ADD_T4_PORTSTAT
5051
5052         return (0);
5053 }
5054
5055 static int
5056 sysctl_int_array(SYSCTL_HANDLER_ARGS)
5057 {
5058         int rc, *i, space = 0;
5059         struct sbuf sb;
5060
5061         sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
5062         for (i = arg1; arg2; arg2 -= sizeof(int), i++) {
5063                 if (space)
5064                         sbuf_printf(&sb, " ");
5065                 sbuf_printf(&sb, "%d", *i);
5066                 space = 1;
5067         }
5068         sbuf_finish(&sb);
5069         rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
5070         sbuf_delete(&sb);
5071         return (rc);
5072 }
5073
5074 static int
5075 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
5076 {
5077         int rc;
5078         struct sbuf *sb;
5079
5080         rc = sysctl_wire_old_buffer(req, 0);
5081         if (rc != 0)
5082                 return(rc);
5083
5084         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5085         if (sb == NULL)
5086                 return (ENOMEM);
5087
5088         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
5089         rc = sbuf_finish(sb);
5090         sbuf_delete(sb);
5091
5092         return (rc);
5093 }
5094
5095 static int
5096 sysctl_btphy(SYSCTL_HANDLER_ARGS)
5097 {
5098         struct port_info *pi = arg1;
5099         int op = arg2;
5100         struct adapter *sc = pi->adapter;
5101         u_int v;
5102         int rc;
5103
5104         rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt");
5105         if (rc)
5106                 return (rc);
5107         /* XXX: magic numbers */
5108         rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
5109             &v);
5110         end_synchronized_op(sc, 0);
5111         if (rc)
5112                 return (rc);
5113         if (op == 0)
5114                 v /= 256;
5115
5116         rc = sysctl_handle_int(oidp, &v, 0, req);
5117         return (rc);
5118 }
5119
5120 static int
5121 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
5122 {
5123         struct port_info *pi = arg1;
5124         int rc, val;
5125
5126         val = pi->rsrv_noflowq;
5127         rc = sysctl_handle_int(oidp, &val, 0, req);
5128         if (rc != 0 || req->newptr == NULL)
5129                 return (rc);
5130
5131         if ((val >= 1) && (pi->ntxq > 1))
5132                 pi->rsrv_noflowq = 1;
5133         else
5134                 pi->rsrv_noflowq = 0;
5135
5136         return (rc);
5137 }
5138
5139 static int
5140 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
5141 {
5142         struct port_info *pi = arg1;
5143         struct adapter *sc = pi->adapter;
5144         int idx, rc, i;
5145         struct sge_rxq *rxq;
5146 #ifdef TCP_OFFLOAD
5147         struct sge_ofld_rxq *ofld_rxq;
5148 #endif
5149         uint8_t v;
5150
5151         idx = pi->tmr_idx;
5152
5153         rc = sysctl_handle_int(oidp, &idx, 0, req);
5154         if (rc != 0 || req->newptr == NULL)
5155                 return (rc);
5156
5157         if (idx < 0 || idx >= SGE_NTIMERS)
5158                 return (EINVAL);
5159
5160         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5161             "t4tmr");
5162         if (rc)
5163                 return (rc);
5164
5165         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
5166         for_each_rxq(pi, i, rxq) {
5167 #ifdef atomic_store_rel_8
5168                 atomic_store_rel_8(&rxq->iq.intr_params, v);
5169 #else
5170                 rxq->iq.intr_params = v;
5171 #endif
5172         }
5173 #ifdef TCP_OFFLOAD
5174         for_each_ofld_rxq(pi, i, ofld_rxq) {
5175 #ifdef atomic_store_rel_8
5176                 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
5177 #else
5178                 ofld_rxq->iq.intr_params = v;
5179 #endif
5180         }
5181 #endif
5182         pi->tmr_idx = idx;
5183
5184         end_synchronized_op(sc, LOCK_HELD);
5185         return (0);
5186 }
5187
5188 static int
5189 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
5190 {
5191         struct port_info *pi = arg1;
5192         struct adapter *sc = pi->adapter;
5193         int idx, rc;
5194
5195         idx = pi->pktc_idx;
5196
5197         rc = sysctl_handle_int(oidp, &idx, 0, req);
5198         if (rc != 0 || req->newptr == NULL)
5199                 return (rc);
5200
5201         if (idx < -1 || idx >= SGE_NCOUNTERS)
5202                 return (EINVAL);
5203
5204         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5205             "t4pktc");
5206         if (rc)
5207                 return (rc);
5208
5209         if (pi->flags & PORT_INIT_DONE)
5210                 rc = EBUSY; /* cannot be changed once the queues are created */
5211         else
5212                 pi->pktc_idx = idx;
5213
5214         end_synchronized_op(sc, LOCK_HELD);
5215         return (rc);
5216 }
5217
5218 static int
5219 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
5220 {
5221         struct port_info *pi = arg1;
5222         struct adapter *sc = pi->adapter;
5223         int qsize, rc;
5224
5225         qsize = pi->qsize_rxq;
5226
5227         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5228         if (rc != 0 || req->newptr == NULL)
5229                 return (rc);
5230
5231         if (qsize < 128 || (qsize & 7))
5232                 return (EINVAL);
5233
5234         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5235             "t4rxqs");
5236         if (rc)
5237                 return (rc);
5238
5239         if (pi->flags & PORT_INIT_DONE)
5240                 rc = EBUSY; /* cannot be changed once the queues are created */
5241         else
5242                 pi->qsize_rxq = qsize;
5243
5244         end_synchronized_op(sc, LOCK_HELD);
5245         return (rc);
5246 }
5247
5248 static int
5249 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
5250 {
5251         struct port_info *pi = arg1;
5252         struct adapter *sc = pi->adapter;
5253         int qsize, rc;
5254
5255         qsize = pi->qsize_txq;
5256
5257         rc = sysctl_handle_int(oidp, &qsize, 0, req);
5258         if (rc != 0 || req->newptr == NULL)
5259                 return (rc);
5260
5261         if (qsize < 128 || qsize > 65536)
5262                 return (EINVAL);
5263
5264         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5265             "t4txqs");
5266         if (rc)
5267                 return (rc);
5268
5269         if (pi->flags & PORT_INIT_DONE)
5270                 rc = EBUSY; /* cannot be changed once the queues are created */
5271         else
5272                 pi->qsize_txq = qsize;
5273
5274         end_synchronized_op(sc, LOCK_HELD);
5275         return (rc);
5276 }
5277
5278 static int
5279 sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
5280 {
5281         struct port_info *pi = arg1;
5282         struct adapter *sc = pi->adapter;
5283         struct link_config *lc = &pi->link_cfg;
5284         int rc;
5285
5286         if (req->newptr == NULL) {
5287                 struct sbuf *sb;
5288                 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX";
5289
5290                 rc = sysctl_wire_old_buffer(req, 0);
5291                 if (rc != 0)
5292                         return(rc);
5293
5294                 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5295                 if (sb == NULL)
5296                         return (ENOMEM);
5297
5298                 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits);
5299                 rc = sbuf_finish(sb);
5300                 sbuf_delete(sb);
5301         } else {
5302                 char s[2];
5303                 int n;
5304
5305                 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX));
5306                 s[1] = 0;
5307
5308                 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
5309                 if (rc != 0)
5310                         return(rc);
5311
5312                 if (s[1] != 0)
5313                         return (EINVAL);
5314                 if (s[0] < '0' || s[0] > '9')
5315                         return (EINVAL);        /* not a number */
5316                 n = s[0] - '0';
5317                 if (n & ~(PAUSE_TX | PAUSE_RX))
5318                         return (EINVAL);        /* some other bit is set too */
5319
5320                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE");
5321                 if (rc)
5322                         return (rc);
5323                 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) {
5324                         int link_ok = lc->link_ok;
5325
5326                         lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX);
5327                         lc->requested_fc |= n;
5328                         rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc);
5329                         lc->link_ok = link_ok;  /* restore */
5330                 }
5331                 end_synchronized_op(sc, 0);
5332         }
5333
5334         return (rc);
5335 }
5336
5337 static int
5338 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
5339 {
5340         struct adapter *sc = arg1;
5341         int reg = arg2;
5342         uint64_t val;
5343
5344         val = t4_read_reg64(sc, reg);
5345
5346         return (sysctl_handle_64(oidp, &val, 0, req));
5347 }
5348
5349 static int
5350 sysctl_temperature(SYSCTL_HANDLER_ARGS)
5351 {
5352         struct adapter *sc = arg1;
5353         int rc, t;
5354         uint32_t param, val;
5355
5356         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
5357         if (rc)
5358                 return (rc);
5359         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5360             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5361             V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
5362         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
5363         end_synchronized_op(sc, 0);
5364         if (rc)
5365                 return (rc);
5366
5367         /* unknown is returned as 0 but we display -1 in that case */
5368         t = val == 0 ? -1 : val;
5369
5370         rc = sysctl_handle_int(oidp, &t, 0, req);
5371         return (rc);
5372 }
5373
5374 #ifdef SBUF_DRAIN
5375 static int
5376 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
5377 {
5378         struct adapter *sc = arg1;
5379         struct sbuf *sb;
5380         int rc, i;
5381         uint16_t incr[NMTUS][NCCTRL_WIN];
5382         static const char *dec_fac[] = {
5383                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
5384                 "0.9375"
5385         };
5386
5387         rc = sysctl_wire_old_buffer(req, 0);
5388         if (rc != 0)
5389                 return (rc);
5390
5391         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5392         if (sb == NULL)
5393                 return (ENOMEM);
5394
5395         t4_read_cong_tbl(sc, incr);
5396
5397         for (i = 0; i < NCCTRL_WIN; ++i) {
5398                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
5399                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
5400                     incr[5][i], incr[6][i], incr[7][i]);
5401                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
5402                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
5403                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
5404                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
5405         }
5406
5407         rc = sbuf_finish(sb);
5408         sbuf_delete(sb);
5409
5410         return (rc);
5411 }
5412
5413 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
5414         "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",   /* ibq's */
5415         "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
5416         "SGE0-RX", "SGE1-RX"    /* additional obq's (T5 onwards) */
5417 };
5418
5419 static int
5420 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
5421 {
5422         struct adapter *sc = arg1;
5423         struct sbuf *sb;
5424         int rc, i, n, qid = arg2;
5425         uint32_t *buf, *p;
5426         char *qtype;
5427         u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
5428
5429         KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
5430             ("%s: bad qid %d\n", __func__, qid));
5431
5432         if (qid < CIM_NUM_IBQ) {
5433                 /* inbound queue */
5434                 qtype = "IBQ";
5435                 n = 4 * CIM_IBQ_SIZE;
5436                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5437                 rc = t4_read_cim_ibq(sc, qid, buf, n);
5438         } else {
5439                 /* outbound queue */
5440                 qtype = "OBQ";
5441                 qid -= CIM_NUM_IBQ;
5442                 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
5443                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5444                 rc = t4_read_cim_obq(sc, qid, buf, n);
5445         }
5446
5447         if (rc < 0) {
5448                 rc = -rc;
5449                 goto done;
5450         }
5451         n = rc * sizeof(uint32_t);      /* rc has # of words actually read */
5452
5453         rc = sysctl_wire_old_buffer(req, 0);
5454         if (rc != 0)
5455                 goto done;
5456
5457         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5458         if (sb == NULL) {
5459                 rc = ENOMEM;
5460                 goto done;
5461         }
5462
5463         sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
5464         for (i = 0, p = buf; i < n; i += 16, p += 4)
5465                 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
5466                     p[2], p[3]);
5467
5468         rc = sbuf_finish(sb);
5469         sbuf_delete(sb);
5470 done:
5471         free(buf, M_CXGBE);
5472         return (rc);
5473 }
5474
5475 static int
5476 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
5477 {
5478         struct adapter *sc = arg1;
5479         u_int cfg;
5480         struct sbuf *sb;
5481         uint32_t *buf, *p;
5482         int rc;
5483
5484         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
5485         if (rc != 0)
5486                 return (rc);
5487
5488         rc = sysctl_wire_old_buffer(req, 0);
5489         if (rc != 0)
5490                 return (rc);
5491
5492         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5493         if (sb == NULL)
5494                 return (ENOMEM);
5495
5496         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
5497             M_ZERO | M_WAITOK);
5498
5499         rc = -t4_cim_read_la(sc, buf, NULL);
5500         if (rc != 0)
5501                 goto done;
5502
5503         sbuf_printf(sb, "Status   Data      PC%s",
5504             cfg & F_UPDBGLACAPTPCONLY ? "" :
5505             "     LS0Stat  LS0Addr             LS0Data");
5506
5507         KASSERT((sc->params.cim_la_size & 7) == 0,
5508             ("%s: p will walk off the end of buf", __func__));
5509
5510         for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
5511                 if (cfg & F_UPDBGLACAPTPCONLY) {
5512                         sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
5513                             p[6], p[7]);
5514                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
5515                             (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
5516                             p[4] & 0xff, p[5] >> 8);
5517                         sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
5518                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5519                             p[1] & 0xf, p[2] >> 4);
5520                 } else {
5521                         sbuf_printf(sb,
5522                             "\n  %02x   %x%07x %x%07x %08x %08x "
5523                             "%08x%08x%08x%08x",
5524                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5525                             p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
5526                             p[6], p[7]);
5527                 }
5528         }
5529
5530         rc = sbuf_finish(sb);
5531         sbuf_delete(sb);
5532 done:
5533         free(buf, M_CXGBE);
5534         return (rc);
5535 }
5536
5537 static int
5538 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
5539 {
5540         struct adapter *sc = arg1;
5541         u_int i;
5542         struct sbuf *sb;
5543         uint32_t *buf, *p;
5544         int rc;
5545
5546         rc = sysctl_wire_old_buffer(req, 0);
5547         if (rc != 0)
5548                 return (rc);
5549
5550         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5551         if (sb == NULL)
5552                 return (ENOMEM);
5553
5554         buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
5555             M_ZERO | M_WAITOK);
5556
5557         t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
5558         p = buf;
5559
5560         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5561                 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
5562                     p[1], p[0]);
5563         }
5564
5565         sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
5566         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5567                 sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
5568                     (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
5569                     (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
5570                     (p[1] >> 2) | ((p[2] & 3) << 30),
5571                     (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
5572                     p[0] & 1);
5573         }
5574
5575         rc = sbuf_finish(sb);
5576         sbuf_delete(sb);
5577         free(buf, M_CXGBE);
5578         return (rc);
5579 }
5580
5581 static int
5582 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
5583 {
5584         struct adapter *sc = arg1;
5585         u_int i;
5586         struct sbuf *sb;
5587         uint32_t *buf, *p;
5588         int rc;
5589
5590         rc = sysctl_wire_old_buffer(req, 0);
5591         if (rc != 0)
5592                 return (rc);
5593
5594         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5595         if (sb == NULL)
5596                 return (ENOMEM);
5597
5598         buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
5599             M_ZERO | M_WAITOK);
5600
5601         t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
5602         p = buf;
5603
5604         sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
5605         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5606                 sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
5607                     (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
5608                     p[4], p[3], p[2], p[1], p[0]);
5609         }
5610
5611         sbuf_printf(sb, "\n\nCntl ID               Data");
5612         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5613                 sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
5614                     (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
5615         }
5616
5617         rc = sbuf_finish(sb);
5618         sbuf_delete(sb);
5619         free(buf, M_CXGBE);
5620         return (rc);
5621 }
5622
5623 static int
5624 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
5625 {
5626         struct adapter *sc = arg1;
5627         struct sbuf *sb;
5628         int rc, i;
5629         uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5630         uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5631         uint16_t thres[CIM_NUM_IBQ];
5632         uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
5633         uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
5634         u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
5635
5636         if (is_t4(sc)) {
5637                 cim_num_obq = CIM_NUM_OBQ;
5638                 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
5639                 obq_rdaddr = A_UP_OBQ_0_REALADDR;
5640         } else {
5641                 cim_num_obq = CIM_NUM_OBQ_T5;
5642                 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
5643                 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
5644         }
5645         nq = CIM_NUM_IBQ + cim_num_obq;
5646
5647         rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
5648         if (rc == 0)
5649                 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
5650         if (rc != 0)
5651                 return (rc);
5652
5653         t4_read_cimq_cfg(sc, base, size, thres);
5654
5655         rc = sysctl_wire_old_buffer(req, 0);
5656         if (rc != 0)
5657                 return (rc);
5658
5659         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5660         if (sb == NULL)
5661                 return (ENOMEM);
5662
5663         sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
5664
5665         for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
5666                 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
5667                     qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
5668                     G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5669                     G_QUEREMFLITS(p[2]) * 16);
5670         for ( ; i < nq; i++, p += 4, wr += 2)
5671                 sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
5672                     base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
5673                     wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5674                     G_QUEREMFLITS(p[2]) * 16);
5675
5676         rc = sbuf_finish(sb);
5677         sbuf_delete(sb);
5678
5679         return (rc);
5680 }
5681
5682 static int
5683 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
5684 {
5685         struct adapter *sc = arg1;
5686         struct sbuf *sb;
5687         int rc;
5688         struct tp_cpl_stats stats;
5689
5690         rc = sysctl_wire_old_buffer(req, 0);
5691         if (rc != 0)
5692                 return (rc);
5693
5694         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5695         if (sb == NULL)
5696                 return (ENOMEM);
5697
5698         t4_tp_get_cpl_stats(sc, &stats);
5699
5700         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
5701             "channel 3\n");
5702         sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
5703                    stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
5704         sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
5705                    stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
5706
5707         rc = sbuf_finish(sb);
5708         sbuf_delete(sb);
5709
5710         return (rc);
5711 }
5712
5713 static int
5714 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
5715 {
5716         struct adapter *sc = arg1;
5717         struct sbuf *sb;
5718         int rc;
5719         struct tp_usm_stats stats;
5720
5721         rc = sysctl_wire_old_buffer(req, 0);
5722         if (rc != 0)
5723                 return(rc);
5724
5725         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5726         if (sb == NULL)
5727                 return (ENOMEM);
5728
5729         t4_get_usm_stats(sc, &stats);
5730
5731         sbuf_printf(sb, "Frames: %u\n", stats.frames);
5732         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
5733         sbuf_printf(sb, "Drops:  %u", stats.drops);
5734
5735         rc = sbuf_finish(sb);
5736         sbuf_delete(sb);
5737
5738         return (rc);
5739 }
5740
5741 const char *devlog_level_strings[] = {
5742         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
5743         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
5744         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
5745         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
5746         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
5747         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
5748 };
5749
5750 const char *devlog_facility_strings[] = {
5751         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
5752         [FW_DEVLOG_FACILITY_CF]         = "CF",
5753         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
5754         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
5755         [FW_DEVLOG_FACILITY_RES]        = "RES",
5756         [FW_DEVLOG_FACILITY_HW]         = "HW",
5757         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
5758         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
5759         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
5760         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
5761         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
5762         [FW_DEVLOG_FACILITY_VI]         = "VI",
5763         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
5764         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
5765         [FW_DEVLOG_FACILITY_TM]         = "TM",
5766         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
5767         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
5768         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
5769         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
5770         [FW_DEVLOG_FACILITY_RI]         = "RI",
5771         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
5772         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
5773         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
5774         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE"
5775 };
5776
5777 static int
5778 sysctl_devlog(SYSCTL_HANDLER_ARGS)
5779 {
5780         struct adapter *sc = arg1;
5781         struct devlog_params *dparams = &sc->params.devlog;
5782         struct fw_devlog_e *buf, *e;
5783         int i, j, rc, nentries, first = 0, m;
5784         struct sbuf *sb;
5785         uint64_t ftstamp = UINT64_MAX;
5786
5787         if (dparams->start == 0) {
5788                 dparams->memtype = FW_MEMTYPE_EDC0;
5789                 dparams->start = 0x84000;
5790                 dparams->size = 32768;
5791         }
5792
5793         nentries = dparams->size / sizeof(struct fw_devlog_e);
5794
5795         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
5796         if (buf == NULL)
5797                 return (ENOMEM);
5798
5799         m = fwmtype_to_hwmtype(dparams->memtype);
5800         rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
5801         if (rc != 0)
5802                 goto done;
5803
5804         for (i = 0; i < nentries; i++) {
5805                 e = &buf[i];
5806
5807                 if (e->timestamp == 0)
5808                         break;  /* end */
5809
5810                 e->timestamp = be64toh(e->timestamp);
5811                 e->seqno = be32toh(e->seqno);
5812                 for (j = 0; j < 8; j++)
5813                         e->params[j] = be32toh(e->params[j]);
5814
5815                 if (e->timestamp < ftstamp) {
5816                         ftstamp = e->timestamp;
5817                         first = i;
5818                 }
5819         }
5820
5821         if (buf[first].timestamp == 0)
5822                 goto done;      /* nothing in the log */
5823
5824         rc = sysctl_wire_old_buffer(req, 0);
5825         if (rc != 0)
5826                 goto done;
5827
5828         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5829         if (sb == NULL) {
5830                 rc = ENOMEM;
5831                 goto done;
5832         }
5833         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
5834             "Seq#", "Tstamp", "Level", "Facility", "Message");
5835
5836         i = first;
5837         do {
5838                 e = &buf[i];
5839                 if (e->timestamp == 0)
5840                         break;  /* end */
5841
5842                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
5843                     e->seqno, e->timestamp,
5844                     (e->level < nitems(devlog_level_strings) ?
5845                         devlog_level_strings[e->level] : "UNKNOWN"),
5846                     (e->facility < nitems(devlog_facility_strings) ?
5847                         devlog_facility_strings[e->facility] : "UNKNOWN"));
5848                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
5849                     e->params[2], e->params[3], e->params[4],
5850                     e->params[5], e->params[6], e->params[7]);
5851
5852                 if (++i == nentries)
5853                         i = 0;
5854         } while (i != first);
5855
5856         rc = sbuf_finish(sb);
5857         sbuf_delete(sb);
5858 done:
5859         free(buf, M_CXGBE);
5860         return (rc);
5861 }
5862
5863 static int
5864 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
5865 {
5866         struct adapter *sc = arg1;
5867         struct sbuf *sb;
5868         int rc;
5869         struct tp_fcoe_stats stats[4];
5870
5871         rc = sysctl_wire_old_buffer(req, 0);
5872         if (rc != 0)
5873                 return (rc);
5874
5875         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5876         if (sb == NULL)
5877                 return (ENOMEM);
5878
5879         t4_get_fcoe_stats(sc, 0, &stats[0]);
5880         t4_get_fcoe_stats(sc, 1, &stats[1]);
5881         t4_get_fcoe_stats(sc, 2, &stats[2]);
5882         t4_get_fcoe_stats(sc, 3, &stats[3]);
5883
5884         sbuf_printf(sb, "                   channel 0        channel 1        "
5885             "channel 2        channel 3\n");
5886         sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
5887             stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
5888             stats[3].octetsDDP);
5889         sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
5890             stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
5891         sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
5892             stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
5893             stats[3].framesDrop);
5894
5895         rc = sbuf_finish(sb);
5896         sbuf_delete(sb);
5897
5898         return (rc);
5899 }
5900
5901 static int
5902 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
5903 {
5904         struct adapter *sc = arg1;
5905         struct sbuf *sb;
5906         int rc, i;
5907         unsigned int map, kbps, ipg, mode;
5908         unsigned int pace_tab[NTX_SCHED];
5909
5910         rc = sysctl_wire_old_buffer(req, 0);
5911         if (rc != 0)
5912                 return (rc);
5913
5914         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5915         if (sb == NULL)
5916                 return (ENOMEM);
5917
5918         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
5919         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
5920         t4_read_pace_tbl(sc, pace_tab);
5921
5922         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
5923             "Class IPG (0.1 ns)   Flow IPG (us)");
5924
5925         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
5926                 t4_get_tx_sched(sc, i, &kbps, &ipg);
5927                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
5928                     (mode & (1 << i)) ? "flow" : "class", map & 3);
5929                 if (kbps)
5930                         sbuf_printf(sb, "%9u     ", kbps);
5931                 else
5932                         sbuf_printf(sb, " disabled     ");
5933
5934                 if (ipg)
5935                         sbuf_printf(sb, "%13u        ", ipg);
5936                 else
5937                         sbuf_printf(sb, "     disabled        ");
5938
5939                 if (pace_tab[i])
5940                         sbuf_printf(sb, "%10u", pace_tab[i]);
5941                 else
5942                         sbuf_printf(sb, "  disabled");
5943         }
5944
5945         rc = sbuf_finish(sb);
5946         sbuf_delete(sb);
5947
5948         return (rc);
5949 }
5950
5951 static int
5952 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
5953 {
5954         struct adapter *sc = arg1;
5955         struct sbuf *sb;
5956         int rc, i, j;
5957         uint64_t *p0, *p1;
5958         struct lb_port_stats s[2];
5959         static const char *stat_name[] = {
5960                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
5961                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
5962                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
5963                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
5964                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
5965                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
5966                 "BG2FramesTrunc:", "BG3FramesTrunc:"
5967         };
5968
5969         rc = sysctl_wire_old_buffer(req, 0);
5970         if (rc != 0)
5971                 return (rc);
5972
5973         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5974         if (sb == NULL)
5975                 return (ENOMEM);
5976
5977         memset(s, 0, sizeof(s));
5978
5979         for (i = 0; i < 4; i += 2) {
5980                 t4_get_lb_stats(sc, i, &s[0]);
5981                 t4_get_lb_stats(sc, i + 1, &s[1]);
5982
5983                 p0 = &s[0].octets;
5984                 p1 = &s[1].octets;
5985                 sbuf_printf(sb, "%s                       Loopback %u"
5986                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
5987
5988                 for (j = 0; j < nitems(stat_name); j++)
5989                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
5990                                    *p0++, *p1++);
5991         }
5992
5993         rc = sbuf_finish(sb);
5994         sbuf_delete(sb);
5995
5996         return (rc);
5997 }
5998
5999 static int
6000 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
6001 {
6002         int rc = 0;
6003         struct port_info *pi = arg1;
6004         struct sbuf *sb;
6005         static const char *linkdnreasons[] = {
6006                 "non-specific", "remote fault", "autoneg failed", "reserved3",
6007                 "PHY overheated", "unknown", "rx los", "reserved7"
6008         };
6009
6010         rc = sysctl_wire_old_buffer(req, 0);
6011         if (rc != 0)
6012                 return(rc);
6013         sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
6014         if (sb == NULL)
6015                 return (ENOMEM);
6016
6017         if (pi->linkdnrc < 0)
6018                 sbuf_printf(sb, "n/a");
6019         else if (pi->linkdnrc < nitems(linkdnreasons))
6020                 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
6021         else
6022                 sbuf_printf(sb, "%d", pi->linkdnrc);
6023
6024         rc = sbuf_finish(sb);
6025         sbuf_delete(sb);
6026
6027         return (rc);
6028 }
6029
6030 struct mem_desc {
6031         unsigned int base;
6032         unsigned int limit;
6033         unsigned int idx;
6034 };
6035
6036 static int
6037 mem_desc_cmp(const void *a, const void *b)
6038 {
6039         return ((const struct mem_desc *)a)->base -
6040                ((const struct mem_desc *)b)->base;
6041 }
6042
6043 static void
6044 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
6045     unsigned int to)
6046 {
6047         unsigned int size;
6048
6049         size = to - from + 1;
6050         if (size == 0)
6051                 return;
6052
6053         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
6054         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
6055 }
6056
6057 static int
6058 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
6059 {
6060         struct adapter *sc = arg1;
6061         struct sbuf *sb;
6062         int rc, i, n;
6063         uint32_t lo, hi, used, alloc;
6064         static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
6065         static const char *region[] = {
6066                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
6067                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
6068                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
6069                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
6070                 "RQUDP region:", "PBL region:", "TXPBL region:",
6071                 "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
6072                 "On-chip queues:"
6073         };
6074         struct mem_desc avail[4];
6075         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
6076         struct mem_desc *md = mem;
6077
6078         rc = sysctl_wire_old_buffer(req, 0);
6079         if (rc != 0)
6080                 return (rc);
6081
6082         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6083         if (sb == NULL)
6084                 return (ENOMEM);
6085
6086         for (i = 0; i < nitems(mem); i++) {
6087                 mem[i].limit = 0;
6088                 mem[i].idx = i;
6089         }
6090
6091         /* Find and sort the populated memory ranges */
6092         i = 0;
6093         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
6094         if (lo & F_EDRAM0_ENABLE) {
6095                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
6096                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
6097                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
6098                 avail[i].idx = 0;
6099                 i++;
6100         }
6101         if (lo & F_EDRAM1_ENABLE) {
6102                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
6103                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
6104                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
6105                 avail[i].idx = 1;
6106                 i++;
6107         }
6108         if (lo & F_EXT_MEM_ENABLE) {
6109                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
6110                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
6111                 avail[i].limit = avail[i].base +
6112                     (G_EXT_MEM_SIZE(hi) << 20);
6113                 avail[i].idx = is_t4(sc) ? 2 : 3;       /* Call it MC for T4 */
6114                 i++;
6115         }
6116         if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
6117                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
6118                 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
6119                 avail[i].limit = avail[i].base +
6120                     (G_EXT_MEM1_SIZE(hi) << 20);
6121                 avail[i].idx = 4;
6122                 i++;
6123         }
6124         if (!i)                                    /* no memory available */
6125                 return 0;
6126         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
6127
6128         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
6129         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
6130         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
6131         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
6132         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
6133         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
6134         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
6135         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
6136         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
6137
6138         /* the next few have explicit upper bounds */
6139         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
6140         md->limit = md->base - 1 +
6141                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
6142                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
6143         md++;
6144
6145         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
6146         md->limit = md->base - 1 +
6147                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
6148                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
6149         md++;
6150
6151         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6152                 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
6153                 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
6154                 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
6155         } else {
6156                 md->base = 0;
6157                 md->idx = nitems(region);  /* hide it */
6158         }
6159         md++;
6160
6161 #define ulp_region(reg) \
6162         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
6163         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
6164
6165         ulp_region(RX_ISCSI);
6166         ulp_region(RX_TDDP);
6167         ulp_region(TX_TPT);
6168         ulp_region(RX_STAG);
6169         ulp_region(RX_RQ);
6170         ulp_region(RX_RQUDP);
6171         ulp_region(RX_PBL);
6172         ulp_region(TX_PBL);
6173 #undef ulp_region
6174
6175         md->base = 0;
6176         md->idx = nitems(region);
6177         if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
6178                 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
6179                 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
6180                     A_SGE_DBVFIFO_SIZE))) << 2) - 1;
6181         }
6182         md++;
6183
6184         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
6185         md->limit = md->base + sc->tids.ntids - 1;
6186         md++;
6187         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
6188         md->limit = md->base + sc->tids.ntids - 1;
6189         md++;
6190
6191         md->base = sc->vres.ocq.start;
6192         if (sc->vres.ocq.size)
6193                 md->limit = md->base + sc->vres.ocq.size - 1;
6194         else
6195                 md->idx = nitems(region);  /* hide it */
6196         md++;
6197
6198         /* add any address-space holes, there can be up to 3 */
6199         for (n = 0; n < i - 1; n++)
6200                 if (avail[n].limit < avail[n + 1].base)
6201                         (md++)->base = avail[n].limit;
6202         if (avail[n].limit)
6203                 (md++)->base = avail[n].limit;
6204
6205         n = md - mem;
6206         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
6207
6208         for (lo = 0; lo < i; lo++)
6209                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
6210                                 avail[lo].limit - 1);
6211
6212         sbuf_printf(sb, "\n");
6213         for (i = 0; i < n; i++) {
6214                 if (mem[i].idx >= nitems(region))
6215                         continue;                        /* skip holes */
6216                 if (!mem[i].limit)
6217                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
6218                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
6219                                 mem[i].limit);
6220         }
6221
6222         sbuf_printf(sb, "\n");
6223         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
6224         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
6225         mem_region_show(sb, "uP RAM:", lo, hi);
6226
6227         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
6228         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
6229         mem_region_show(sb, "uP Extmem2:", lo, hi);
6230
6231         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
6232         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
6233                    G_PMRXMAXPAGE(lo),
6234                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
6235                    (lo & F_PMRXNUMCHN) ? 2 : 1);
6236
6237         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
6238         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
6239         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
6240                    G_PMTXMAXPAGE(lo),
6241                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
6242                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
6243         sbuf_printf(sb, "%u p-structs\n",
6244                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
6245
6246         for (i = 0; i < 4; i++) {
6247                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
6248                 if (is_t4(sc)) {
6249                         used = G_USED(lo);
6250                         alloc = G_ALLOC(lo);
6251                 } else {
6252                         used = G_T5_USED(lo);
6253                         alloc = G_T5_ALLOC(lo);
6254                 }
6255                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
6256                            i, used, alloc);
6257         }
6258         for (i = 0; i < 4; i++) {
6259                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
6260                 if (is_t4(sc)) {
6261                         used = G_USED(lo);
6262                         alloc = G_ALLOC(lo);
6263                 } else {
6264                         used = G_T5_USED(lo);
6265                         alloc = G_T5_ALLOC(lo);
6266                 }
6267                 sbuf_printf(sb,
6268                            "\nLoopback %d using %u pages out of %u allocated",
6269                            i, used, alloc);
6270         }
6271
6272         rc = sbuf_finish(sb);
6273         sbuf_delete(sb);
6274
6275         return (rc);
6276 }
6277
6278 static inline void
6279 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
6280 {
6281         *mask = x | y;
6282         y = htobe64(y);
6283         memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
6284 }
6285
6286 static int
6287 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
6288 {
6289         struct adapter *sc = arg1;
6290         struct sbuf *sb;
6291         int rc, i, n;
6292
6293         rc = sysctl_wire_old_buffer(req, 0);
6294         if (rc != 0)
6295                 return (rc);
6296
6297         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6298         if (sb == NULL)
6299                 return (ENOMEM);
6300
6301         sbuf_printf(sb,
6302             "Idx  Ethernet address     Mask     Vld Ports PF"
6303             "  VF              Replication             P0 P1 P2 P3  ML");
6304         n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
6305             NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
6306         for (i = 0; i < n; i++) {
6307                 uint64_t tcamx, tcamy, mask;
6308                 uint32_t cls_lo, cls_hi;
6309                 uint8_t addr[ETHER_ADDR_LEN];
6310
6311                 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
6312                 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
6313                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
6314                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
6315
6316                 if (tcamx & tcamy)
6317                         continue;
6318
6319                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
6320                 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
6321                            "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
6322                            addr[3], addr[4], addr[5], (uintmax_t)mask,
6323                            (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
6324                            G_PORTMAP(cls_hi), G_PF(cls_lo),
6325                            (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
6326
6327                 if (cls_lo & F_REPLICATE) {
6328                         struct fw_ldst_cmd ldst_cmd;
6329
6330                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
6331                         ldst_cmd.op_to_addrspace =
6332                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
6333                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
6334                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
6335                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
6336                         ldst_cmd.u.mps.rplc.fid_idx =
6337                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
6338                                 V_FW_LDST_CMD_IDX(i));
6339
6340                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
6341                             "t4mps");
6342                         if (rc)
6343                                 break;
6344                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
6345                             sizeof(ldst_cmd), &ldst_cmd);
6346                         end_synchronized_op(sc, 0);
6347
6348                         if (rc != 0) {
6349                                 sbuf_printf(sb,
6350                                     " ------------ error %3u ------------", rc);
6351                                 rc = 0;
6352                         } else {
6353                                 sbuf_printf(sb, " %08x %08x %08x %08x",
6354                                     be32toh(ldst_cmd.u.mps.rplc.rplc127_96),
6355                                     be32toh(ldst_cmd.u.mps.rplc.rplc95_64),
6356                                     be32toh(ldst_cmd.u.mps.rplc.rplc63_32),
6357                                     be32toh(ldst_cmd.u.mps.rplc.rplc31_0));
6358                         }
6359                 } else
6360                         sbuf_printf(sb, "%36s", "");
6361
6362                 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
6363                     G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
6364                     G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
6365         }
6366
6367         if (rc)
6368                 (void) sbuf_finish(sb);
6369         else
6370                 rc = sbuf_finish(sb);
6371         sbuf_delete(sb);
6372
6373         return (rc);
6374 }
6375
6376 static int
6377 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
6378 {
6379         struct adapter *sc = arg1;
6380         struct sbuf *sb;
6381         int rc;
6382         uint16_t mtus[NMTUS];
6383
6384         rc = sysctl_wire_old_buffer(req, 0);
6385         if (rc != 0)
6386                 return (rc);
6387
6388         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6389         if (sb == NULL)
6390                 return (ENOMEM);
6391
6392         t4_read_mtu_tbl(sc, mtus, NULL);
6393
6394         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
6395             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
6396             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
6397             mtus[14], mtus[15]);
6398
6399         rc = sbuf_finish(sb);
6400         sbuf_delete(sb);
6401
6402         return (rc);
6403 }
6404
6405 static int
6406 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
6407 {
6408         struct adapter *sc = arg1;
6409         struct sbuf *sb;
6410         int rc, i;
6411         uint32_t cnt[PM_NSTATS];
6412         uint64_t cyc[PM_NSTATS];
6413         static const char *rx_stats[] = {
6414                 "Read:", "Write bypass:", "Write mem:", "Flush:"
6415         };
6416         static const char *tx_stats[] = {
6417                 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
6418         };
6419
6420         rc = sysctl_wire_old_buffer(req, 0);
6421         if (rc != 0)
6422                 return (rc);
6423
6424         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6425         if (sb == NULL)
6426                 return (ENOMEM);
6427
6428         t4_pmtx_get_stats(sc, cnt, cyc);
6429         sbuf_printf(sb, "                Tx pcmds             Tx bytes");
6430         for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
6431                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
6432                     cyc[i]);
6433
6434         t4_pmrx_get_stats(sc, cnt, cyc);
6435         sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
6436         for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
6437                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
6438                     cyc[i]);
6439
6440         rc = sbuf_finish(sb);
6441         sbuf_delete(sb);
6442
6443         return (rc);
6444 }
6445
6446 static int
6447 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
6448 {
6449         struct adapter *sc = arg1;
6450         struct sbuf *sb;
6451         int rc;
6452         struct tp_rdma_stats stats;
6453
6454         rc = sysctl_wire_old_buffer(req, 0);
6455         if (rc != 0)
6456                 return (rc);
6457
6458         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6459         if (sb == NULL)
6460                 return (ENOMEM);
6461
6462         t4_tp_get_rdma_stats(sc, &stats);
6463         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
6464         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
6465
6466         rc = sbuf_finish(sb);
6467         sbuf_delete(sb);
6468
6469         return (rc);
6470 }
6471
6472 static int
6473 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
6474 {
6475         struct adapter *sc = arg1;
6476         struct sbuf *sb;
6477         int rc;
6478         struct tp_tcp_stats v4, v6;
6479
6480         rc = sysctl_wire_old_buffer(req, 0);
6481         if (rc != 0)
6482                 return (rc);
6483
6484         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6485         if (sb == NULL)
6486                 return (ENOMEM);
6487
6488         t4_tp_get_tcp_stats(sc, &v4, &v6);
6489         sbuf_printf(sb,
6490             "                                IP                 IPv6\n");
6491         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
6492             v4.tcpOutRsts, v6.tcpOutRsts);
6493         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
6494             v4.tcpInSegs, v6.tcpInSegs);
6495         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
6496             v4.tcpOutSegs, v6.tcpOutSegs);
6497         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
6498             v4.tcpRetransSegs, v6.tcpRetransSegs);
6499
6500         rc = sbuf_finish(sb);
6501         sbuf_delete(sb);
6502
6503         return (rc);
6504 }
6505
6506 static int
6507 sysctl_tids(SYSCTL_HANDLER_ARGS)
6508 {
6509         struct adapter *sc = arg1;
6510         struct sbuf *sb;
6511         int rc;
6512         struct tid_info *t = &sc->tids;
6513
6514         rc = sysctl_wire_old_buffer(req, 0);
6515         if (rc != 0)
6516                 return (rc);
6517
6518         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6519         if (sb == NULL)
6520                 return (ENOMEM);
6521
6522         if (t->natids) {
6523                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
6524                     t->atids_in_use);
6525         }
6526
6527         if (t->ntids) {
6528                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6529                         uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
6530
6531                         if (b) {
6532                                 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
6533                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6534                                     t->ntids - 1);
6535                         } else {
6536                                 sbuf_printf(sb, "TID range: %u-%u",
6537                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6538                                     t->ntids - 1);
6539                         }
6540                 } else
6541                         sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
6542                 sbuf_printf(sb, ", in use: %u\n",
6543                     atomic_load_acq_int(&t->tids_in_use));
6544         }
6545
6546         if (t->nstids) {
6547                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
6548                     t->stid_base + t->nstids - 1, t->stids_in_use);
6549         }
6550
6551         if (t->nftids) {
6552                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
6553                     t->ftid_base + t->nftids - 1);
6554         }
6555
6556         if (t->netids) {
6557                 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
6558                     t->etid_base + t->netids - 1);
6559         }
6560
6561         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
6562             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
6563             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
6564
6565         rc = sbuf_finish(sb);
6566         sbuf_delete(sb);
6567
6568         return (rc);
6569 }
6570
6571 static int
6572 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
6573 {
6574         struct adapter *sc = arg1;
6575         struct sbuf *sb;
6576         int rc;
6577         struct tp_err_stats stats;
6578
6579         rc = sysctl_wire_old_buffer(req, 0);
6580         if (rc != 0)
6581                 return (rc);
6582
6583         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6584         if (sb == NULL)
6585                 return (ENOMEM);
6586
6587         t4_tp_get_err_stats(sc, &stats);
6588
6589         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
6590                       "channel 3\n");
6591         sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
6592             stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
6593             stats.macInErrs[3]);
6594         sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
6595             stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
6596             stats.hdrInErrs[3]);
6597         sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
6598             stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
6599             stats.tcpInErrs[3]);
6600         sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
6601             stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
6602             stats.tcp6InErrs[3]);
6603         sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
6604             stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
6605             stats.tnlCongDrops[3]);
6606         sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
6607             stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
6608             stats.tnlTxDrops[3]);
6609         sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
6610             stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
6611             stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
6612         sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
6613             stats.ofldChanDrops[0], stats.ofldChanDrops[1],
6614             stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
6615         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
6616             stats.ofldNoNeigh, stats.ofldCongDefer);
6617
6618         rc = sbuf_finish(sb);
6619         sbuf_delete(sb);
6620
6621         return (rc);
6622 }
6623
6624 struct field_desc {
6625         const char *name;
6626         u_int start;
6627         u_int width;
6628 };
6629
6630 static void
6631 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
6632 {
6633         char buf[32];
6634         int line_size = 0;
6635
6636         while (f->name) {
6637                 uint64_t mask = (1ULL << f->width) - 1;
6638                 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
6639                     ((uintmax_t)v >> f->start) & mask);
6640
6641                 if (line_size + len >= 79) {
6642                         line_size = 8;
6643                         sbuf_printf(sb, "\n        ");
6644                 }
6645                 sbuf_printf(sb, "%s ", buf);
6646                 line_size += len + 1;
6647                 f++;
6648         }
6649         sbuf_printf(sb, "\n");
6650 }
6651
6652 static struct field_desc tp_la0[] = {
6653         { "RcfOpCodeOut", 60, 4 },
6654         { "State", 56, 4 },
6655         { "WcfState", 52, 4 },
6656         { "RcfOpcSrcOut", 50, 2 },
6657         { "CRxError", 49, 1 },
6658         { "ERxError", 48, 1 },
6659         { "SanityFailed", 47, 1 },
6660         { "SpuriousMsg", 46, 1 },
6661         { "FlushInputMsg", 45, 1 },
6662         { "FlushInputCpl", 44, 1 },
6663         { "RssUpBit", 43, 1 },
6664         { "RssFilterHit", 42, 1 },
6665         { "Tid", 32, 10 },
6666         { "InitTcb", 31, 1 },
6667         { "LineNumber", 24, 7 },
6668         { "Emsg", 23, 1 },
6669         { "EdataOut", 22, 1 },
6670         { "Cmsg", 21, 1 },
6671         { "CdataOut", 20, 1 },
6672         { "EreadPdu", 19, 1 },
6673         { "CreadPdu", 18, 1 },
6674         { "TunnelPkt", 17, 1 },
6675         { "RcfPeerFin", 16, 1 },
6676         { "RcfReasonOut", 12, 4 },
6677         { "TxCchannel", 10, 2 },
6678         { "RcfTxChannel", 8, 2 },
6679         { "RxEchannel", 6, 2 },
6680         { "RcfRxChannel", 5, 1 },
6681         { "RcfDataOutSrdy", 4, 1 },
6682         { "RxDvld", 3, 1 },
6683         { "RxOoDvld", 2, 1 },
6684         { "RxCongestion", 1, 1 },
6685         { "TxCongestion", 0, 1 },
6686         { NULL }
6687 };
6688
6689 static struct field_desc tp_la1[] = {
6690         { "CplCmdIn", 56, 8 },
6691         { "CplCmdOut", 48, 8 },
6692         { "ESynOut", 47, 1 },
6693         { "EAckOut", 46, 1 },
6694         { "EFinOut", 45, 1 },
6695         { "ERstOut", 44, 1 },
6696         { "SynIn", 43, 1 },
6697         { "AckIn", 42, 1 },
6698         { "FinIn", 41, 1 },
6699         { "RstIn", 40, 1 },
6700         { "DataIn", 39, 1 },
6701         { "DataInVld", 38, 1 },
6702         { "PadIn", 37, 1 },
6703         { "RxBufEmpty", 36, 1 },
6704         { "RxDdp", 35, 1 },
6705         { "RxFbCongestion", 34, 1 },
6706         { "TxFbCongestion", 33, 1 },
6707         { "TxPktSumSrdy", 32, 1 },
6708         { "RcfUlpType", 28, 4 },
6709         { "Eread", 27, 1 },
6710         { "Ebypass", 26, 1 },
6711         { "Esave", 25, 1 },
6712         { "Static0", 24, 1 },
6713         { "Cread", 23, 1 },
6714         { "Cbypass", 22, 1 },
6715         { "Csave", 21, 1 },
6716         { "CPktOut", 20, 1 },
6717         { "RxPagePoolFull", 18, 2 },
6718         { "RxLpbkPkt", 17, 1 },
6719         { "TxLpbkPkt", 16, 1 },
6720         { "RxVfValid", 15, 1 },
6721         { "SynLearned", 14, 1 },
6722         { "SetDelEntry", 13, 1 },
6723         { "SetInvEntry", 12, 1 },
6724         { "CpcmdDvld", 11, 1 },
6725         { "CpcmdSave", 10, 1 },
6726         { "RxPstructsFull", 8, 2 },
6727         { "EpcmdDvld", 7, 1 },
6728         { "EpcmdFlush", 6, 1 },
6729         { "EpcmdTrimPrefix", 5, 1 },
6730         { "EpcmdTrimPostfix", 4, 1 },
6731         { "ERssIp4Pkt", 3, 1 },
6732         { "ERssIp6Pkt", 2, 1 },
6733         { "ERssTcpUdpPkt", 1, 1 },
6734         { "ERssFceFipPkt", 0, 1 },
6735         { NULL }
6736 };
6737
6738 static struct field_desc tp_la2[] = {
6739         { "CplCmdIn", 56, 8 },
6740         { "MpsVfVld", 55, 1 },
6741         { "MpsPf", 52, 3 },
6742         { "MpsVf", 44, 8 },
6743         { "SynIn", 43, 1 },
6744         { "AckIn", 42, 1 },
6745         { "FinIn", 41, 1 },
6746         { "RstIn", 40, 1 },
6747         { "DataIn", 39, 1 },
6748         { "DataInVld", 38, 1 },
6749         { "PadIn", 37, 1 },
6750         { "RxBufEmpty", 36, 1 },
6751         { "RxDdp", 35, 1 },
6752         { "RxFbCongestion", 34, 1 },
6753         { "TxFbCongestion", 33, 1 },
6754         { "TxPktSumSrdy", 32, 1 },
6755         { "RcfUlpType", 28, 4 },
6756         { "Eread", 27, 1 },
6757         { "Ebypass", 26, 1 },
6758         { "Esave", 25, 1 },
6759         { "Static0", 24, 1 },
6760         { "Cread", 23, 1 },
6761         { "Cbypass", 22, 1 },
6762         { "Csave", 21, 1 },
6763         { "CPktOut", 20, 1 },
6764         { "RxPagePoolFull", 18, 2 },
6765         { "RxLpbkPkt", 17, 1 },
6766         { "TxLpbkPkt", 16, 1 },
6767         { "RxVfValid", 15, 1 },
6768         { "SynLearned", 14, 1 },
6769         { "SetDelEntry", 13, 1 },
6770         { "SetInvEntry", 12, 1 },
6771         { "CpcmdDvld", 11, 1 },
6772         { "CpcmdSave", 10, 1 },
6773         { "RxPstructsFull", 8, 2 },
6774         { "EpcmdDvld", 7, 1 },
6775         { "EpcmdFlush", 6, 1 },
6776         { "EpcmdTrimPrefix", 5, 1 },
6777         { "EpcmdTrimPostfix", 4, 1 },
6778         { "ERssIp4Pkt", 3, 1 },
6779         { "ERssIp6Pkt", 2, 1 },
6780         { "ERssTcpUdpPkt", 1, 1 },
6781         { "ERssFceFipPkt", 0, 1 },
6782         { NULL }
6783 };
6784
6785 static void
6786 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
6787 {
6788
6789         field_desc_show(sb, *p, tp_la0);
6790 }
6791
6792 static void
6793 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
6794 {
6795
6796         if (idx)
6797                 sbuf_printf(sb, "\n");
6798         field_desc_show(sb, p[0], tp_la0);
6799         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6800                 field_desc_show(sb, p[1], tp_la0);
6801 }
6802
6803 static void
6804 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
6805 {
6806
6807         if (idx)
6808                 sbuf_printf(sb, "\n");
6809         field_desc_show(sb, p[0], tp_la0);
6810         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6811                 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
6812 }
6813
6814 static int
6815 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
6816 {
6817         struct adapter *sc = arg1;
6818         struct sbuf *sb;
6819         uint64_t *buf, *p;
6820         int rc;
6821         u_int i, inc;
6822         void (*show_func)(struct sbuf *, uint64_t *, int);
6823
6824         rc = sysctl_wire_old_buffer(req, 0);
6825         if (rc != 0)
6826                 return (rc);
6827
6828         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6829         if (sb == NULL)
6830                 return (ENOMEM);
6831
6832         buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
6833
6834         t4_tp_read_la(sc, buf, NULL);
6835         p = buf;
6836
6837         switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
6838         case 2:
6839                 inc = 2;
6840                 show_func = tp_la_show2;
6841                 break;
6842         case 3:
6843                 inc = 2;
6844                 show_func = tp_la_show3;
6845                 break;
6846         default:
6847                 inc = 1;
6848                 show_func = tp_la_show;
6849         }
6850
6851         for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
6852                 (*show_func)(sb, p, i);
6853
6854         rc = sbuf_finish(sb);
6855         sbuf_delete(sb);
6856         free(buf, M_CXGBE);
6857         return (rc);
6858 }
6859
6860 static int
6861 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
6862 {
6863         struct adapter *sc = arg1;
6864         struct sbuf *sb;
6865         int rc;
6866         u64 nrate[NCHAN], orate[NCHAN];
6867
6868         rc = sysctl_wire_old_buffer(req, 0);
6869         if (rc != 0)
6870                 return (rc);
6871
6872         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6873         if (sb == NULL)
6874                 return (ENOMEM);
6875
6876         t4_get_chan_txrate(sc, nrate, orate);
6877         sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
6878                  "channel 3\n");
6879         sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
6880             nrate[0], nrate[1], nrate[2], nrate[3]);
6881         sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
6882             orate[0], orate[1], orate[2], orate[3]);
6883
6884         rc = sbuf_finish(sb);
6885         sbuf_delete(sb);
6886
6887         return (rc);
6888 }
6889
6890 static int
6891 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
6892 {
6893         struct adapter *sc = arg1;
6894         struct sbuf *sb;
6895         uint32_t *buf, *p;
6896         int rc, i;
6897
6898         rc = sysctl_wire_old_buffer(req, 0);
6899         if (rc != 0)
6900                 return (rc);
6901
6902         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6903         if (sb == NULL)
6904                 return (ENOMEM);
6905
6906         buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
6907             M_ZERO | M_WAITOK);
6908
6909         t4_ulprx_read_la(sc, buf);
6910         p = buf;
6911
6912         sbuf_printf(sb, "      Pcmd        Type   Message"
6913             "                Data");
6914         for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
6915                 sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
6916                     p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
6917         }
6918
6919         rc = sbuf_finish(sb);
6920         sbuf_delete(sb);
6921         free(buf, M_CXGBE);
6922         return (rc);
6923 }
6924
6925 static int
6926 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
6927 {
6928         struct adapter *sc = arg1;
6929         struct sbuf *sb;
6930         int rc, v;
6931
6932         rc = sysctl_wire_old_buffer(req, 0);
6933         if (rc != 0)
6934                 return (rc);
6935
6936         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6937         if (sb == NULL)
6938                 return (ENOMEM);
6939
6940         v = t4_read_reg(sc, A_SGE_STAT_CFG);
6941         if (G_STATSOURCE_T5(v) == 7) {
6942                 if (G_STATMODE(v) == 0) {
6943                         sbuf_printf(sb, "total %d, incomplete %d",
6944                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6945                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6946                 } else if (G_STATMODE(v) == 1) {
6947                         sbuf_printf(sb, "total %d, data overflow %d",
6948                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6949                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6950                 }
6951         }
6952         rc = sbuf_finish(sb);
6953         sbuf_delete(sb);
6954
6955         return (rc);
6956 }
6957 #endif
6958
6959 static uint32_t
6960 fconf_to_mode(uint32_t fconf)
6961 {
6962         uint32_t mode;
6963
6964         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
6965             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
6966
6967         if (fconf & F_FRAGMENTATION)
6968                 mode |= T4_FILTER_IP_FRAGMENT;
6969
6970         if (fconf & F_MPSHITTYPE)
6971                 mode |= T4_FILTER_MPS_HIT_TYPE;
6972
6973         if (fconf & F_MACMATCH)
6974                 mode |= T4_FILTER_MAC_IDX;
6975
6976         if (fconf & F_ETHERTYPE)
6977                 mode |= T4_FILTER_ETH_TYPE;
6978
6979         if (fconf & F_PROTOCOL)
6980                 mode |= T4_FILTER_IP_PROTO;
6981
6982         if (fconf & F_TOS)
6983                 mode |= T4_FILTER_IP_TOS;
6984
6985         if (fconf & F_VLAN)
6986                 mode |= T4_FILTER_VLAN;
6987
6988         if (fconf & F_VNIC_ID)
6989                 mode |= T4_FILTER_VNIC;
6990
6991         if (fconf & F_PORT)
6992                 mode |= T4_FILTER_PORT;
6993
6994         if (fconf & F_FCOE)
6995                 mode |= T4_FILTER_FCoE;
6996
6997         return (mode);
6998 }
6999
7000 static uint32_t
7001 mode_to_fconf(uint32_t mode)
7002 {
7003         uint32_t fconf = 0;
7004
7005         if (mode & T4_FILTER_IP_FRAGMENT)
7006                 fconf |= F_FRAGMENTATION;
7007
7008         if (mode & T4_FILTER_MPS_HIT_TYPE)
7009                 fconf |= F_MPSHITTYPE;
7010
7011         if (mode & T4_FILTER_MAC_IDX)
7012                 fconf |= F_MACMATCH;
7013
7014         if (mode & T4_FILTER_ETH_TYPE)
7015                 fconf |= F_ETHERTYPE;
7016
7017         if (mode & T4_FILTER_IP_PROTO)
7018                 fconf |= F_PROTOCOL;
7019
7020         if (mode & T4_FILTER_IP_TOS)
7021                 fconf |= F_TOS;
7022
7023         if (mode & T4_FILTER_VLAN)
7024                 fconf |= F_VLAN;
7025
7026         if (mode & T4_FILTER_VNIC)
7027                 fconf |= F_VNIC_ID;
7028
7029         if (mode & T4_FILTER_PORT)
7030                 fconf |= F_PORT;
7031
7032         if (mode & T4_FILTER_FCoE)
7033                 fconf |= F_FCOE;
7034
7035         return (fconf);
7036 }
7037
7038 static uint32_t
7039 fspec_to_fconf(struct t4_filter_specification *fs)
7040 {
7041         uint32_t fconf = 0;
7042
7043         if (fs->val.frag || fs->mask.frag)
7044                 fconf |= F_FRAGMENTATION;
7045
7046         if (fs->val.matchtype || fs->mask.matchtype)
7047                 fconf |= F_MPSHITTYPE;
7048
7049         if (fs->val.macidx || fs->mask.macidx)
7050                 fconf |= F_MACMATCH;
7051
7052         if (fs->val.ethtype || fs->mask.ethtype)
7053                 fconf |= F_ETHERTYPE;
7054
7055         if (fs->val.proto || fs->mask.proto)
7056                 fconf |= F_PROTOCOL;
7057
7058         if (fs->val.tos || fs->mask.tos)
7059                 fconf |= F_TOS;
7060
7061         if (fs->val.vlan_vld || fs->mask.vlan_vld)
7062                 fconf |= F_VLAN;
7063
7064         if (fs->val.vnic_vld || fs->mask.vnic_vld)
7065                 fconf |= F_VNIC_ID;
7066
7067         if (fs->val.iport || fs->mask.iport)
7068                 fconf |= F_PORT;
7069
7070         if (fs->val.fcoe || fs->mask.fcoe)
7071                 fconf |= F_FCOE;
7072
7073         return (fconf);
7074 }
7075
7076 static int
7077 get_filter_mode(struct adapter *sc, uint32_t *mode)
7078 {
7079         int rc;
7080         uint32_t fconf;
7081
7082         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7083             "t4getfm");
7084         if (rc)
7085                 return (rc);
7086
7087         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
7088             A_TP_VLAN_PRI_MAP);
7089
7090         if (sc->params.tp.vlan_pri_map != fconf) {
7091                 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
7092                     device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
7093                     fconf);
7094         }
7095
7096         *mode = fconf_to_mode(fconf);
7097
7098         end_synchronized_op(sc, LOCK_HELD);
7099         return (0);
7100 }
7101
7102 static int
7103 set_filter_mode(struct adapter *sc, uint32_t mode)
7104 {
7105         uint32_t fconf;
7106         int rc;
7107
7108         fconf = mode_to_fconf(mode);
7109
7110         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7111             "t4setfm");
7112         if (rc)
7113                 return (rc);
7114
7115         if (sc->tids.ftids_in_use > 0) {
7116                 rc = EBUSY;
7117                 goto done;
7118         }
7119
7120 #ifdef TCP_OFFLOAD
7121         if (uld_active(sc, ULD_TOM)) {
7122                 rc = EBUSY;
7123                 goto done;
7124         }
7125 #endif
7126
7127         rc = -t4_set_filter_mode(sc, fconf);
7128 done:
7129         end_synchronized_op(sc, LOCK_HELD);
7130         return (rc);
7131 }
7132
7133 static inline uint64_t
7134 get_filter_hits(struct adapter *sc, uint32_t fid)
7135 {
7136         uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
7137         uint64_t hits;
7138
7139         memwin_info(sc, 0, &mw_base, NULL);
7140         off = position_memwin(sc, 0,
7141             tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
7142         if (is_t4(sc)) {
7143                 hits = t4_read_reg64(sc, mw_base + off + 16);
7144                 hits = be64toh(hits);
7145         } else {
7146                 hits = t4_read_reg(sc, mw_base + off + 24);
7147                 hits = be32toh(hits);
7148         }
7149
7150         return (hits);
7151 }
7152
7153 static int
7154 get_filter(struct adapter *sc, struct t4_filter *t)
7155 {
7156         int i, rc, nfilters = sc->tids.nftids;
7157         struct filter_entry *f;
7158
7159         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7160             "t4getf");
7161         if (rc)
7162                 return (rc);
7163
7164         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
7165             t->idx >= nfilters) {
7166                 t->idx = 0xffffffff;
7167                 goto done;
7168         }
7169
7170         f = &sc->tids.ftid_tab[t->idx];
7171         for (i = t->idx; i < nfilters; i++, f++) {
7172                 if (f->valid) {
7173                         t->idx = i;
7174                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
7175                         t->smtidx = f->smtidx;
7176                         if (f->fs.hitcnts)
7177                                 t->hits = get_filter_hits(sc, t->idx);
7178                         else
7179                                 t->hits = UINT64_MAX;
7180                         t->fs = f->fs;
7181
7182                         goto done;
7183                 }
7184         }
7185
7186         t->idx = 0xffffffff;
7187 done:
7188         end_synchronized_op(sc, LOCK_HELD);
7189         return (0);
7190 }
7191
7192 static int
7193 set_filter(struct adapter *sc, struct t4_filter *t)
7194 {
7195         unsigned int nfilters, nports;
7196         struct filter_entry *f;
7197         int i, rc;
7198
7199         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
7200         if (rc)
7201                 return (rc);
7202
7203         nfilters = sc->tids.nftids;
7204         nports = sc->params.nports;
7205
7206         if (nfilters == 0) {
7207                 rc = ENOTSUP;
7208                 goto done;
7209         }
7210
7211         if (!(sc->flags & FULL_INIT_DONE)) {
7212                 rc = EAGAIN;
7213                 goto done;
7214         }
7215
7216         if (t->idx >= nfilters) {
7217                 rc = EINVAL;
7218                 goto done;
7219         }
7220
7221         /* Validate against the global filter mode */
7222         if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
7223             sc->params.tp.vlan_pri_map) {
7224                 rc = E2BIG;
7225                 goto done;
7226         }
7227
7228         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
7229                 rc = EINVAL;
7230                 goto done;
7231         }
7232
7233         if (t->fs.val.iport >= nports) {
7234                 rc = EINVAL;
7235                 goto done;
7236         }
7237
7238         /* Can't specify an iq if not steering to it */
7239         if (!t->fs.dirsteer && t->fs.iq) {
7240                 rc = EINVAL;
7241                 goto done;
7242         }
7243
7244         /* IPv6 filter idx must be 4 aligned */
7245         if (t->fs.type == 1 &&
7246             ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
7247                 rc = EINVAL;
7248                 goto done;
7249         }
7250
7251         if (sc->tids.ftid_tab == NULL) {
7252                 KASSERT(sc->tids.ftids_in_use == 0,
7253                     ("%s: no memory allocated but filters_in_use > 0",
7254                     __func__));
7255
7256                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
7257                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
7258                 if (sc->tids.ftid_tab == NULL) {
7259                         rc = ENOMEM;
7260                         goto done;
7261                 }
7262                 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
7263         }
7264
7265         for (i = 0; i < 4; i++) {
7266                 f = &sc->tids.ftid_tab[t->idx + i];
7267
7268                 if (f->pending || f->valid) {
7269                         rc = EBUSY;
7270                         goto done;
7271                 }
7272                 if (f->locked) {
7273                         rc = EPERM;
7274                         goto done;
7275                 }
7276
7277                 if (t->fs.type == 0)
7278                         break;
7279         }
7280
7281         f = &sc->tids.ftid_tab[t->idx];
7282         f->fs = t->fs;
7283
7284         rc = set_filter_wr(sc, t->idx);
7285 done:
7286         end_synchronized_op(sc, 0);
7287
7288         if (rc == 0) {
7289                 mtx_lock(&sc->tids.ftid_lock);
7290                 for (;;) {
7291                         if (f->pending == 0) {
7292                                 rc = f->valid ? 0 : EIO;
7293                                 break;
7294                         }
7295
7296                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7297                             PCATCH, "t4setfw", 0)) {
7298                                 rc = EINPROGRESS;
7299                                 break;
7300                         }
7301                 }
7302                 mtx_unlock(&sc->tids.ftid_lock);
7303         }
7304         return (rc);
7305 }
7306
7307 static int
7308 del_filter(struct adapter *sc, struct t4_filter *t)
7309 {
7310         unsigned int nfilters;
7311         struct filter_entry *f;
7312         int rc;
7313
7314         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
7315         if (rc)
7316                 return (rc);
7317
7318         nfilters = sc->tids.nftids;
7319
7320         if (nfilters == 0) {
7321                 rc = ENOTSUP;
7322                 goto done;
7323         }
7324
7325         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
7326             t->idx >= nfilters) {
7327                 rc = EINVAL;
7328                 goto done;
7329         }
7330
7331         if (!(sc->flags & FULL_INIT_DONE)) {
7332                 rc = EAGAIN;
7333                 goto done;
7334         }
7335
7336         f = &sc->tids.ftid_tab[t->idx];
7337
7338         if (f->pending) {
7339                 rc = EBUSY;
7340                 goto done;
7341         }
7342         if (f->locked) {
7343                 rc = EPERM;
7344                 goto done;
7345         }
7346
7347         if (f->valid) {
7348                 t->fs = f->fs;  /* extra info for the caller */
7349                 rc = del_filter_wr(sc, t->idx);
7350         }
7351
7352 done:
7353         end_synchronized_op(sc, 0);
7354
7355         if (rc == 0) {
7356                 mtx_lock(&sc->tids.ftid_lock);
7357                 for (;;) {
7358                         if (f->pending == 0) {
7359                                 rc = f->valid ? EIO : 0;
7360                                 break;
7361                         }
7362
7363                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7364                             PCATCH, "t4delfw", 0)) {
7365                                 rc = EINPROGRESS;
7366                                 break;
7367                         }
7368                 }
7369                 mtx_unlock(&sc->tids.ftid_lock);
7370         }
7371
7372         return (rc);
7373 }
7374
7375 static void
7376 clear_filter(struct filter_entry *f)
7377 {
7378         if (f->l2t)
7379                 t4_l2t_release(f->l2t);
7380
7381         bzero(f, sizeof (*f));
7382 }
7383
7384 static int
7385 set_filter_wr(struct adapter *sc, int fidx)
7386 {
7387         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7388         struct fw_filter_wr *fwr;
7389         unsigned int ftid;
7390         struct wrq_cookie cookie;
7391
7392         ASSERT_SYNCHRONIZED_OP(sc);
7393
7394         if (f->fs.newdmac || f->fs.newvlan) {
7395                 /* This filter needs an L2T entry; allocate one. */
7396                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
7397                 if (f->l2t == NULL)
7398                         return (EAGAIN);
7399                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
7400                     f->fs.dmac)) {
7401                         t4_l2t_release(f->l2t);
7402                         f->l2t = NULL;
7403                         return (ENOMEM);
7404                 }
7405         }
7406
7407         ftid = sc->tids.ftid_base + fidx;
7408
7409         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
7410         if (fwr == NULL)
7411                 return (ENOMEM);
7412         bzero(fwr, sizeof(*fwr));
7413
7414         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
7415         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
7416         fwr->tid_to_iq =
7417             htobe32(V_FW_FILTER_WR_TID(ftid) |
7418                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
7419                 V_FW_FILTER_WR_NOREPLY(0) |
7420                 V_FW_FILTER_WR_IQ(f->fs.iq));
7421         fwr->del_filter_to_l2tix =
7422             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
7423                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
7424                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
7425                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
7426                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
7427                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
7428                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
7429                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
7430                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
7431                     f->fs.newvlan == VLAN_REWRITE) |
7432                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
7433                     f->fs.newvlan == VLAN_REWRITE) |
7434                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
7435                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
7436                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
7437                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
7438         fwr->ethtype = htobe16(f->fs.val.ethtype);
7439         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
7440         fwr->frag_to_ovlan_vldm =
7441             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
7442                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
7443                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
7444                 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
7445                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
7446                 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
7447         fwr->smac_sel = 0;
7448         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
7449             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
7450         fwr->maci_to_matchtypem =
7451             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
7452                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
7453                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
7454                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
7455                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
7456                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
7457                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
7458                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
7459         fwr->ptcl = f->fs.val.proto;
7460         fwr->ptclm = f->fs.mask.proto;
7461         fwr->ttyp = f->fs.val.tos;
7462         fwr->ttypm = f->fs.mask.tos;
7463         fwr->ivlan = htobe16(f->fs.val.vlan);
7464         fwr->ivlanm = htobe16(f->fs.mask.vlan);
7465         fwr->ovlan = htobe16(f->fs.val.vnic);
7466         fwr->ovlanm = htobe16(f->fs.mask.vnic);
7467         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
7468         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
7469         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
7470         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
7471         fwr->lp = htobe16(f->fs.val.dport);
7472         fwr->lpm = htobe16(f->fs.mask.dport);
7473         fwr->fp = htobe16(f->fs.val.sport);
7474         fwr->fpm = htobe16(f->fs.mask.sport);
7475         if (f->fs.newsmac)
7476                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
7477
7478         f->pending = 1;
7479         sc->tids.ftids_in_use++;
7480
7481         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
7482         return (0);
7483 }
7484
7485 static int
7486 del_filter_wr(struct adapter *sc, int fidx)
7487 {
7488         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7489         struct fw_filter_wr *fwr;
7490         unsigned int ftid;
7491         struct wrq_cookie cookie;
7492
7493         ftid = sc->tids.ftid_base + fidx;
7494
7495         fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
7496         if (fwr == NULL)
7497                 return (ENOMEM);
7498         bzero(fwr, sizeof (*fwr));
7499
7500         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
7501
7502         f->pending = 1;
7503         commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
7504         return (0);
7505 }
7506
7507 int
7508 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
7509 {
7510         struct adapter *sc = iq->adapter;
7511         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
7512         unsigned int idx = GET_TID(rpl);
7513         unsigned int rc;
7514         struct filter_entry *f;
7515
7516         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
7517             rss->opcode));
7518
7519         if (is_ftid(sc, idx)) {
7520
7521                 idx -= sc->tids.ftid_base;
7522                 f = &sc->tids.ftid_tab[idx];
7523                 rc = G_COOKIE(rpl->cookie);
7524
7525                 mtx_lock(&sc->tids.ftid_lock);
7526                 if (rc == FW_FILTER_WR_FLT_ADDED) {
7527                         KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
7528                             __func__, idx));
7529                         f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
7530                         f->pending = 0;  /* asynchronous setup completed */
7531                         f->valid = 1;
7532                 } else {
7533                         if (rc != FW_FILTER_WR_FLT_DELETED) {
7534                                 /* Add or delete failed, display an error */
7535                                 log(LOG_ERR,
7536                                     "filter %u setup failed with error %u\n",
7537                                     idx, rc);
7538                         }
7539
7540                         clear_filter(f);
7541                         sc->tids.ftids_in_use--;
7542                 }
7543                 wakeup(&sc->tids.ftid_tab);
7544                 mtx_unlock(&sc->tids.ftid_lock);
7545         }
7546
7547         return (0);
7548 }
7549
7550 static int
7551 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
7552 {
7553         int rc;
7554
7555         if (cntxt->cid > M_CTXTQID)
7556                 return (EINVAL);
7557
7558         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
7559             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
7560                 return (EINVAL);
7561
7562         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
7563         if (rc)
7564                 return (rc);
7565
7566         if (sc->flags & FW_OK) {
7567                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
7568                     &cntxt->data[0]);
7569                 if (rc == 0)
7570                         goto done;
7571         }
7572
7573         /*
7574          * Read via firmware failed or wasn't even attempted.  Read directly via
7575          * the backdoor.
7576          */
7577         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
7578 done:
7579         end_synchronized_op(sc, 0);
7580         return (rc);
7581 }
7582
7583 static int
7584 load_fw(struct adapter *sc, struct t4_data *fw)
7585 {
7586         int rc;
7587         uint8_t *fw_data;
7588
7589         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
7590         if (rc)
7591                 return (rc);
7592
7593         if (sc->flags & FULL_INIT_DONE) {
7594                 rc = EBUSY;
7595                 goto done;
7596         }
7597
7598         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
7599         if (fw_data == NULL) {
7600                 rc = ENOMEM;
7601                 goto done;
7602         }
7603
7604         rc = copyin(fw->data, fw_data, fw->len);
7605         if (rc == 0)
7606                 rc = -t4_load_fw(sc, fw_data, fw->len);
7607
7608         free(fw_data, M_CXGBE);
7609 done:
7610         end_synchronized_op(sc, 0);
7611         return (rc);
7612 }
7613
7614 static int
7615 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
7616 {
7617         uint32_t addr, off, remaining, i, n;
7618         uint32_t *buf, *b;
7619         uint32_t mw_base, mw_aperture;
7620         int rc;
7621         uint8_t *dst;
7622
7623         rc = validate_mem_range(sc, mr->addr, mr->len);
7624         if (rc != 0)
7625                 return (rc);
7626
7627         memwin_info(sc, win, &mw_base, &mw_aperture);
7628         buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
7629         addr = mr->addr;
7630         remaining = mr->len;
7631         dst = (void *)mr->data;
7632
7633         while (remaining) {
7634                 off = position_memwin(sc, win, addr);
7635
7636                 /* number of bytes that we'll copy in the inner loop */
7637                 n = min(remaining, mw_aperture - off);
7638                 for (i = 0; i < n; i += 4)
7639                         *b++ = t4_read_reg(sc, mw_base + off + i);
7640
7641                 rc = copyout(buf, dst, n);
7642                 if (rc != 0)
7643                         break;
7644
7645                 b = buf;
7646                 dst += n;
7647                 remaining -= n;
7648                 addr += n;
7649         }
7650
7651         free(buf, M_CXGBE);
7652         return (rc);
7653 }
7654
7655 static int
7656 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
7657 {
7658         int rc;
7659
7660         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
7661                 return (EINVAL);
7662
7663         if (i2cd->len > sizeof(i2cd->data))
7664                 return (EFBIG);
7665
7666         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
7667         if (rc)
7668                 return (rc);
7669         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
7670             i2cd->offset, i2cd->len, &i2cd->data[0]);
7671         end_synchronized_op(sc, 0);
7672
7673         return (rc);
7674 }
7675
7676 static int
7677 in_range(int val, int lo, int hi)
7678 {
7679
7680         return (val < 0 || (val <= hi && val >= lo));
7681 }
7682
7683 static int
7684 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
7685 {
7686         int fw_subcmd, fw_type, rc;
7687
7688         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
7689         if (rc)
7690                 return (rc);
7691
7692         if (!(sc->flags & FULL_INIT_DONE)) {
7693                 rc = EAGAIN;
7694                 goto done;
7695         }
7696
7697         /*
7698          * Translate the cxgbetool parameters into T4 firmware parameters.  (The
7699          * sub-command and type are in common locations.)
7700          */
7701         if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
7702                 fw_subcmd = FW_SCHED_SC_CONFIG;
7703         else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
7704                 fw_subcmd = FW_SCHED_SC_PARAMS;
7705         else {
7706                 rc = EINVAL;
7707                 goto done;
7708         }
7709         if (p->type == SCHED_CLASS_TYPE_PACKET)
7710                 fw_type = FW_SCHED_TYPE_PKTSCHED;
7711         else {
7712                 rc = EINVAL;
7713                 goto done;
7714         }
7715
7716         if (fw_subcmd == FW_SCHED_SC_CONFIG) {
7717                 /* Vet our parameters ..*/
7718                 if (p->u.config.minmax < 0) {
7719                         rc = EINVAL;
7720                         goto done;
7721                 }
7722
7723                 /* And pass the request to the firmware ...*/
7724                 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1);
7725                 goto done;
7726         }
7727
7728         if (fw_subcmd == FW_SCHED_SC_PARAMS) {
7729                 int fw_level;
7730                 int fw_mode;
7731                 int fw_rateunit;
7732                 int fw_ratemode;
7733
7734                 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
7735                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
7736                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
7737                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
7738                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
7739                         fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
7740                 else {
7741                         rc = EINVAL;
7742                         goto done;
7743                 }
7744
7745                 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
7746                         fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
7747                 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
7748                         fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
7749                 else {
7750                         rc = EINVAL;
7751                         goto done;
7752                 }
7753
7754                 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
7755                         fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
7756                 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
7757                         fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
7758                 else {
7759                         rc = EINVAL;
7760                         goto done;
7761                 }
7762
7763                 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
7764                         fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
7765                 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
7766                         fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
7767                 else {
7768                         rc = EINVAL;
7769                         goto done;
7770                 }
7771
7772                 /* Vet our parameters ... */
7773                 if (!in_range(p->u.params.channel, 0, 3) ||
7774                     !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
7775                     !in_range(p->u.params.minrate, 0, 10000000) ||
7776                     !in_range(p->u.params.maxrate, 0, 10000000) ||
7777                     !in_range(p->u.params.weight, 0, 100)) {
7778                         rc = ERANGE;
7779                         goto done;
7780                 }
7781
7782                 /*
7783                  * Translate any unset parameters into the firmware's
7784                  * nomenclature and/or fail the call if the parameters
7785                  * are required ...
7786                  */
7787                 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
7788                     p->u.params.channel < 0 || p->u.params.cl < 0) {
7789                         rc = EINVAL;
7790                         goto done;
7791                 }
7792                 if (p->u.params.minrate < 0)
7793                         p->u.params.minrate = 0;
7794                 if (p->u.params.maxrate < 0) {
7795                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7796                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7797                                 rc = EINVAL;
7798                                 goto done;
7799                         } else
7800                                 p->u.params.maxrate = 0;
7801                 }
7802                 if (p->u.params.weight < 0) {
7803                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
7804                                 rc = EINVAL;
7805                                 goto done;
7806                         } else
7807                                 p->u.params.weight = 0;
7808                 }
7809                 if (p->u.params.pktsize < 0) {
7810                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7811                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7812                                 rc = EINVAL;
7813                                 goto done;
7814                         } else
7815                                 p->u.params.pktsize = 0;
7816                 }
7817
7818                 /* See what the firmware thinks of the request ... */
7819                 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
7820                     fw_rateunit, fw_ratemode, p->u.params.channel,
7821                     p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
7822                     p->u.params.weight, p->u.params.pktsize, 1);
7823                 goto done;
7824         }
7825
7826         rc = EINVAL;
7827 done:
7828         end_synchronized_op(sc, 0);
7829         return (rc);
7830 }
7831
7832 static int
7833 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
7834 {
7835         struct port_info *pi = NULL;
7836         struct sge_txq *txq;
7837         uint32_t fw_mnem, fw_queue, fw_class;
7838         int i, rc;
7839
7840         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
7841         if (rc)
7842                 return (rc);
7843
7844         if (!(sc->flags & FULL_INIT_DONE)) {
7845                 rc = EAGAIN;
7846                 goto done;
7847         }
7848
7849         if (p->port >= sc->params.nports) {
7850                 rc = EINVAL;
7851                 goto done;
7852         }
7853
7854         pi = sc->port[p->port];
7855         if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
7856                 rc = EINVAL;
7857                 goto done;
7858         }
7859
7860         /*
7861          * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
7862          * Scheduling Class in this case).
7863          */
7864         fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
7865             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
7866         fw_class = p->cl < 0 ? 0xffffffff : p->cl;
7867
7868         /*
7869          * If op.queue is non-negative, then we're only changing the scheduling
7870          * on a single specified TX queue.
7871          */
7872         if (p->queue >= 0) {
7873                 txq = &sc->sge.txq[pi->first_txq + p->queue];
7874                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7875                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7876                     &fw_class);
7877                 goto done;
7878         }
7879
7880         /*
7881          * Change the scheduling on all the TX queues for the
7882          * interface.
7883          */
7884         for_each_txq(pi, i, txq) {
7885                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7886                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7887                     &fw_class);
7888                 if (rc)
7889                         goto done;
7890         }
7891
7892         rc = 0;
7893 done:
7894         end_synchronized_op(sc, 0);
7895         return (rc);
7896 }
7897
7898 int
7899 t4_os_find_pci_capability(struct adapter *sc, int cap)
7900 {
7901         int i;
7902
7903         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
7904 }
7905
7906 int
7907 t4_os_pci_save_state(struct adapter *sc)
7908 {
7909         device_t dev;
7910         struct pci_devinfo *dinfo;
7911
7912         dev = sc->dev;
7913         dinfo = device_get_ivars(dev);
7914
7915         pci_cfg_save(dev, dinfo, 0);
7916         return (0);
7917 }
7918
7919 int
7920 t4_os_pci_restore_state(struct adapter *sc)
7921 {
7922         device_t dev;
7923         struct pci_devinfo *dinfo;
7924
7925         dev = sc->dev;
7926         dinfo = device_get_ivars(dev);
7927
7928         pci_cfg_restore(dev, dinfo);
7929         return (0);
7930 }
7931
7932 void
7933 t4_os_portmod_changed(const struct adapter *sc, int idx)
7934 {
7935         struct port_info *pi = sc->port[idx];
7936         static const char *mod_str[] = {
7937                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
7938         };
7939
7940         build_medialist(pi, &pi->media);
7941 #ifdef DEV_NETMAP
7942         build_medialist(pi, &pi->nm_media);
7943 #endif
7944
7945         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
7946                 if_printf(pi->ifp, "transceiver unplugged.\n");
7947         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
7948                 if_printf(pi->ifp, "unknown transceiver inserted.\n");
7949         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
7950                 if_printf(pi->ifp, "unsupported transceiver inserted.\n");
7951         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
7952                 if_printf(pi->ifp, "%s transceiver inserted.\n",
7953                     mod_str[pi->mod_type]);
7954         } else {
7955                 if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
7956                     pi->mod_type);
7957         }
7958 }
7959
7960 void
7961 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
7962 {
7963         struct port_info *pi = sc->port[idx];
7964         struct ifnet *ifp = pi->ifp;
7965
7966         if (link_stat) {
7967                 pi->linkdnrc = -1;
7968                 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
7969                 if_link_state_change(ifp, LINK_STATE_UP);
7970         } else {
7971                 if (reason >= 0)
7972                         pi->linkdnrc = reason;
7973                 if_link_state_change(ifp, LINK_STATE_DOWN);
7974         }
7975 }
7976
7977 void
7978 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
7979 {
7980         struct adapter *sc;
7981
7982         sx_slock(&t4_list_lock);
7983         SLIST_FOREACH(sc, &t4_list, link) {
7984                 /*
7985                  * func should not make any assumptions about what state sc is
7986                  * in - the only guarantee is that sc->sc_lock is a valid lock.
7987                  */
7988                 func(sc, arg);
7989         }
7990         sx_sunlock(&t4_list_lock);
7991 }
7992
7993 static int
7994 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
7995 {
7996        return (0);
7997 }
7998
7999 static int
8000 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
8001 {
8002        return (0);
8003 }
8004
8005 static int
8006 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
8007     struct thread *td)
8008 {
8009         int rc;
8010         struct adapter *sc = dev->si_drv1;
8011
8012         rc = priv_check(td, PRIV_DRIVER);
8013         if (rc != 0)
8014                 return (rc);
8015
8016         switch (cmd) {
8017         case CHELSIO_T4_GETREG: {
8018                 struct t4_reg *edata = (struct t4_reg *)data;
8019
8020                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8021                         return (EFAULT);
8022
8023                 if (edata->size == 4)
8024                         edata->val = t4_read_reg(sc, edata->addr);
8025                 else if (edata->size == 8)
8026                         edata->val = t4_read_reg64(sc, edata->addr);
8027                 else
8028                         return (EINVAL);
8029
8030                 break;
8031         }
8032         case CHELSIO_T4_SETREG: {
8033                 struct t4_reg *edata = (struct t4_reg *)data;
8034
8035                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
8036                         return (EFAULT);
8037
8038                 if (edata->size == 4) {
8039                         if (edata->val & 0xffffffff00000000)
8040                                 return (EINVAL);
8041                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
8042                 } else if (edata->size == 8)
8043                         t4_write_reg64(sc, edata->addr, edata->val);
8044                 else
8045                         return (EINVAL);
8046                 break;
8047         }
8048         case CHELSIO_T4_REGDUMP: {
8049                 struct t4_regdump *regs = (struct t4_regdump *)data;
8050                 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
8051                 uint8_t *buf;
8052
8053                 if (regs->len < reglen) {
8054                         regs->len = reglen; /* hint to the caller */
8055                         return (ENOBUFS);
8056                 }
8057
8058                 regs->len = reglen;
8059                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
8060                 t4_get_regs(sc, regs, buf);
8061                 rc = copyout(buf, regs->data, reglen);
8062                 free(buf, M_CXGBE);
8063                 break;
8064         }
8065         case CHELSIO_T4_GET_FILTER_MODE:
8066                 rc = get_filter_mode(sc, (uint32_t *)data);
8067                 break;
8068         case CHELSIO_T4_SET_FILTER_MODE:
8069                 rc = set_filter_mode(sc, *(uint32_t *)data);
8070                 break;
8071         case CHELSIO_T4_GET_FILTER:
8072                 rc = get_filter(sc, (struct t4_filter *)data);
8073                 break;
8074         case CHELSIO_T4_SET_FILTER:
8075                 rc = set_filter(sc, (struct t4_filter *)data);
8076                 break;
8077         case CHELSIO_T4_DEL_FILTER:
8078                 rc = del_filter(sc, (struct t4_filter *)data);
8079                 break;
8080         case CHELSIO_T4_GET_SGE_CONTEXT:
8081                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
8082                 break;
8083         case CHELSIO_T4_LOAD_FW:
8084                 rc = load_fw(sc, (struct t4_data *)data);
8085                 break;
8086         case CHELSIO_T4_GET_MEM:
8087                 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
8088                 break;
8089         case CHELSIO_T4_GET_I2C:
8090                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
8091                 break;
8092         case CHELSIO_T4_CLEAR_STATS: {
8093                 int i;
8094                 u_int port_id = *(uint32_t *)data;
8095                 struct port_info *pi;
8096
8097                 if (port_id >= sc->params.nports)
8098                         return (EINVAL);
8099                 pi = sc->port[port_id];
8100
8101                 /* MAC stats */
8102                 t4_clr_port_stats(sc, pi->tx_chan);
8103                 pi->tx_parse_error = 0;
8104
8105                 if (pi->flags & PORT_INIT_DONE) {
8106                         struct sge_rxq *rxq;
8107                         struct sge_txq *txq;
8108                         struct sge_wrq *wrq;
8109
8110                         for_each_rxq(pi, i, rxq) {
8111 #if defined(INET) || defined(INET6)
8112                                 rxq->lro.lro_queued = 0;
8113                                 rxq->lro.lro_flushed = 0;
8114 #endif
8115                                 rxq->rxcsum = 0;
8116                                 rxq->vlan_extraction = 0;
8117                         }
8118
8119                         for_each_txq(pi, i, txq) {
8120                                 txq->txcsum = 0;
8121                                 txq->tso_wrs = 0;
8122                                 txq->vlan_insertion = 0;
8123                                 txq->imm_wrs = 0;
8124                                 txq->sgl_wrs = 0;
8125                                 txq->txpkt_wrs = 0;
8126                                 txq->txpkts0_wrs = 0;
8127                                 txq->txpkts1_wrs = 0;
8128                                 txq->txpkts0_pkts = 0;
8129                                 txq->txpkts1_pkts = 0;
8130                                 mp_ring_reset_stats(txq->r);
8131                         }
8132
8133 #ifdef TCP_OFFLOAD
8134                         /* nothing to clear for each ofld_rxq */
8135
8136                         for_each_ofld_txq(pi, i, wrq) {
8137                                 wrq->tx_wrs_direct = 0;
8138                                 wrq->tx_wrs_copied = 0;
8139                         }
8140 #endif
8141                         wrq = &sc->sge.ctrlq[pi->port_id];
8142                         wrq->tx_wrs_direct = 0;
8143                         wrq->tx_wrs_copied = 0;
8144                 }
8145                 break;
8146         }
8147         case CHELSIO_T4_SCHED_CLASS:
8148                 rc = set_sched_class(sc, (struct t4_sched_params *)data);
8149                 break;
8150         case CHELSIO_T4_SCHED_QUEUE:
8151                 rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
8152                 break;
8153         case CHELSIO_T4_GET_TRACER:
8154                 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
8155                 break;
8156         case CHELSIO_T4_SET_TRACER:
8157                 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
8158                 break;
8159         default:
8160                 rc = EINVAL;
8161         }
8162
8163         return (rc);
8164 }
8165
8166 #ifdef TCP_OFFLOAD
8167 void
8168 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask,
8169     const unsigned int *pgsz_order)
8170 {
8171         struct port_info *pi = ifp->if_softc;
8172         struct adapter *sc = pi->adapter;
8173
8174         t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
8175         t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |
8176                 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) |
8177                 V_HPZ3(pgsz_order[3]));
8178 }
8179
8180 static int
8181 toe_capability(struct port_info *pi, int enable)
8182 {
8183         int rc;
8184         struct adapter *sc = pi->adapter;
8185
8186         ASSERT_SYNCHRONIZED_OP(sc);
8187
8188         if (!is_offload(sc))
8189                 return (ENODEV);
8190
8191         if (enable) {
8192                 /*
8193                  * We need the port's queues around so that we're able to send
8194                  * and receive CPLs to/from the TOE even if the ifnet for this
8195                  * port has never been UP'd administratively.
8196                  */
8197                 if (!(pi->flags & PORT_INIT_DONE)) {
8198                         rc = cxgbe_init_synchronized(pi);
8199                         if (rc)
8200                                 return (rc);
8201                 }
8202
8203                 if (isset(&sc->offload_map, pi->port_id))
8204                         return (0);
8205
8206                 if (!uld_active(sc, ULD_TOM)) {
8207                         rc = t4_activate_uld(sc, ULD_TOM);
8208                         if (rc == EAGAIN) {
8209                                 log(LOG_WARNING,
8210                                     "You must kldload t4_tom.ko before trying "
8211                                     "to enable TOE on a cxgbe interface.\n");
8212                         }
8213                         if (rc != 0)
8214                                 return (rc);
8215                         KASSERT(sc->tom_softc != NULL,
8216                             ("%s: TOM activated but softc NULL", __func__));
8217                         KASSERT(uld_active(sc, ULD_TOM),
8218                             ("%s: TOM activated but flag not set", __func__));
8219                 }
8220
8221                 /* Activate iWARP and iSCSI too, if the modules are loaded. */
8222                 if (!uld_active(sc, ULD_IWARP))
8223                         (void) t4_activate_uld(sc, ULD_IWARP);
8224                 if (!uld_active(sc, ULD_ISCSI))
8225                         (void) t4_activate_uld(sc, ULD_ISCSI);
8226
8227                 setbit(&sc->offload_map, pi->port_id);
8228         } else {
8229                 if (!isset(&sc->offload_map, pi->port_id))
8230                         return (0);
8231
8232                 KASSERT(uld_active(sc, ULD_TOM),
8233                     ("%s: TOM never initialized?", __func__));
8234                 clrbit(&sc->offload_map, pi->port_id);
8235         }
8236
8237         return (0);
8238 }
8239
8240 /*
8241  * Add an upper layer driver to the global list.
8242  */
8243 int
8244 t4_register_uld(struct uld_info *ui)
8245 {
8246         int rc = 0;
8247         struct uld_info *u;
8248
8249         sx_xlock(&t4_uld_list_lock);
8250         SLIST_FOREACH(u, &t4_uld_list, link) {
8251             if (u->uld_id == ui->uld_id) {
8252                     rc = EEXIST;
8253                     goto done;
8254             }
8255         }
8256
8257         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
8258         ui->refcount = 0;
8259 done:
8260         sx_xunlock(&t4_uld_list_lock);
8261         return (rc);
8262 }
8263
8264 int
8265 t4_unregister_uld(struct uld_info *ui)
8266 {
8267         int rc = EINVAL;
8268         struct uld_info *u;
8269
8270         sx_xlock(&t4_uld_list_lock);
8271
8272         SLIST_FOREACH(u, &t4_uld_list, link) {
8273             if (u == ui) {
8274                     if (ui->refcount > 0) {
8275                             rc = EBUSY;
8276                             goto done;
8277                     }
8278
8279                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
8280                     rc = 0;
8281                     goto done;
8282             }
8283         }
8284 done:
8285         sx_xunlock(&t4_uld_list_lock);
8286         return (rc);
8287 }
8288
8289 int
8290 t4_activate_uld(struct adapter *sc, int id)
8291 {
8292         int rc;
8293         struct uld_info *ui;
8294
8295         ASSERT_SYNCHRONIZED_OP(sc);
8296
8297         if (id < 0 || id > ULD_MAX)
8298                 return (EINVAL);
8299         rc = EAGAIN;    /* kldoad the module with this ULD and try again. */
8300
8301         sx_slock(&t4_uld_list_lock);
8302
8303         SLIST_FOREACH(ui, &t4_uld_list, link) {
8304                 if (ui->uld_id == id) {
8305                         if (!(sc->flags & FULL_INIT_DONE)) {
8306                                 rc = adapter_full_init(sc);
8307                                 if (rc != 0)
8308                                         break;
8309                         }
8310
8311                         rc = ui->activate(sc);
8312                         if (rc == 0) {
8313                                 setbit(&sc->active_ulds, id);
8314                                 ui->refcount++;
8315                         }
8316                         break;
8317                 }
8318         }
8319
8320         sx_sunlock(&t4_uld_list_lock);
8321
8322         return (rc);
8323 }
8324
8325 int
8326 t4_deactivate_uld(struct adapter *sc, int id)
8327 {
8328         int rc;
8329         struct uld_info *ui;
8330
8331         ASSERT_SYNCHRONIZED_OP(sc);
8332
8333         if (id < 0 || id > ULD_MAX)
8334                 return (EINVAL);
8335         rc = ENXIO;
8336
8337         sx_slock(&t4_uld_list_lock);
8338
8339         SLIST_FOREACH(ui, &t4_uld_list, link) {
8340                 if (ui->uld_id == id) {
8341                         rc = ui->deactivate(sc);
8342                         if (rc == 0) {
8343                                 clrbit(&sc->active_ulds, id);
8344                                 ui->refcount--;
8345                         }
8346                         break;
8347                 }
8348         }
8349
8350         sx_sunlock(&t4_uld_list_lock);
8351
8352         return (rc);
8353 }
8354
8355 int
8356 uld_active(struct adapter *sc, int uld_id)
8357 {
8358
8359         MPASS(uld_id >= 0 && uld_id <= ULD_MAX);
8360
8361         return (isset(&sc->active_ulds, uld_id));
8362 }
8363 #endif
8364
8365 /*
8366  * Come up with reasonable defaults for some of the tunables, provided they're
8367  * not set by the user (in which case we'll use the values as is).
8368  */
8369 static void
8370 tweak_tunables(void)
8371 {
8372         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
8373
8374         if (t4_ntxq10g < 1)
8375                 t4_ntxq10g = min(nc, NTXQ_10G);
8376
8377         if (t4_ntxq1g < 1)
8378                 t4_ntxq1g = min(nc, NTXQ_1G);
8379
8380         if (t4_nrxq10g < 1)
8381                 t4_nrxq10g = min(nc, NRXQ_10G);
8382
8383         if (t4_nrxq1g < 1)
8384                 t4_nrxq1g = min(nc, NRXQ_1G);
8385
8386 #ifdef TCP_OFFLOAD
8387         if (t4_nofldtxq10g < 1)
8388                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
8389
8390         if (t4_nofldtxq1g < 1)
8391                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
8392
8393         if (t4_nofldrxq10g < 1)
8394                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
8395
8396         if (t4_nofldrxq1g < 1)
8397                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
8398
8399         if (t4_toecaps_allowed == -1)
8400                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
8401 #else
8402         if (t4_toecaps_allowed == -1)
8403                 t4_toecaps_allowed = 0;
8404 #endif
8405
8406 #ifdef DEV_NETMAP
8407         if (t4_nnmtxq10g < 1)
8408                 t4_nnmtxq10g = min(nc, NNMTXQ_10G);
8409
8410         if (t4_nnmtxq1g < 1)
8411                 t4_nnmtxq1g = min(nc, NNMTXQ_1G);
8412
8413         if (t4_nnmrxq10g < 1)
8414                 t4_nnmrxq10g = min(nc, NNMRXQ_10G);
8415
8416         if (t4_nnmrxq1g < 1)
8417                 t4_nnmrxq1g = min(nc, NNMRXQ_1G);
8418 #endif
8419
8420         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
8421                 t4_tmr_idx_10g = TMR_IDX_10G;
8422
8423         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
8424                 t4_pktc_idx_10g = PKTC_IDX_10G;
8425
8426         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
8427                 t4_tmr_idx_1g = TMR_IDX_1G;
8428
8429         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
8430                 t4_pktc_idx_1g = PKTC_IDX_1G;
8431
8432         if (t4_qsize_txq < 128)
8433                 t4_qsize_txq = 128;
8434
8435         if (t4_qsize_rxq < 128)
8436                 t4_qsize_rxq = 128;
8437         while (t4_qsize_rxq & 7)
8438                 t4_qsize_rxq++;
8439
8440         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
8441 }
8442
8443 static struct sx mlu;   /* mod load unload */
8444 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
8445
8446 static int
8447 mod_event(module_t mod, int cmd, void *arg)
8448 {
8449         int rc = 0;
8450         static int loaded = 0;
8451
8452         switch (cmd) {
8453         case MOD_LOAD:
8454                 sx_xlock(&mlu);
8455                 if (loaded++ == 0) {
8456                         t4_sge_modload();
8457                         sx_init(&t4_list_lock, "T4/T5 adapters");
8458                         SLIST_INIT(&t4_list);
8459 #ifdef TCP_OFFLOAD
8460                         sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
8461                         SLIST_INIT(&t4_uld_list);
8462 #endif
8463                         t4_tracer_modload();
8464                         tweak_tunables();
8465                 }
8466                 sx_xunlock(&mlu);
8467                 break;
8468
8469         case MOD_UNLOAD:
8470                 sx_xlock(&mlu);
8471                 if (--loaded == 0) {
8472                         int tries;
8473
8474                         sx_slock(&t4_list_lock);
8475                         if (!SLIST_EMPTY(&t4_list)) {
8476                                 rc = EBUSY;
8477                                 sx_sunlock(&t4_list_lock);
8478                                 goto done_unload;
8479                         }
8480 #ifdef TCP_OFFLOAD
8481                         sx_slock(&t4_uld_list_lock);
8482                         if (!SLIST_EMPTY(&t4_uld_list)) {
8483                                 rc = EBUSY;
8484                                 sx_sunlock(&t4_uld_list_lock);
8485                                 sx_sunlock(&t4_list_lock);
8486                                 goto done_unload;
8487                         }
8488 #endif
8489                         tries = 0;
8490                         while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
8491                                 uprintf("%ju clusters with custom free routine "
8492                                     "still is use.\n", t4_sge_extfree_refs());
8493                                 pause("t4unload", 2 * hz);
8494                         }
8495 #ifdef TCP_OFFLOAD
8496                         sx_sunlock(&t4_uld_list_lock);
8497 #endif
8498                         sx_sunlock(&t4_list_lock);
8499
8500                         if (t4_sge_extfree_refs() == 0) {
8501                                 t4_tracer_modunload();
8502 #ifdef TCP_OFFLOAD
8503                                 sx_destroy(&t4_uld_list_lock);
8504 #endif
8505                                 sx_destroy(&t4_list_lock);
8506                                 t4_sge_modunload();
8507                                 loaded = 0;
8508                         } else {
8509                                 rc = EBUSY;
8510                                 loaded++;       /* undo earlier decrement */
8511                         }
8512                 }
8513 done_unload:
8514                 sx_xunlock(&mlu);
8515                 break;
8516         }
8517
8518         return (rc);
8519 }
8520
8521 static devclass_t t4_devclass, t5_devclass;
8522 static devclass_t cxgbe_devclass, cxl_devclass;
8523
8524 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
8525 MODULE_VERSION(t4nex, 1);
8526 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
8527
8528 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
8529 MODULE_VERSION(t5nex, 1);
8530 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
8531
8532 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
8533 MODULE_VERSION(cxgbe, 1);
8534
8535 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
8536 MODULE_VERSION(cxl, 1);