sys/dev/cxgbe/t4_main.c

   1 /*-
   2  * Copyright (c) 2011 Chelsio Communications, Inc.
   3  * All rights reserved.
   4  * Written by: Navdeep Parhar <np@FreeBSD.org>
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_inet.h"
  32 #include "opt_inet6.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/conf.h>
  36 #include <sys/priv.h>
  37 #include <sys/kernel.h>
  38 #include <sys/bus.h>
  39 #include <sys/module.h>
  40 #include <sys/malloc.h>
  41 #include <sys/queue.h>
  42 #include <sys/taskqueue.h>
  43 #include <sys/pciio.h>
  44 #include <dev/pci/pcireg.h>
  45 #include <dev/pci/pcivar.h>
  46 #include <dev/pci/pci_private.h>
  47 #include <sys/firmware.h>
  48 #include <sys/sbuf.h>
  49 #include <sys/smp.h>
  50 #include <sys/socket.h>
  51 #include <sys/sockio.h>
  52 #include <sys/sysctl.h>
  53 #include <net/ethernet.h>
  54 #include <net/if.h>
  55 #include <net/if_types.h>
  56 #include <net/if_dl.h>
  57 #include <net/if_vlan_var.h>
  58 #if defined(__i386__) || defined(__amd64__)
  59 #include <vm/vm.h>
  60 #include <vm/pmap.h>
  61 #endif
  62
  63 #include "common/common.h"
  64 #include "common/t4_msg.h"
  65 #include "common/t4_regs.h"
  66 #include "common/t4_regs_values.h"
  67 #include "t4_ioctl.h"
  68 #include "t4_l2t.h"
  69
  70 /* T4 bus driver interface */
  71 static int t4_probe(device_t);
  72 static int t4_attach(device_t);
  73 static int t4_detach(device_t);
  74 static device_method_t t4_methods[] = {
  75         DEVMETHOD(device_probe,         t4_probe),
  76         DEVMETHOD(device_attach,        t4_attach),
  77         DEVMETHOD(device_detach,        t4_detach),
  78
  79         DEVMETHOD_END
  80 };
  81 static driver_t t4_driver = {
  82         "t4nex",
  83         t4_methods,
  84         sizeof(struct adapter)
  85 };
  86
  87
  88 /* T4 port (cxgbe) interface */
  89 static int cxgbe_probe(device_t);
  90 static int cxgbe_attach(device_t);
  91 static int cxgbe_detach(device_t);
  92 static device_method_t cxgbe_methods[] = {
  93         DEVMETHOD(device_probe,         cxgbe_probe),
  94         DEVMETHOD(device_attach,        cxgbe_attach),
  95         DEVMETHOD(device_detach,        cxgbe_detach),
  96         { 0, 0 }
  97 };
  98 static driver_t cxgbe_driver = {
  99         "cxgbe",
 100         cxgbe_methods,
 101         sizeof(struct port_info)
 102 };
 103
 104 static d_ioctl_t t4_ioctl;
 105 static d_open_t t4_open;
 106 static d_close_t t4_close;
 107
 108 static struct cdevsw t4_cdevsw = {
 109        .d_version = D_VERSION,
 110        .d_flags = 0,
 111        .d_open = t4_open,
 112        .d_close = t4_close,
 113        .d_ioctl = t4_ioctl,
 114        .d_name = "t4nex",
 115 };
 116
 117 /* T5 bus driver interface */
 118 static int t5_probe(device_t);
 119 static device_method_t t5_methods[] = {
 120         DEVMETHOD(device_probe,         t5_probe),
 121         DEVMETHOD(device_attach,        t4_attach),
 122         DEVMETHOD(device_detach,        t4_detach),
 123
 124         DEVMETHOD_END
 125 };
 126 static driver_t t5_driver = {
 127         "t5nex",
 128         t5_methods,
 129         sizeof(struct adapter)
 130 };
 131
 132
 133 /* T5 port (cxl) interface */
 134 static driver_t cxl_driver = {
 135         "cxl",
 136         cxgbe_methods,
 137         sizeof(struct port_info)
 138 };
 139
 140 static struct cdevsw t5_cdevsw = {
 141        .d_version = D_VERSION,
 142        .d_flags = 0,
 143        .d_open = t4_open,
 144        .d_close = t4_close,
 145        .d_ioctl = t4_ioctl,
 146        .d_name = "t5nex",
 147 };
 148
 149 /* ifnet + media interface */
 150 static void cxgbe_init(void *);
 151 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
 152 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
 153 static void cxgbe_qflush(struct ifnet *);
 154 static int cxgbe_media_change(struct ifnet *);
 155 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 156
 157 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
 158
 159 /*
 160  * Correct lock order when you need to acquire multiple locks is t4_list_lock,
 161  * then ADAPTER_LOCK, then t4_uld_list_lock.
 162  */
 163 static struct sx t4_list_lock;
 164 SLIST_HEAD(, adapter) t4_list;
 165 #ifdef TCP_OFFLOAD
 166 static struct sx t4_uld_list_lock;
 167 SLIST_HEAD(, uld_info) t4_uld_list;
 168 #endif
 169
 170 /*
 171  * Tunables.  See tweak_tunables() too.
 172  *
 173  * Each tunable is set to a default value here if it's known at compile-time.
 174  * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
 175  * provide a reasonable default when the driver is loaded.
 176  *
 177  * Tunables applicable to both T4 and T5 are under hw.cxgbe.  Those specific to
 178  * T5 are under hw.cxl.
 179  */
 180
 181 /*
 182  * Number of queues for tx and rx, 10G and 1G, NIC and offload.
 183  */
 184 #define NTXQ_10G 16
 185 static int t4_ntxq10g = -1;
 186 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
 187
 188 #define NRXQ_10G 8
 189 static int t4_nrxq10g = -1;
 190 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
 191
 192 #define NTXQ_1G 4
 193 static int t4_ntxq1g = -1;
 194 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
 195
 196 #define NRXQ_1G 2
 197 static int t4_nrxq1g = -1;
 198 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
 199
 200 static int t4_rsrv_noflowq = 0;
 201 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
 202
 203 #ifdef TCP_OFFLOAD
 204 #define NOFLDTXQ_10G 8
 205 static int t4_nofldtxq10g = -1;
 206 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
 207
 208 #define NOFLDRXQ_10G 2
 209 static int t4_nofldrxq10g = -1;
 210 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
 211
 212 #define NOFLDTXQ_1G 2
 213 static int t4_nofldtxq1g = -1;
 214 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
 215
 216 #define NOFLDRXQ_1G 1
 217 static int t4_nofldrxq1g = -1;
 218 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 219 #endif
 220
 221 /*
 222  * Holdoff parameters for 10G and 1G ports.
 223  */
 224 #define TMR_IDX_10G 1
 225 static int t4_tmr_idx_10g = TMR_IDX_10G;
 226 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
 227
 228 #define PKTC_IDX_10G (-1)
 229 static int t4_pktc_idx_10g = PKTC_IDX_10G;
 230 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
 231
 232 #define TMR_IDX_1G 1
 233 static int t4_tmr_idx_1g = TMR_IDX_1G;
 234 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
 235
 236 #define PKTC_IDX_1G (-1)
 237 static int t4_pktc_idx_1g = PKTC_IDX_1G;
 238 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
 239
 240 /*
 241  * Size (# of entries) of each tx and rx queue.
 242  */
 243 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
 244 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
 245
 246 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
 247 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
 248
 249 /*
 250  * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
 251  */
 252 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
 253 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
 254
 255 /*
 256  * Configuration file.
 257  */
 258 #define DEFAULT_CF      "default"
 259 #define FLASH_CF        "flash"
 260 #define UWIRE_CF        "uwire"
 261 #define FPGA_CF         "fpga"
 262 static char t4_cfg_file[32] = DEFAULT_CF;
 263 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
 264
 265 /*
 266  * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
 267  * encouraged respectively).
 268  */
 269 static unsigned int t4_fw_install = 1;
 270 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
 271
 272 /*
 273  * ASIC features that will be used.  Disable the ones you don't want so that the
 274  * chip resources aren't wasted on features that will not be used.
 275  */
 276 static int t4_linkcaps_allowed = 0;     /* No DCBX, PPP, etc. by default */
 277 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
 278
 279 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
 280 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
 281
 282 static int t4_toecaps_allowed = -1;
 283 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
 284
 285 static int t4_rdmacaps_allowed = 0;
 286 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
 287
 288 static int t4_iscsicaps_allowed = 0;
 289 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
 290
 291 static int t4_fcoecaps_allowed = 0;
 292 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
 293
 294 static int t5_write_combine = 0;
 295 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 296
 297 struct intrs_and_queues {
 298         int intr_type;          /* INTx, MSI, or MSI-X */
 299         int nirq;               /* Number of vectors */
 300         int intr_flags;
 301         int ntxq10g;            /* # of NIC txq's for each 10G port */
 302         int nrxq10g;            /* # of NIC rxq's for each 10G port */
 303         int ntxq1g;             /* # of NIC txq's for each 1G port */
 304         int nrxq1g;             /* # of NIC rxq's for each 1G port */
 305         int rsrv_noflowq;       /* Flag whether to reserve queue 0 */
 306 #ifdef TCP_OFFLOAD
 307         int nofldtxq10g;        /* # of TOE txq's for each 10G port */
 308         int nofldrxq10g;        /* # of TOE rxq's for each 10G port */
 309         int nofldtxq1g;         /* # of TOE txq's for each 1G port */
 310         int nofldrxq1g;         /* # of TOE rxq's for each 1G port */
 311 #endif
 312 };
 313
 314 struct filter_entry {
 315         uint32_t valid:1;       /* filter allocated and valid */
 316         uint32_t locked:1;      /* filter is administratively locked */
 317         uint32_t pending:1;     /* filter action is pending firmware reply */
 318         uint32_t smtidx:8;      /* Source MAC Table index for smac */
 319         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 320
 321         struct t4_filter_specification fs;
 322 };
 323
 324 enum {
 325         XGMAC_MTU       = (1 << 0),
 326         XGMAC_PROMISC   = (1 << 1),
 327         XGMAC_ALLMULTI  = (1 << 2),
 328         XGMAC_VLANEX    = (1 << 3),
 329         XGMAC_UCADDR    = (1 << 4),
 330         XGMAC_MCADDRS   = (1 << 5),
 331
 332         XGMAC_ALL       = 0xffff
 333 };
 334
 335 static int map_bars_0_and_4(struct adapter *);
 336 static int map_bar_2(struct adapter *);
 337 static void setup_memwin(struct adapter *);
 338 static int validate_mem_range(struct adapter *, uint32_t, int);
 339 static int fwmtype_to_hwmtype(int);
 340 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
 341     uint32_t *);
 342 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
 343 static uint32_t position_memwin(struct adapter *, int, uint32_t);
 344 static int cfg_itype_and_nqueues(struct adapter *, int, int,
 345     struct intrs_and_queues *);
 346 static int prep_firmware(struct adapter *);
 347 static int partition_resources(struct adapter *, const struct firmware *,
 348     const char *);
 349 static int get_params__pre_init(struct adapter *);
 350 static int get_params__post_init(struct adapter *);
 351 static int set_params__post_init(struct adapter *);
 352 static void t4_set_desc(struct adapter *);
 353 static void build_medialist(struct port_info *);
 354 static int update_mac_settings(struct port_info *, int);
 355 static int cxgbe_init_synchronized(struct port_info *);
 356 static int cxgbe_uninit_synchronized(struct port_info *);
 357 static int setup_intr_handlers(struct adapter *);
 358 static int adapter_full_init(struct adapter *);
 359 static int adapter_full_uninit(struct adapter *);
 360 static int port_full_init(struct port_info *);
 361 static int port_full_uninit(struct port_info *);
 362 static void quiesce_eq(struct adapter *, struct sge_eq *);
 363 static void quiesce_iq(struct adapter *, struct sge_iq *);
 364 static void quiesce_fl(struct adapter *, struct sge_fl *);
 365 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
 366     driver_intr_t *, void *, char *);
 367 static int t4_free_irq(struct adapter *, struct irq *);
 368 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
 369     unsigned int);
 370 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
 371 static void cxgbe_tick(void *);
 372 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
 373 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
 374     struct mbuf *);
 375 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
 376 static int fw_msg_not_handled(struct adapter *, const __be64 *);
 377 static int t4_sysctls(struct adapter *);
 378 static int cxgbe_sysctls(struct port_info *);
 379 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
 380 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
 381 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
 382 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
 383 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
 384 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
 385 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
 386 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
 387 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
 388 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
 389 #ifdef SBUF_DRAIN
 390 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
 391 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
 392 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
 393 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
 394 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
 395 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
 396 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
 397 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
 398 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
 399 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
 400 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
 401 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
 402 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
 403 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
 404 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
 405 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
 406 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
 407 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
 408 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
 409 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
 410 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
 411 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
 412 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 413 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 414 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 415 #endif
 416 static inline void txq_start(struct ifnet *, struct sge_txq *);
 417 static uint32_t fconf_to_mode(uint32_t);
 418 static uint32_t mode_to_fconf(uint32_t);
 419 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
 420 static int get_filter_mode(struct adapter *, uint32_t *);
 421 static int set_filter_mode(struct adapter *, uint32_t);
 422 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
 423 static int get_filter(struct adapter *, struct t4_filter *);
 424 static int set_filter(struct adapter *, struct t4_filter *);
 425 static int del_filter(struct adapter *, struct t4_filter *);
 426 static void clear_filter(struct filter_entry *);
 427 static int set_filter_wr(struct adapter *, int);
 428 static int del_filter_wr(struct adapter *, int);
 429 static int get_sge_context(struct adapter *, struct t4_sge_context *);
 430 static int load_fw(struct adapter *, struct t4_data *);
 431 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
 432 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 433 static int set_sched_class(struct adapter *, struct t4_sched_params *);
 434 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
 435 #ifdef TCP_OFFLOAD
 436 static int toe_capability(struct port_info *, int);
 437 #endif
 438 static int mod_event(module_t, int, void *);
 439
 440 struct {
 441         uint16_t device;
 442         char *desc;
 443 } t4_pciids[] = {
 444         {0xa000, "Chelsio Terminator 4 FPGA"},
 445         {0x4400, "Chelsio T440-dbg"},
 446         {0x4401, "Chelsio T420-CR"},
 447         {0x4402, "Chelsio T422-CR"},
 448         {0x4403, "Chelsio T440-CR"},
 449         {0x4404, "Chelsio T420-BCH"},
 450         {0x4405, "Chelsio T440-BCH"},
 451         {0x4406, "Chelsio T440-CH"},
 452         {0x4407, "Chelsio T420-SO"},
 453         {0x4408, "Chelsio T420-CX"},
 454         {0x4409, "Chelsio T420-BT"},
 455         {0x440a, "Chelsio T404-BT"},
 456         {0x440e, "Chelsio T440-LP-CR"},
 457 }, t5_pciids[] = {
 458         {0xb000, "Chelsio Terminator 5 FPGA"},
 459         {0x5400, "Chelsio T580-dbg"},
 460         {0x5401,  "Chelsio T520-CR"},           /* 2 x 10G */
 461         {0x5402,  "Chelsio T522-CR"},           /* 2 x 10G, 2 X 1G */
 462         {0x5403,  "Chelsio T540-CR"},           /* 4 x 10G */
 463         {0x5407,  "Chelsio T520-SO"},           /* 2 x 10G, nomem */
 464         {0x5409,  "Chelsio T520-BT"},           /* 2 x 10GBaseT */
 465         {0x540a,  "Chelsio T504-BT"},           /* 4 x 1G */
 466         {0x540d,  "Chelsio T580-CR"},           /* 2 x 40G */
 467         {0x540e,  "Chelsio T540-LP-CR"},        /* 4 x 10G */
 468         {0x5410,  "Chelsio T580-LP-CR"},        /* 2 x 40G */
 469         {0x5411,  "Chelsio T520-LL-CR"},        /* 2 x 10G */
 470         {0x5412,  "Chelsio T560-CR"},           /* 1 x 40G, 2 x 10G */
 471         {0x5414,  "Chelsio T580-LP-SO-CR"},     /* 2 x 40G, nomem */
 472 #ifdef notyet
 473         {0x5404,  "Chelsio T520-BCH"},
 474         {0x5405,  "Chelsio T540-BCH"},
 475         {0x5406,  "Chelsio T540-CH"},
 476         {0x5408,  "Chelsio T520-CX"},
 477         {0x540b,  "Chelsio B520-SR"},
 478         {0x540c,  "Chelsio B504-BT"},
 479         {0x540f,  "Chelsio Amsterdam"},
 480         {0x5413,  "Chelsio T580-CHR"},
 481 #endif
 482 };
 483
 484 #ifdef TCP_OFFLOAD
 485 /*
 486  * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
 487  * exactly the same for both rxq and ofld_rxq.
 488  */
 489 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 490 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
 491 #endif
 492
 493 /* No easy way to include t4_msg.h before adapter.h so we check this way */
 494 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
 495 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
 496
 497 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
 498
 499 static int
 500 t4_probe(device_t dev)
 501 {
 502         int i;
 503         uint16_t v = pci_get_vendor(dev);
 504         uint16_t d = pci_get_device(dev);
 505         uint8_t f = pci_get_function(dev);
 506
 507         if (v != PCI_VENDOR_ID_CHELSIO)
 508                 return (ENXIO);
 509
 510         /* Attach only to PF0 of the FPGA */
 511         if (d == 0xa000 && f != 0)
 512                 return (ENXIO);
 513
 514         for (i = 0; i < nitems(t4_pciids); i++) {
 515                 if (d == t4_pciids[i].device) {
 516                         device_set_desc(dev, t4_pciids[i].desc);
 517                         return (BUS_PROBE_DEFAULT);
 518                 }
 519         }
 520
 521         return (ENXIO);
 522 }
 523
 524 static int
 525 t5_probe(device_t dev)
 526 {
 527         int i;
 528         uint16_t v = pci_get_vendor(dev);
 529         uint16_t d = pci_get_device(dev);
 530         uint8_t f = pci_get_function(dev);
 531
 532         if (v != PCI_VENDOR_ID_CHELSIO)
 533                 return (ENXIO);
 534
 535         /* Attach only to PF0 of the FPGA */
 536         if (d == 0xb000 && f != 0)
 537                 return (ENXIO);
 538
 539         for (i = 0; i < nitems(t5_pciids); i++) {
 540                 if (d == t5_pciids[i].device) {
 541                         device_set_desc(dev, t5_pciids[i].desc);
 542                         return (BUS_PROBE_DEFAULT);
 543                 }
 544         }
 545
 546         return (ENXIO);
 547 }
 548
 549 static int
 550 t4_attach(device_t dev)
 551 {
 552         struct adapter *sc;
 553         int rc = 0, i, n10g, n1g, rqidx, tqidx;
 554         struct intrs_and_queues iaq;
 555         struct sge *s;
 556 #ifdef TCP_OFFLOAD
 557         int ofld_rqidx, ofld_tqidx;
 558 #endif
 559
 560         sc = device_get_softc(dev);
 561         sc->dev = dev;
 562
 563         pci_enable_busmaster(dev);
 564         if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
 565                 uint32_t v;
 566
 567                 pci_set_max_read_req(dev, 4096);
 568                 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
 569                 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
 570                 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
 571         }
 572
 573         sc->traceq = -1;
 574         mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
 575         snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
 576             device_get_nameunit(dev));
 577
 578         snprintf(sc->lockname, sizeof(sc->lockname), "%s",
 579             device_get_nameunit(dev));
 580         mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
 581         sx_xlock(&t4_list_lock);
 582         SLIST_INSERT_HEAD(&t4_list, sc, link);
 583         sx_xunlock(&t4_list_lock);
 584
 585         mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
 586         TAILQ_INIT(&sc->sfl);
 587         callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
 588
 589         rc = map_bars_0_and_4(sc);
 590         if (rc != 0)
 591                 goto done; /* error message displayed already */
 592
 593         /*
 594          * This is the real PF# to which we're attaching.  Works from within PCI
 595          * passthrough environments too, where pci_get_function() could return a
 596          * different PF# depending on the passthrough configuration.  We need to
 597          * use the real PF# in all our communication with the firmware.
 598          */
 599         sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
 600         sc->mbox = sc->pf;
 601
 602         memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
 603         sc->an_handler = an_not_handled;
 604         for (i = 0; i < nitems(sc->cpl_handler); i++)
 605                 sc->cpl_handler[i] = cpl_not_handled;
 606         for (i = 0; i < nitems(sc->fw_msg_handler); i++)
 607                 sc->fw_msg_handler[i] = fw_msg_not_handled;
 608         t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
 609         t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
 610         t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt);
 611         t4_init_sge_cpl_handlers(sc);
 612
 613         /* Prepare the adapter for operation */
 614         rc = -t4_prep_adapter(sc);
 615         if (rc != 0) {
 616                 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
 617                 goto done;
 618         }
 619
 620         /*
 621          * Do this really early, with the memory windows set up even before the
 622          * character device.  The userland tool's register i/o and mem read
 623          * will work even in "recovery mode".
 624          */
 625         setup_memwin(sc);
 626         sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
 627             device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
 628             device_get_nameunit(dev));
 629         if (sc->cdev == NULL)
 630                 device_printf(dev, "failed to create nexus char device.\n");
 631         else
 632                 sc->cdev->si_drv1 = sc;
 633
 634         /* Go no further if recovery mode has been requested. */
 635         if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
 636                 device_printf(dev, "recovery mode.\n");
 637                 goto done;
 638         }
 639
 640         /* Prepare the firmware for operation */
 641         rc = prep_firmware(sc);
 642         if (rc != 0)
 643                 goto done; /* error message displayed already */
 644
 645         rc = get_params__post_init(sc);
 646         if (rc != 0)
 647                 goto done; /* error message displayed already */
 648
 649         rc = set_params__post_init(sc);
 650         if (rc != 0)
 651                 goto done; /* error message displayed already */
 652
 653         rc = map_bar_2(sc);
 654         if (rc != 0)
 655                 goto done; /* error message displayed already */
 656
 657         rc = t4_create_dma_tag(sc);
 658         if (rc != 0)
 659                 goto done; /* error message displayed already */
 660
 661         /*
 662          * First pass over all the ports - allocate VIs and initialize some
 663          * basic parameters like mac address, port type, etc.  We also figure
 664          * out whether a port is 10G or 1G and use that information when
 665          * calculating how many interrupts to attempt to allocate.
 666          */
 667         n10g = n1g = 0;
 668         for_each_port(sc, i) {
 669                 struct port_info *pi;
 670
 671                 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
 672                 sc->port[i] = pi;
 673
 674                 /* These must be set before t4_port_init */
 675                 pi->adapter = sc;
 676                 pi->port_id = i;
 677
 678                 /* Allocate the vi and initialize parameters like mac addr */
 679                 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
 680                 if (rc != 0) {
 681                         device_printf(dev, "unable to initialize port %d: %d\n",
 682                             i, rc);
 683                         free(pi, M_CXGBE);
 684                         sc->port[i] = NULL;
 685                         goto done;
 686                 }
 687
 688                 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 689                     device_get_nameunit(dev), i);
 690                 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
 691                 sc->chan_map[pi->tx_chan] = i;
 692
 693                 if (is_10G_port(pi) || is_40G_port(pi)) {
 694                         n10g++;
 695                         pi->tmr_idx = t4_tmr_idx_10g;
 696                         pi->pktc_idx = t4_pktc_idx_10g;
 697                 } else {
 698                         n1g++;
 699                         pi->tmr_idx = t4_tmr_idx_1g;
 700                         pi->pktc_idx = t4_pktc_idx_1g;
 701                 }
 702
 703                 pi->xact_addr_filt = -1;
 704                 pi->linkdnrc = -1;
 705
 706                 pi->qsize_rxq = t4_qsize_rxq;
 707                 pi->qsize_txq = t4_qsize_txq;
 708
 709                 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
 710                 if (pi->dev == NULL) {
 711                         device_printf(dev,
 712                             "failed to add device for port %d.\n", i);
 713                         rc = ENXIO;
 714                         goto done;
 715                 }
 716                 device_set_softc(pi->dev, pi);
 717         }
 718
 719         /*
 720          * Interrupt type, # of interrupts, # of rx/tx queues, etc.
 721          */
 722         rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
 723         if (rc != 0)
 724                 goto done; /* error message displayed already */
 725
 726         sc->intr_type = iaq.intr_type;
 727         sc->intr_count = iaq.nirq;
 728         sc->flags |= iaq.intr_flags;
 729
 730         s = &sc->sge;
 731         s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 732         s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 733         s->neq = s->ntxq + s->nrxq;     /* the free list in an rxq is an eq */
 734         s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 735         s->niq = s->nrxq + 1;           /* 1 extra for firmware event queue */
 736
 737 #ifdef TCP_OFFLOAD
 738         if (is_offload(sc)) {
 739
 740                 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 741                 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 742                 s->neq += s->nofldtxq + s->nofldrxq;
 743                 s->niq += s->nofldrxq;
 744
 745                 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
 746                     M_CXGBE, M_ZERO | M_WAITOK);
 747                 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
 748                     M_CXGBE, M_ZERO | M_WAITOK);
 749         }
 750 #endif
 751
 752         s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 753             M_ZERO | M_WAITOK);
 754         s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 755             M_ZERO | M_WAITOK);
 756         s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
 757             M_ZERO | M_WAITOK);
 758         s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
 759             M_ZERO | M_WAITOK);
 760         s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
 761             M_ZERO | M_WAITOK);
 762
 763         sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
 764             M_ZERO | M_WAITOK);
 765
 766         t4_init_l2t(sc, M_WAITOK);
 767
 768         /*
 769          * Second pass over the ports.  This time we know the number of rx and
 770          * tx queues that each port should get.
 771          */
 772         rqidx = tqidx = 0;
 773 #ifdef TCP_OFFLOAD
 774         ofld_rqidx = ofld_tqidx = 0;
 775 #endif
 776         for_each_port(sc, i) {
 777                 struct port_info *pi = sc->port[i];
 778
 779                 if (pi == NULL)
 780                         continue;
 781
 782                 pi->first_rxq = rqidx;
 783                 pi->first_txq = tqidx;
 784                 if (is_10G_port(pi) || is_40G_port(pi)) {
 785                         pi->nrxq = iaq.nrxq10g;
 786                         pi->ntxq = iaq.ntxq10g;
 787                 } else {
 788                         pi->nrxq = iaq.nrxq1g;
 789                         pi->ntxq = iaq.ntxq1g;
 790                 }
 791
 792                 if (pi->ntxq > 1)
 793                         pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
 794                 else
 795                         pi->rsrv_noflowq = 0;
 796
 797                 rqidx += pi->nrxq;
 798                 tqidx += pi->ntxq;
 799
 800 #ifdef TCP_OFFLOAD
 801                 if (is_offload(sc)) {
 802                         pi->first_ofld_rxq = ofld_rqidx;
 803                         pi->first_ofld_txq = ofld_tqidx;
 804                         if (is_10G_port(pi) || is_40G_port(pi)) {
 805                                 pi->nofldrxq = iaq.nofldrxq10g;
 806                                 pi->nofldtxq = iaq.nofldtxq10g;
 807                         } else {
 808                                 pi->nofldrxq = iaq.nofldrxq1g;
 809                                 pi->nofldtxq = iaq.nofldtxq1g;
 810                         }
 811                         ofld_rqidx += pi->nofldrxq;
 812                         ofld_tqidx += pi->nofldtxq;
 813                 }
 814 #endif
 815         }
 816
 817         rc = setup_intr_handlers(sc);
 818         if (rc != 0) {
 819                 device_printf(dev,
 820                     "failed to setup interrupt handlers: %d\n", rc);
 821                 goto done;
 822         }
 823
 824         rc = bus_generic_attach(dev);
 825         if (rc != 0) {
 826                 device_printf(dev,
 827                     "failed to attach all child ports: %d\n", rc);
 828                 goto done;
 829         }
 830
 831         device_printf(dev,
 832             "PCIe x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
 833             sc->params.pci.width, sc->params.nports, sc->intr_count,
 834             sc->intr_type == INTR_MSIX ? "MSI-X" :
 835             (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
 836             sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
 837
 838         t4_set_desc(sc);
 839
 840 done:
 841         if (rc != 0 && sc->cdev) {
 842                 /* cdev was created and so cxgbetool works; recover that way. */
 843                 device_printf(dev,
 844                     "error during attach, adapter is now in recovery mode.\n");
 845                 rc = 0;
 846         }
 847
 848         if (rc != 0)
 849                 t4_detach(dev);
 850         else
 851                 t4_sysctls(sc);
 852
 853         return (rc);
 854 }
 855
 856 /*
 857  * Idempotent
 858  */
 859 static int
 860 t4_detach(device_t dev)
 861 {
 862         struct adapter *sc;
 863         struct port_info *pi;
 864         int i, rc;
 865
 866         sc = device_get_softc(dev);
 867
 868         if (sc->flags & FULL_INIT_DONE)
 869                 t4_intr_disable(sc);
 870
 871         if (sc->cdev) {
 872                 destroy_dev(sc->cdev);
 873                 sc->cdev = NULL;
 874         }
 875
 876         rc = bus_generic_detach(dev);
 877         if (rc) {
 878                 device_printf(dev,
 879                     "failed to detach child devices: %d\n", rc);
 880                 return (rc);
 881         }
 882
 883         for (i = 0; i < sc->intr_count; i++)
 884                 t4_free_irq(sc, &sc->irq[i]);
 885
 886         for (i = 0; i < MAX_NPORTS; i++) {
 887                 pi = sc->port[i];
 888                 if (pi) {
 889                         t4_free_vi(pi->adapter, sc->mbox, sc->pf, 0, pi->viid);
 890                         if (pi->dev)
 891                                 device_delete_child(dev, pi->dev);
 892
 893                         mtx_destroy(&pi->pi_lock);
 894                         free(pi, M_CXGBE);
 895                 }
 896         }
 897
 898         if (sc->flags & FULL_INIT_DONE)
 899                 adapter_full_uninit(sc);
 900
 901         if (sc->flags & FW_OK)
 902                 t4_fw_bye(sc, sc->mbox);
 903
 904         if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
 905                 pci_release_msi(dev);
 906
 907         if (sc->regs_res)
 908                 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
 909                     sc->regs_res);
 910
 911         if (sc->udbs_res)
 912                 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
 913                     sc->udbs_res);
 914
 915         if (sc->msix_res)
 916                 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
 917                     sc->msix_res);
 918
 919         if (sc->l2t)
 920                 t4_free_l2t(sc->l2t);
 921
 922 #ifdef TCP_OFFLOAD
 923         free(sc->sge.ofld_rxq, M_CXGBE);
 924         free(sc->sge.ofld_txq, M_CXGBE);
 925 #endif
 926         free(sc->irq, M_CXGBE);
 927         free(sc->sge.rxq, M_CXGBE);
 928         free(sc->sge.txq, M_CXGBE);
 929         free(sc->sge.ctrlq, M_CXGBE);
 930         free(sc->sge.iqmap, M_CXGBE);
 931         free(sc->sge.eqmap, M_CXGBE);
 932         free(sc->tids.ftid_tab, M_CXGBE);
 933         t4_destroy_dma_tag(sc);
 934         if (mtx_initialized(&sc->sc_lock)) {
 935                 sx_xlock(&t4_list_lock);
 936                 SLIST_REMOVE(&t4_list, sc, adapter, link);
 937                 sx_xunlock(&t4_list_lock);
 938                 mtx_destroy(&sc->sc_lock);
 939         }
 940
 941         if (mtx_initialized(&sc->tids.ftid_lock))
 942                 mtx_destroy(&sc->tids.ftid_lock);
 943         if (mtx_initialized(&sc->sfl_lock))
 944                 mtx_destroy(&sc->sfl_lock);
 945         if (mtx_initialized(&sc->ifp_lock))
 946                 mtx_destroy(&sc->ifp_lock);
 947
 948         bzero(sc, sizeof(*sc));
 949
 950         return (0);
 951 }
 952
 953
 954 static int
 955 cxgbe_probe(device_t dev)
 956 {
 957         char buf[128];
 958         struct port_info *pi = device_get_softc(dev);
 959
 960         snprintf(buf, sizeof(buf), "port %d", pi->port_id);
 961         device_set_desc_copy(dev, buf);
 962
 963         return (BUS_PROBE_DEFAULT);
 964 }
 965
 966 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
 967     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
 968     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
 969 #define T4_CAP_ENABLE (T4_CAP)
 970
 971 static int
 972 cxgbe_attach(device_t dev)
 973 {
 974         struct port_info *pi = device_get_softc(dev);
 975         struct ifnet *ifp;
 976
 977         /* Allocate an ifnet and set it up */
 978         ifp = if_alloc(IFT_ETHER);
 979         if (ifp == NULL) {
 980                 device_printf(dev, "Cannot allocate ifnet\n");
 981                 return (ENOMEM);
 982         }
 983         pi->ifp = ifp;
 984         ifp->if_softc = pi;
 985
 986         callout_init(&pi->tick, CALLOUT_MPSAFE);
 987
 988         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 989         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 990
 991         ifp->if_init = cxgbe_init;
 992         ifp->if_ioctl = cxgbe_ioctl;
 993         ifp->if_transmit = cxgbe_transmit;
 994         ifp->if_qflush = cxgbe_qflush;
 995
 996         ifp->if_capabilities = T4_CAP;
 997 #ifdef TCP_OFFLOAD
 998         if (is_offload(pi->adapter))
 999                 ifp->if_capabilities |= IFCAP_TOE;
1000 #endif
1001         ifp->if_capenable = T4_CAP_ENABLE;
1002         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1003             CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1004
1005         /* Initialize ifmedia for this port */
1006         ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1007             cxgbe_media_status);
1008         build_medialist(pi);
1009
1010         pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1011             EVENTHANDLER_PRI_ANY);
1012
1013         ether_ifattach(ifp, pi->hw_addr);
1014
1015 #ifdef TCP_OFFLOAD
1016         if (is_offload(pi->adapter)) {
1017                 device_printf(dev,
1018                     "%d txq, %d rxq (NIC); %d txq, %d rxq (TOE)\n",
1019                     pi->ntxq, pi->nrxq, pi->nofldtxq, pi->nofldrxq);
1020         } else
1021 #endif
1022                 device_printf(dev, "%d txq, %d rxq\n", pi->ntxq, pi->nrxq);
1023
1024         cxgbe_sysctls(pi);
1025
1026         return (0);
1027 }
1028
1029 static int
1030 cxgbe_detach(device_t dev)
1031 {
1032         struct port_info *pi = device_get_softc(dev);
1033         struct adapter *sc = pi->adapter;
1034         struct ifnet *ifp = pi->ifp;
1035
1036         /* Tell if_ioctl and if_init that the port is going away */
1037         ADAPTER_LOCK(sc);
1038         SET_DOOMED(pi);
1039         wakeup(&sc->flags);
1040         while (IS_BUSY(sc))
1041                 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
1042         SET_BUSY(sc);
1043 #ifdef INVARIANTS
1044         sc->last_op = "t4detach";
1045         sc->last_op_thr = curthread;
1046 #endif
1047         ADAPTER_UNLOCK(sc);
1048
1049         if (pi->flags & HAS_TRACEQ) {
1050                 sc->traceq = -1;        /* cloner should not create ifnet */
1051                 t4_tracer_port_detach(sc);
1052         }
1053
1054         if (pi->vlan_c)
1055                 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
1056
1057         PORT_LOCK(pi);
1058         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1059         callout_stop(&pi->tick);
1060         PORT_UNLOCK(pi);
1061         callout_drain(&pi->tick);
1062
1063         /* Let detach proceed even if these fail. */
1064         cxgbe_uninit_synchronized(pi);
1065         port_full_uninit(pi);
1066
1067         ifmedia_removeall(&pi->media);
1068         ether_ifdetach(pi->ifp);
1069         if_free(pi->ifp);
1070
1071         ADAPTER_LOCK(sc);
1072         CLR_BUSY(sc);
1073         wakeup(&sc->flags);
1074         ADAPTER_UNLOCK(sc);
1075
1076         return (0);
1077 }
1078
1079 static void
1080 cxgbe_init(void *arg)
1081 {
1082         struct port_info *pi = arg;
1083         struct adapter *sc = pi->adapter;
1084
1085         if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
1086                 return;
1087         cxgbe_init_synchronized(pi);
1088         end_synchronized_op(sc, 0);
1089 }
1090
1091 static int
1092 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1093 {
1094         int rc = 0, mtu, flags;
1095         struct port_info *pi = ifp->if_softc;
1096         struct adapter *sc = pi->adapter;
1097         struct ifreq *ifr = (struct ifreq *)data;
1098         uint32_t mask;
1099
1100         switch (cmd) {
1101         case SIOCSIFMTU:
1102                 mtu = ifr->ifr_mtu;
1103                 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1104                         return (EINVAL);
1105
1106                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
1107                 if (rc)
1108                         return (rc);
1109                 ifp->if_mtu = mtu;
1110                 if (pi->flags & PORT_INIT_DONE) {
1111                         t4_update_fl_bufsize(ifp);
1112                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1113                                 rc = update_mac_settings(pi, XGMAC_MTU);
1114                 }
1115                 end_synchronized_op(sc, 0);
1116                 break;
1117
1118         case SIOCSIFFLAGS:
1119                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4flg");
1120                 if (rc)
1121                         return (rc);
1122
1123                 if (ifp->if_flags & IFF_UP) {
1124                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1125                                 flags = pi->if_flags;
1126                                 if ((ifp->if_flags ^ flags) &
1127                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1128                                         rc = update_mac_settings(pi,
1129                                             XGMAC_PROMISC | XGMAC_ALLMULTI);
1130                                 }
1131                         } else
1132                                 rc = cxgbe_init_synchronized(pi);
1133                         pi->if_flags = ifp->if_flags;
1134                 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1135                         rc = cxgbe_uninit_synchronized(pi);
1136                 end_synchronized_op(sc, 0);
1137                 break;
1138
1139         case SIOCADDMULTI:
1140         case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1141                 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
1142                 if (rc)
1143                         return (rc);
1144                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1145                         rc = update_mac_settings(pi, XGMAC_MCADDRS);
1146                 end_synchronized_op(sc, LOCK_HELD);
1147                 break;
1148
1149         case SIOCSIFCAP:
1150                 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
1151                 if (rc)
1152                         return (rc);
1153
1154                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1155                 if (mask & IFCAP_TXCSUM) {
1156                         ifp->if_capenable ^= IFCAP_TXCSUM;
1157                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1158
1159                         if (IFCAP_TSO4 & ifp->if_capenable &&
1160                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1161                                 ifp->if_capenable &= ~IFCAP_TSO4;
1162                                 if_printf(ifp,
1163                                     "tso4 disabled due to -txcsum.\n");
1164                         }
1165                 }
1166                 if (mask & IFCAP_TXCSUM_IPV6) {
1167                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1168                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1169
1170                         if (IFCAP_TSO6 & ifp->if_capenable &&
1171                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1172                                 ifp->if_capenable &= ~IFCAP_TSO6;
1173                                 if_printf(ifp,
1174                                     "tso6 disabled due to -txcsum6.\n");
1175                         }
1176                 }
1177                 if (mask & IFCAP_RXCSUM)
1178                         ifp->if_capenable ^= IFCAP_RXCSUM;
1179                 if (mask & IFCAP_RXCSUM_IPV6)
1180                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1181
1182                 /*
1183                  * Note that we leave CSUM_TSO alone (it is always set).  The
1184                  * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1185                  * sending a TSO request our way, so it's sufficient to toggle
1186                  * IFCAP_TSOx only.
1187                  */
1188                 if (mask & IFCAP_TSO4) {
1189                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1190                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
1191                                 if_printf(ifp, "enable txcsum first.\n");
1192                                 rc = EAGAIN;
1193                                 goto fail;
1194                         }
1195                         ifp->if_capenable ^= IFCAP_TSO4;
1196                 }
1197                 if (mask & IFCAP_TSO6) {
1198                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1199                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1200                                 if_printf(ifp, "enable txcsum6 first.\n");
1201                                 rc = EAGAIN;
1202                                 goto fail;
1203                         }
1204                         ifp->if_capenable ^= IFCAP_TSO6;
1205                 }
1206                 if (mask & IFCAP_LRO) {
1207 #if defined(INET) || defined(INET6)
1208                         int i;
1209                         struct sge_rxq *rxq;
1210
1211                         ifp->if_capenable ^= IFCAP_LRO;
1212                         for_each_rxq(pi, i, rxq) {
1213                                 if (ifp->if_capenable & IFCAP_LRO)
1214                                         rxq->iq.flags |= IQ_LRO_ENABLED;
1215                                 else
1216                                         rxq->iq.flags &= ~IQ_LRO_ENABLED;
1217                         }
1218 #endif
1219                 }
1220 #ifdef TCP_OFFLOAD
1221                 if (mask & IFCAP_TOE) {
1222                         int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1223
1224                         rc = toe_capability(pi, enable);
1225                         if (rc != 0)
1226                                 goto fail;
1227
1228                         ifp->if_capenable ^= mask;
1229                 }
1230 #endif
1231                 if (mask & IFCAP_VLAN_HWTAGGING) {
1232                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1233                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1234                                 rc = update_mac_settings(pi, XGMAC_VLANEX);
1235                 }
1236                 if (mask & IFCAP_VLAN_MTU) {
1237                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
1238
1239                         /* Need to find out how to disable auto-mtu-inflation */
1240                 }
1241                 if (mask & IFCAP_VLAN_HWTSO)
1242                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1243                 if (mask & IFCAP_VLAN_HWCSUM)
1244                         ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1245
1246 #ifdef VLAN_CAPABILITIES
1247                 VLAN_CAPABILITIES(ifp);
1248 #endif
1249 fail:
1250                 end_synchronized_op(sc, 0);
1251                 break;
1252
1253         case SIOCSIFMEDIA:
1254         case SIOCGIFMEDIA:
1255                 ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1256                 break;
1257
1258         default:
1259                 rc = ether_ioctl(ifp, cmd, data);
1260         }
1261
1262         return (rc);
1263 }
1264
1265 static int
1266 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1267 {
1268         struct port_info *pi = ifp->if_softc;
1269         struct adapter *sc = pi->adapter;
1270         struct sge_txq *txq = &sc->sge.txq[pi->first_txq];
1271         struct buf_ring *br;
1272         int rc;
1273
1274         M_ASSERTPKTHDR(m);
1275
1276         if (__predict_false(pi->link_cfg.link_ok == 0)) {
1277                 m_freem(m);
1278                 return (ENETDOWN);
1279         }
1280
1281         if (m->m_flags & M_FLOWID)
1282                 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq))
1283                     + pi->rsrv_noflowq);
1284         br = txq->br;
1285
1286         if (TXQ_TRYLOCK(txq) == 0) {
1287                 struct sge_eq *eq = &txq->eq;
1288
1289                 /*
1290                  * It is possible that t4_eth_tx finishes up and releases the
1291                  * lock between the TRYLOCK above and the drbr_enqueue here.  We
1292                  * need to make sure that this mbuf doesn't just sit there in
1293                  * the drbr.
1294                  */
1295
1296                 rc = drbr_enqueue(ifp, br, m);
1297                 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 &&
1298                     !(eq->flags & EQ_DOOMED))
1299                         callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq);
1300                 return (rc);
1301         }
1302
1303         /*
1304          * txq->m is the mbuf that is held up due to a temporary shortage of
1305          * resources and it should be put on the wire first.  Then what's in
1306          * drbr and finally the mbuf that was just passed in to us.
1307          *
1308          * Return code should indicate the fate of the mbuf that was passed in
1309          * this time.
1310          */
1311
1312         TXQ_LOCK_ASSERT_OWNED(txq);
1313         if (drbr_needs_enqueue(ifp, br) || txq->m) {
1314
1315                 /* Queued for transmission. */
1316
1317                 rc = drbr_enqueue(ifp, br, m);
1318                 m = txq->m ? txq->m : drbr_dequeue(ifp, br);
1319                 (void) t4_eth_tx(ifp, txq, m);
1320                 TXQ_UNLOCK(txq);
1321                 return (rc);
1322         }
1323
1324         /* Direct transmission. */
1325         rc = t4_eth_tx(ifp, txq, m);
1326         if (rc != 0 && txq->m)
1327                 rc = 0; /* held, will be transmitted soon (hopefully) */
1328
1329         TXQ_UNLOCK(txq);
1330         return (rc);
1331 }
1332
1333 static void
1334 cxgbe_qflush(struct ifnet *ifp)
1335 {
1336         struct port_info *pi = ifp->if_softc;
1337         struct sge_txq *txq;
1338         int i;
1339         struct mbuf *m;
1340
1341         /* queues do not exist if !PORT_INIT_DONE. */
1342         if (pi->flags & PORT_INIT_DONE) {
1343                 for_each_txq(pi, i, txq) {
1344                         TXQ_LOCK(txq);
1345                         m_freem(txq->m);
1346                         txq->m = NULL;
1347                         while ((m = buf_ring_dequeue_sc(txq->br)) != NULL)
1348                                 m_freem(m);
1349                         TXQ_UNLOCK(txq);
1350                 }
1351         }
1352         if_qflush(ifp);
1353 }
1354
1355 static int
1356 cxgbe_media_change(struct ifnet *ifp)
1357 {
1358         struct port_info *pi = ifp->if_softc;
1359
1360         device_printf(pi->dev, "%s unimplemented.\n", __func__);
1361
1362         return (EOPNOTSUPP);
1363 }
1364
1365 static void
1366 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1367 {
1368         struct port_info *pi = ifp->if_softc;
1369         struct ifmedia_entry *cur = pi->media.ifm_cur;
1370         int speed = pi->link_cfg.speed;
1371         int data = (pi->port_type << 8) | pi->mod_type;
1372
1373         if (cur->ifm_data != data) {
1374                 build_medialist(pi);
1375                 cur = pi->media.ifm_cur;
1376         }
1377
1378         ifmr->ifm_status = IFM_AVALID;
1379         if (!pi->link_cfg.link_ok)
1380                 return;
1381
1382         ifmr->ifm_status |= IFM_ACTIVE;
1383
1384         /* active and current will differ iff current media is autoselect. */
1385         if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1386                 return;
1387
1388         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1389         if (speed == SPEED_10000)
1390                 ifmr->ifm_active |= IFM_10G_T;
1391         else if (speed == SPEED_1000)
1392                 ifmr->ifm_active |= IFM_1000_T;
1393         else if (speed == SPEED_100)
1394                 ifmr->ifm_active |= IFM_100_TX;
1395         else if (speed == SPEED_10)
1396                 ifmr->ifm_active |= IFM_10_T;
1397         else
1398                 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1399                             speed));
1400 }
1401
1402 void
1403 t4_fatal_err(struct adapter *sc)
1404 {
1405         t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1406         t4_intr_disable(sc);
1407         log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1408             device_get_nameunit(sc->dev));
1409 }
1410
1411 static int
1412 map_bars_0_and_4(struct adapter *sc)
1413 {
1414         sc->regs_rid = PCIR_BAR(0);
1415         sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1416             &sc->regs_rid, RF_ACTIVE);
1417         if (sc->regs_res == NULL) {
1418                 device_printf(sc->dev, "cannot map registers.\n");
1419                 return (ENXIO);
1420         }
1421         sc->bt = rman_get_bustag(sc->regs_res);
1422         sc->bh = rman_get_bushandle(sc->regs_res);
1423         sc->mmio_len = rman_get_size(sc->regs_res);
1424         setbit(&sc->doorbells, DOORBELL_KDB);
1425
1426         sc->msix_rid = PCIR_BAR(4);
1427         sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1428             &sc->msix_rid, RF_ACTIVE);
1429         if (sc->msix_res == NULL) {
1430                 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1431                 return (ENXIO);
1432         }
1433
1434         return (0);
1435 }
1436
1437 static int
1438 map_bar_2(struct adapter *sc)
1439 {
1440
1441         /*
1442          * T4: only iWARP driver uses the userspace doorbells.  There is no need
1443          * to map it if RDMA is disabled.
1444          */
1445         if (is_t4(sc) && sc->rdmacaps == 0)
1446                 return (0);
1447
1448         sc->udbs_rid = PCIR_BAR(2);
1449         sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1450             &sc->udbs_rid, RF_ACTIVE);
1451         if (sc->udbs_res == NULL) {
1452                 device_printf(sc->dev, "cannot map doorbell BAR.\n");
1453                 return (ENXIO);
1454         }
1455         sc->udbs_base = rman_get_virtual(sc->udbs_res);
1456
1457         if (is_t5(sc)) {
1458                 setbit(&sc->doorbells, DOORBELL_UDB);
1459 #if defined(__i386__) || defined(__amd64__)
1460                 if (t5_write_combine) {
1461                         int rc;
1462
1463                         /*
1464                          * Enable write combining on BAR2.  This is the
1465                          * userspace doorbell BAR and is split into 128B
1466                          * (UDBS_SEG_SIZE) doorbell regions, each associated
1467                          * with an egress queue.  The first 64B has the doorbell
1468                          * and the second 64B can be used to submit a tx work
1469                          * request with an implicit doorbell.
1470                          */
1471
1472                         rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
1473                             rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
1474                         if (rc == 0) {
1475                                 clrbit(&sc->doorbells, DOORBELL_UDB);
1476                                 setbit(&sc->doorbells, DOORBELL_WCWR);
1477                                 setbit(&sc->doorbells, DOORBELL_UDBWC);
1478                         } else {
1479                                 device_printf(sc->dev,
1480                                     "couldn't enable write combining: %d\n",
1481                                     rc);
1482                         }
1483
1484                         t4_write_reg(sc, A_SGE_STAT_CFG,
1485                             V_STATSOURCE_T5(7) | V_STATMODE(0));
1486                 }
1487 #endif
1488         }
1489
1490         return (0);
1491 }
1492
1493 static const struct memwin t4_memwin[] = {
1494         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1495         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1496         { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
1497 };
1498
1499 static const struct memwin t5_memwin[] = {
1500         { MEMWIN0_BASE, MEMWIN0_APERTURE },
1501         { MEMWIN1_BASE, MEMWIN1_APERTURE },
1502         { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1503 };
1504
1505 static void
1506 setup_memwin(struct adapter *sc)
1507 {
1508         const struct memwin *mw;
1509         int i, n;
1510         uint32_t bar0;
1511
1512         if (is_t4(sc)) {
1513                 /*
1514                  * Read low 32b of bar0 indirectly via the hardware backdoor
1515                  * mechanism.  Works from within PCI passthrough environments
1516                  * too, where rman_get_start() can return a different value.  We
1517                  * need to program the T4 memory window decoders with the actual
1518                  * addresses that will be coming across the PCIe link.
1519                  */
1520                 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1521                 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1522
1523                 mw = &t4_memwin[0];
1524                 n = nitems(t4_memwin);
1525         } else {
1526                 /* T5 uses the relative offset inside the PCIe BAR */
1527                 bar0 = 0;
1528
1529                 mw = &t5_memwin[0];
1530                 n = nitems(t5_memwin);
1531         }
1532
1533         for (i = 0; i < n; i++, mw++) {
1534                 t4_write_reg(sc,
1535                     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
1536                     (mw->base + bar0) | V_BIR(0) |
1537                     V_WINDOW(ilog2(mw->aperture) - 10));
1538         }
1539
1540         /* flush */
1541         t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1542 }
1543
1544 /*
1545  * Verify that the memory range specified by the addr/len pair is valid and lies
1546  * entirely within a single region (EDCx or MCx).
1547  */
1548 static int
1549 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
1550 {
1551         uint32_t em, addr_len, maddr, mlen;
1552
1553         /* Memory can only be accessed in naturally aligned 4 byte units */
1554         if (addr & 3 || len & 3 || len == 0)
1555                 return (EINVAL);
1556
1557         /* Enabled memories */
1558         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1559         if (em & F_EDRAM0_ENABLE) {
1560                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1561                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1562                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1563                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1564                     addr + len <= maddr + mlen)
1565                         return (0);
1566         }
1567         if (em & F_EDRAM1_ENABLE) {
1568                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1569                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1570                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1571                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1572                     addr + len <= maddr + mlen)
1573                         return (0);
1574         }
1575         if (em & F_EXT_MEM_ENABLE) {
1576                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1577                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1578                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1579                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1580                     addr + len <= maddr + mlen)
1581                         return (0);
1582         }
1583         if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
1584                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1585                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1586                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1587                 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1588                     addr + len <= maddr + mlen)
1589                         return (0);
1590         }
1591
1592         return (EFAULT);
1593 }
1594
1595 static int
1596 fwmtype_to_hwmtype(int mtype)
1597 {
1598
1599         switch (mtype) {
1600         case FW_MEMTYPE_EDC0:
1601                 return (MEM_EDC0);
1602         case FW_MEMTYPE_EDC1:
1603                 return (MEM_EDC1);
1604         case FW_MEMTYPE_EXTMEM:
1605                 return (MEM_MC0);
1606         case FW_MEMTYPE_EXTMEM1:
1607                 return (MEM_MC1);
1608         default:
1609                 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
1610         }
1611 }
1612
1613 /*
1614  * Verify that the memory range specified by the memtype/offset/len pair is
1615  * valid and lies entirely within the memtype specified.  The global address of
1616  * the start of the range is returned in addr.
1617  */
1618 static int
1619 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1620     uint32_t *addr)
1621 {
1622         uint32_t em, addr_len, maddr, mlen;
1623
1624         /* Memory can only be accessed in naturally aligned 4 byte units */
1625         if (off & 3 || len & 3 || len == 0)
1626                 return (EINVAL);
1627
1628         em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1629         switch (fwmtype_to_hwmtype(mtype)) {
1630         case MEM_EDC0:
1631                 if (!(em & F_EDRAM0_ENABLE))
1632                         return (EINVAL);
1633                 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1634                 maddr = G_EDRAM0_BASE(addr_len) << 20;
1635                 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1636                 break;
1637         case MEM_EDC1:
1638                 if (!(em & F_EDRAM1_ENABLE))
1639                         return (EINVAL);
1640                 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1641                 maddr = G_EDRAM1_BASE(addr_len) << 20;
1642                 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1643                 break;
1644         case MEM_MC:
1645                 if (!(em & F_EXT_MEM_ENABLE))
1646                         return (EINVAL);
1647                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1648                 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1649                 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1650                 break;
1651         case MEM_MC1:
1652                 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
1653                         return (EINVAL);
1654                 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1655                 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1656                 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1657                 break;
1658         default:
1659                 return (EINVAL);
1660         }
1661
1662         if (mlen > 0 && off < mlen && off + len <= mlen) {
1663                 *addr = maddr + off;    /* global address */
1664                 return (0);
1665         }
1666
1667         return (EFAULT);
1668 }
1669
1670 static void
1671 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1672 {
1673         const struct memwin *mw;
1674
1675         if (is_t4(sc)) {
1676                 KASSERT(win >= 0 && win < nitems(t4_memwin),
1677                     ("%s: incorrect memwin# (%d)", __func__, win));
1678                 mw = &t4_memwin[win];
1679         } else {
1680                 KASSERT(win >= 0 && win < nitems(t5_memwin),
1681                     ("%s: incorrect memwin# (%d)", __func__, win));
1682                 mw = &t5_memwin[win];
1683         }
1684
1685         if (base != NULL)
1686                 *base = mw->base;
1687         if (aperture != NULL)
1688                 *aperture = mw->aperture;
1689 }
1690
1691 /*
1692  * Positions the memory window such that it can be used to access the specified
1693  * address in the chip's address space.  The return value is the offset of addr
1694  * from the start of the window.
1695  */
1696 static uint32_t
1697 position_memwin(struct adapter *sc, int n, uint32_t addr)
1698 {
1699         uint32_t start, pf;
1700         uint32_t reg;
1701
1702         KASSERT(n >= 0 && n <= 3,
1703             ("%s: invalid window %d.", __func__, n));
1704         KASSERT((addr & 3) == 0,
1705             ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
1706
1707         if (is_t4(sc)) {
1708                 pf = 0;
1709                 start = addr & ~0xf;    /* start must be 16B aligned */
1710         } else {
1711                 pf = V_PFNUM(sc->pf);
1712                 start = addr & ~0x7f;   /* start must be 128B aligned */
1713         }
1714         reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1715
1716         t4_write_reg(sc, reg, start | pf);
1717         t4_read_reg(sc, reg);
1718
1719         return (addr - start);
1720 }
1721
1722 static int
1723 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1724     struct intrs_and_queues *iaq)
1725 {
1726         int rc, itype, navail, nrxq10g, nrxq1g, n;
1727         int nofldrxq10g = 0, nofldrxq1g = 0;
1728
1729         bzero(iaq, sizeof(*iaq));
1730
1731         iaq->ntxq10g = t4_ntxq10g;
1732         iaq->ntxq1g = t4_ntxq1g;
1733         iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1734         iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1735         iaq->rsrv_noflowq = t4_rsrv_noflowq;
1736 #ifdef TCP_OFFLOAD
1737         if (is_offload(sc)) {
1738                 iaq->nofldtxq10g = t4_nofldtxq10g;
1739                 iaq->nofldtxq1g = t4_nofldtxq1g;
1740                 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1741                 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1742         }
1743 #endif
1744
1745         for (itype = INTR_MSIX; itype; itype >>= 1) {
1746
1747                 if ((itype & t4_intr_types) == 0)
1748                         continue;       /* not allowed */
1749
1750                 if (itype == INTR_MSIX)
1751                         navail = pci_msix_count(sc->dev);
1752                 else if (itype == INTR_MSI)
1753                         navail = pci_msi_count(sc->dev);
1754                 else
1755                         navail = 1;
1756 restart:
1757                 if (navail == 0)
1758                         continue;
1759
1760                 iaq->intr_type = itype;
1761                 iaq->intr_flags = 0;
1762
1763                 /*
1764                  * Best option: an interrupt vector for errors, one for the
1765                  * firmware event queue, and one each for each rxq (NIC as well
1766                  * as offload).
1767                  */
1768                 iaq->nirq = T4_EXTRA_INTR;
1769                 iaq->nirq += n10g * (nrxq10g + nofldrxq10g);
1770                 iaq->nirq += n1g * (nrxq1g + nofldrxq1g);
1771                 if (iaq->nirq <= navail &&
1772                     (itype != INTR_MSI || powerof2(iaq->nirq))) {
1773                         iaq->intr_flags |= INTR_DIRECT;
1774                         goto allocate;
1775                 }
1776
1777                 /*
1778                  * Second best option: an interrupt vector for errors, one for
1779                  * the firmware event queue, and one each for either NIC or
1780                  * offload rxq's.
1781                  */
1782                 iaq->nirq = T4_EXTRA_INTR;
1783                 iaq->nirq += n10g * max(nrxq10g, nofldrxq10g);
1784                 iaq->nirq += n1g * max(nrxq1g, nofldrxq1g);
1785                 if (iaq->nirq <= navail &&
1786                     (itype != INTR_MSI || powerof2(iaq->nirq)))
1787                         goto allocate;
1788
1789                 /*
1790                  * Next best option: an interrupt vector for errors, one for the
1791                  * firmware event queue, and at least one per port.  At this
1792                  * point we know we'll have to downsize nrxq or nofldrxq to fit
1793                  * what's available to us.
1794                  */
1795                 iaq->nirq = T4_EXTRA_INTR;
1796                 iaq->nirq += n10g + n1g;
1797                 if (iaq->nirq <= navail) {
1798                         int leftover = navail - iaq->nirq;
1799
1800                         if (n10g > 0) {
1801                                 int target = max(nrxq10g, nofldrxq10g);
1802
1803                                 n = 1;
1804                                 while (n < target && leftover >= n10g) {
1805                                         leftover -= n10g;
1806                                         iaq->nirq += n10g;
1807                                         n++;
1808                                 }
1809                                 iaq->nrxq10g = min(n, nrxq10g);
1810 #ifdef TCP_OFFLOAD
1811                                 if (is_offload(sc))
1812                                         iaq->nofldrxq10g = min(n, nofldrxq10g);
1813 #endif
1814                         }
1815
1816                         if (n1g > 0) {
1817                                 int target = max(nrxq1g, nofldrxq1g);
1818
1819                                 n = 1;
1820                                 while (n < target && leftover >= n1g) {
1821                                         leftover -= n1g;
1822                                         iaq->nirq += n1g;
1823                                         n++;
1824                                 }
1825                                 iaq->nrxq1g = min(n, nrxq1g);
1826 #ifdef TCP_OFFLOAD
1827                                 if (is_offload(sc))
1828                                         iaq->nofldrxq1g = min(n, nofldrxq1g);
1829 #endif
1830                         }
1831
1832                         if (itype != INTR_MSI || powerof2(iaq->nirq))
1833                                 goto allocate;
1834                 }
1835
1836                 /*
1837                  * Least desirable option: one interrupt vector for everything.
1838                  */
1839                 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
1840 #ifdef TCP_OFFLOAD
1841                 if (is_offload(sc))
1842                         iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
1843 #endif
1844
1845 allocate:
1846                 navail = iaq->nirq;
1847                 rc = 0;
1848                 if (itype == INTR_MSIX)
1849                         rc = pci_alloc_msix(sc->dev, &navail);
1850                 else if (itype == INTR_MSI)
1851                         rc = pci_alloc_msi(sc->dev, &navail);
1852
1853                 if (rc == 0) {
1854                         if (navail == iaq->nirq)
1855                                 return (0);
1856
1857                         /*
1858                          * Didn't get the number requested.  Use whatever number
1859                          * the kernel is willing to allocate (it's in navail).
1860                          */
1861                         device_printf(sc->dev, "fewer vectors than requested, "
1862                             "type=%d, req=%d, rcvd=%d; will downshift req.\n",
1863                             itype, iaq->nirq, navail);
1864                         pci_release_msi(sc->dev);
1865                         goto restart;
1866                 }
1867
1868                 device_printf(sc->dev,
1869                     "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
1870                     itype, rc, iaq->nirq, navail);
1871         }
1872
1873         device_printf(sc->dev,
1874             "failed to find a usable interrupt type.  "
1875             "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
1876             pci_msix_count(sc->dev), pci_msi_count(sc->dev));
1877
1878         return (ENXIO);
1879 }
1880
1881 #define FW_VERSION(chip) ( \
1882     V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
1883     V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
1884     V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
1885     V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
1886 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
1887
1888 struct fw_info {
1889         uint8_t chip;
1890         char *kld_name;
1891         char *fw_mod_name;
1892         struct fw_hdr fw_hdr;   /* XXX: waste of space, need a sparse struct */
1893 } fw_info[] = {
1894         {
1895                 .chip = CHELSIO_T4,
1896                 .kld_name = "t4fw_cfg",
1897                 .fw_mod_name = "t4fw",
1898                 .fw_hdr = {
1899                         .chip = FW_HDR_CHIP_T4,
1900                         .fw_ver = htobe32_const(FW_VERSION(T4)),
1901                         .intfver_nic = FW_INTFVER(T4, NIC),
1902                         .intfver_vnic = FW_INTFVER(T4, VNIC),
1903                         .intfver_ofld = FW_INTFVER(T4, OFLD),
1904                         .intfver_ri = FW_INTFVER(T4, RI),
1905                         .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
1906                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
1907                         .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
1908                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
1909                 },
1910         }, {
1911                 .chip = CHELSIO_T5,
1912                 .kld_name = "t5fw_cfg",
1913                 .fw_mod_name = "t5fw",
1914                 .fw_hdr = {
1915                         .chip = FW_HDR_CHIP_T5,
1916                         .fw_ver = htobe32_const(FW_VERSION(T5)),
1917                         .intfver_nic = FW_INTFVER(T5, NIC),
1918                         .intfver_vnic = FW_INTFVER(T5, VNIC),
1919                         .intfver_ofld = FW_INTFVER(T5, OFLD),
1920                         .intfver_ri = FW_INTFVER(T5, RI),
1921                         .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
1922                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
1923                         .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
1924                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
1925                 },
1926         }
1927 };
1928
1929 static struct fw_info *
1930 find_fw_info(int chip)
1931 {
1932         int i;
1933
1934         for (i = 0; i < nitems(fw_info); i++) {
1935                 if (fw_info[i].chip == chip)
1936                         return (&fw_info[i]);
1937         }
1938         return (NULL);
1939 }
1940
1941 /*
1942  * Is the given firmware API compatible with the one the driver was compiled
1943  * with?
1944  */
1945 static int
1946 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
1947 {
1948
1949         /* short circuit if it's the exact same firmware version */
1950         if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
1951                 return (1);
1952
1953         /*
1954          * XXX: Is this too conservative?  Perhaps I should limit this to the
1955          * features that are supported in the driver.
1956          */
1957 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
1958         if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
1959             SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
1960             SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
1961                 return (1);
1962 #undef SAME_INTF
1963
1964         return (0);
1965 }
1966
1967 /*
1968  * The firmware in the KLD is usable, but should it be installed?  This routine
1969  * explains itself in detail if it indicates the KLD firmware should be
1970  * installed.
1971  */
1972 static int
1973 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
1974 {
1975         const char *reason;
1976
1977         if (!card_fw_usable) {
1978                 reason = "incompatible or unusable";
1979                 goto install;
1980         }
1981
1982         if (k > c) {
1983                 reason = "older than the version bundled with this driver";
1984                 goto install;
1985         }
1986
1987         if (t4_fw_install == 2 && k != c) {
1988                 reason = "different than the version bundled with this driver";
1989                 goto install;
1990         }
1991
1992         return (0);
1993
1994 install:
1995         if (t4_fw_install == 0) {
1996                 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
1997                     "but the driver is prohibited from installing a different "
1998                     "firmware on the card.\n",
1999                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2000                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2001
2002                 return (0);
2003         }
2004
2005         device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2006             "installing firmware %u.%u.%u.%u on card.\n",
2007             G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2008             G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2009             G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2010             G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2011
2012         return (1);
2013 }
2014 /*
2015  * Establish contact with the firmware and determine if we are the master driver
2016  * or not, and whether we are responsible for chip initialization.
2017  */
2018 static int
2019 prep_firmware(struct adapter *sc)
2020 {
2021         const struct firmware *fw = NULL, *default_cfg;
2022         int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2023         enum dev_state state;
2024         struct fw_info *fw_info;
2025         struct fw_hdr *card_fw;         /* fw on the card */
2026         const struct fw_hdr *kld_fw;    /* fw in the KLD */
2027         const struct fw_hdr *drv_fw;    /* fw header the driver was compiled
2028                                            against */
2029
2030         /* Contact firmware. */
2031         rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2032         if (rc < 0 || state == DEV_STATE_ERR) {
2033                 rc = -rc;
2034                 device_printf(sc->dev,
2035                     "failed to connect to the firmware: %d, %d.\n", rc, state);
2036                 return (rc);
2037         }
2038         pf = rc;
2039         if (pf == sc->mbox)
2040                 sc->flags |= MASTER_PF;
2041         else if (state == DEV_STATE_UNINIT) {
2042                 /*
2043                  * We didn't get to be the master so we definitely won't be
2044                  * configuring the chip.  It's a bug if someone else hasn't
2045                  * configured it already.
2046                  */
2047                 device_printf(sc->dev, "couldn't be master(%d), "
2048                     "device not already initialized either(%d).\n", rc, state);
2049                 return (EDOOFUS);
2050         }
2051
2052         /* This is the firmware whose headers the driver was compiled against */
2053         fw_info = find_fw_info(chip_id(sc));
2054         if (fw_info == NULL) {
2055                 device_printf(sc->dev,
2056                     "unable to look up firmware information for chip %d.\n",
2057                     chip_id(sc));
2058                 return (EINVAL);
2059         }
2060         drv_fw = &fw_info->fw_hdr;
2061
2062         /*
2063          * The firmware KLD contains many modules.  The KLD name is also the
2064          * name of the module that contains the default config file.
2065          */
2066         default_cfg = firmware_get(fw_info->kld_name);
2067
2068         /* Read the header of the firmware on the card */
2069         card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2070         rc = -t4_read_flash(sc, FLASH_FW_START,
2071             sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2072         if (rc == 0)
2073                 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2074         else {
2075                 device_printf(sc->dev,
2076                     "Unable to read card's firmware header: %d\n", rc);
2077                 card_fw_usable = 0;
2078         }
2079
2080         /* This is the firmware in the KLD */
2081         fw = firmware_get(fw_info->fw_mod_name);
2082         if (fw != NULL) {
2083                 kld_fw = (const void *)fw->data;
2084                 kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2085         } else {
2086                 kld_fw = NULL;
2087                 kld_fw_usable = 0;
2088         }
2089
2090         if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2091             (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2092                 /*
2093                  * Common case: the firmware on the card is an exact match and
2094                  * the KLD is an exact match too, or the KLD is
2095                  * absent/incompatible.  Note that t4_fw_install = 2 is ignored
2096                  * here -- use cxgbetool loadfw if you want to reinstall the
2097                  * same firmware as the one on the card.
2098                  */
2099         } else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2100             should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2101             be32toh(card_fw->fw_ver))) {
2102
2103                 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
2104                 if (rc != 0) {
2105                         device_printf(sc->dev,
2106                             "failed to install firmware: %d\n", rc);
2107                         goto done;
2108                 }
2109
2110                 /* Installed successfully, update the cached header too. */
2111                 memcpy(card_fw, kld_fw, sizeof(*card_fw));
2112                 card_fw_usable = 1;
2113                 need_fw_reset = 0;      /* already reset as part of load_fw */
2114         }
2115
2116         if (!card_fw_usable) {
2117                 uint32_t d, c, k;
2118
2119                 d = ntohl(drv_fw->fw_ver);
2120                 c = ntohl(card_fw->fw_ver);
2121                 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
2122
2123                 device_printf(sc->dev, "Cannot find a usable firmware: "
2124                     "fw_install %d, chip state %d, "
2125                     "driver compiled with %d.%d.%d.%d, "
2126                     "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
2127                     t4_fw_install, state,
2128                     G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
2129                     G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
2130                     G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2131                     G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
2132                     G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2133                     G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2134                 rc = EINVAL;
2135                 goto done;
2136         }
2137
2138         /* We're using whatever's on the card and it's known to be good. */
2139         sc->params.fw_vers = ntohl(card_fw->fw_ver);
2140         snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
2141             G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2142             G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2143             G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2144             G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2145         t4_get_tp_version(sc, &sc->params.tp_vers);
2146
2147         /* Reset device */
2148         if (need_fw_reset &&
2149             (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
2150                 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
2151                 if (rc != ETIMEDOUT && rc != EIO)
2152                         t4_fw_bye(sc, sc->mbox);
2153                 goto done;
2154         }
2155         sc->flags |= FW_OK;
2156
2157         rc = get_params__pre_init(sc);
2158         if (rc != 0)
2159                 goto done; /* error message displayed already */
2160
2161         /* Partition adapter resources as specified in the config file. */
2162         if (state == DEV_STATE_UNINIT) {
2163
2164                 KASSERT(sc->flags & MASTER_PF,
2165                     ("%s: trying to change chip settings when not master.",
2166                     __func__));
2167
2168                 rc = partition_resources(sc, default_cfg, fw_info->kld_name);
2169                 if (rc != 0)
2170                         goto done;      /* error message displayed already */
2171
2172                 t4_tweak_chip_settings(sc);
2173
2174                 /* get basic stuff going */
2175                 rc = -t4_fw_initialize(sc, sc->mbox);
2176                 if (rc != 0) {
2177                         device_printf(sc->dev, "fw init failed: %d.\n", rc);
2178                         goto done;
2179                 }
2180         } else {
2181                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
2182                 sc->cfcsum = 0;
2183         }
2184
2185 done:
2186         free(card_fw, M_CXGBE);
2187         if (fw != NULL)
2188                 firmware_put(fw, FIRMWARE_UNLOAD);
2189         if (default_cfg != NULL)
2190                 firmware_put(default_cfg, FIRMWARE_UNLOAD);
2191
2192         return (rc);
2193 }
2194
2195 #define FW_PARAM_DEV(param) \
2196         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2197          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2198 #define FW_PARAM_PFVF(param) \
2199         (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2200          V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2201
2202 /*
2203  * Partition chip resources for use between various PFs, VFs, etc.
2204  */
2205 static int
2206 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
2207     const char *name_prefix)
2208 {
2209         const struct firmware *cfg = NULL;
2210         int rc = 0;
2211         struct fw_caps_config_cmd caps;
2212         uint32_t mtype, moff, finicsum, cfcsum;
2213
2214         /*
2215          * Figure out what configuration file to use.  Pick the default config
2216          * file for the card if the user hasn't specified one explicitly.
2217          */
2218         snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
2219         if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
2220                 /* Card specific overrides go here. */
2221                 if (pci_get_device(sc->dev) == 0x440a)
2222                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
2223                 if (is_fpga(sc))
2224                         snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
2225         }
2226
2227         /*
2228          * We need to load another module if the profile is anything except
2229          * "default" or "flash".
2230          */
2231         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
2232             strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2233                 char s[32];
2234
2235                 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
2236                 cfg = firmware_get(s);
2237                 if (cfg == NULL) {
2238                         if (default_cfg != NULL) {
2239                                 device_printf(sc->dev,
2240                                     "unable to load module \"%s\" for "
2241                                     "configuration profile \"%s\", will use "
2242                                     "the default config file instead.\n",
2243                                     s, sc->cfg_file);
2244                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2245                                     "%s", DEFAULT_CF);
2246                         } else {
2247                                 device_printf(sc->dev,
2248                                     "unable to load module \"%s\" for "
2249                                     "configuration profile \"%s\", will use "
2250                                     "the config file on the card's flash "
2251                                     "instead.\n", s, sc->cfg_file);
2252                                 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2253                                     "%s", FLASH_CF);
2254                         }
2255                 }
2256         }
2257
2258         if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
2259             default_cfg == NULL) {
2260                 device_printf(sc->dev,
2261                     "default config file not available, will use the config "
2262                     "file on the card's flash instead.\n");
2263                 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
2264         }
2265
2266         if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2267                 u_int cflen, i, n;
2268                 const uint32_t *cfdata;
2269                 uint32_t param, val, addr, off, mw_base, mw_aperture;
2270
2271                 KASSERT(cfg != NULL || default_cfg != NULL,
2272                     ("%s: no config to upload", __func__));
2273
2274                 /*
2275                  * Ask the firmware where it wants us to upload the config file.
2276                  */
2277                 param = FW_PARAM_DEV(CF);
2278                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2279                 if (rc != 0) {
2280                         /* No support for config file?  Shouldn't happen. */
2281                         device_printf(sc->dev,
2282                             "failed to query config file location: %d.\n", rc);
2283                         goto done;
2284                 }
2285                 mtype = G_FW_PARAMS_PARAM_Y(val);
2286                 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
2287
2288                 /*
2289                  * XXX: sheer laziness.  We deliberately added 4 bytes of
2290                  * useless stuffing/comments at the end of the config file so
2291                  * it's ok to simply throw away the last remaining bytes when
2292                  * the config file is not an exact multiple of 4.  This also
2293                  * helps with the validate_mt_off_len check.
2294                  */
2295                 if (cfg != NULL) {
2296                         cflen = cfg->datasize & ~3;
2297                         cfdata = cfg->data;
2298                 } else {
2299                         cflen = default_cfg->datasize & ~3;
2300                         cfdata = default_cfg->data;
2301                 }
2302
2303                 if (cflen > FLASH_CFG_MAX_SIZE) {
2304                         device_printf(sc->dev,
2305                             "config file too long (%d, max allowed is %d).  "
2306                             "Will try to use the config on the card, if any.\n",
2307                             cflen, FLASH_CFG_MAX_SIZE);
2308                         goto use_config_on_flash;
2309                 }
2310
2311                 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
2312                 if (rc != 0) {
2313                         device_printf(sc->dev,
2314                             "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
2315                             "Will try to use the config on the card, if any.\n",
2316                             __func__, mtype, moff, cflen, rc);
2317                         goto use_config_on_flash;
2318                 }
2319
2320                 memwin_info(sc, 2, &mw_base, &mw_aperture);
2321                 while (cflen) {
2322                         off = position_memwin(sc, 2, addr);
2323                         n = min(cflen, mw_aperture - off);
2324                         for (i = 0; i < n; i += 4)
2325                                 t4_write_reg(sc, mw_base + off + i, *cfdata++);
2326                         cflen -= n;
2327                         addr += n;
2328                 }
2329         } else {
2330 use_config_on_flash:
2331                 mtype = FW_MEMTYPE_FLASH;
2332                 moff = t4_flash_cfg_addr(sc);
2333         }
2334
2335         bzero(&caps, sizeof(caps));
2336         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2337             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2338         caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
2339             V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
2340             V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
2341         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2342         if (rc != 0) {
2343                 device_printf(sc->dev,
2344                     "failed to pre-process config file: %d "
2345                     "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
2346                 goto done;
2347         }
2348
2349         finicsum = be32toh(caps.finicsum);
2350         cfcsum = be32toh(caps.cfcsum);
2351         if (finicsum != cfcsum) {
2352                 device_printf(sc->dev,
2353                     "WARNING: config file checksum mismatch: %08x %08x\n",
2354                     finicsum, cfcsum);
2355         }
2356         sc->cfcsum = cfcsum;
2357
2358 #define LIMIT_CAPS(x) do { \
2359         caps.x &= htobe16(t4_##x##_allowed); \
2360         sc->x = htobe16(caps.x); \
2361 } while (0)
2362
2363         /*
2364          * Let the firmware know what features will (not) be used so it can tune
2365          * things accordingly.
2366          */
2367         LIMIT_CAPS(linkcaps);
2368         LIMIT_CAPS(niccaps);
2369         LIMIT_CAPS(toecaps);
2370         LIMIT_CAPS(rdmacaps);
2371         LIMIT_CAPS(iscsicaps);
2372         LIMIT_CAPS(fcoecaps);
2373 #undef LIMIT_CAPS
2374
2375         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2376             F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
2377         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2378         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
2379         if (rc != 0) {
2380                 device_printf(sc->dev,
2381                     "failed to process config file: %d.\n", rc);
2382         }
2383 done:
2384         if (cfg != NULL)
2385                 firmware_put(cfg, FIRMWARE_UNLOAD);
2386         return (rc);
2387 }
2388
2389 /*
2390  * Retrieve parameters that are needed (or nice to have) very early.
2391  */
2392 static int
2393 get_params__pre_init(struct adapter *sc)
2394 {
2395         int rc;
2396         uint32_t param[2], val[2];
2397         struct fw_devlog_cmd cmd;
2398         struct devlog_params *dlog = &sc->params.devlog;
2399
2400         param[0] = FW_PARAM_DEV(PORTVEC);
2401         param[1] = FW_PARAM_DEV(CCLK);
2402         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2403         if (rc != 0) {
2404                 device_printf(sc->dev,
2405                     "failed to query parameters (pre_init): %d.\n", rc);
2406                 return (rc);
2407         }
2408
2409         sc->params.portvec = val[0];
2410         sc->params.nports = bitcount32(val[0]);
2411         sc->params.vpd.cclk = val[1];
2412
2413         /* Read device log parameters. */
2414         bzero(&cmd, sizeof(cmd));
2415         cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
2416             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2417         cmd.retval_len16 = htobe32(FW_LEN16(cmd));
2418         rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
2419         if (rc != 0) {
2420                 device_printf(sc->dev,
2421                     "failed to get devlog parameters: %d.\n", rc);
2422                 bzero(dlog, sizeof (*dlog));
2423                 rc = 0; /* devlog isn't critical for device operation */
2424         } else {
2425                 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
2426                 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
2427                 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
2428                 dlog->size = be32toh(cmd.memsize_devlog);
2429         }
2430
2431         return (rc);
2432 }
2433
2434 /*
2435  * Retrieve various parameters that are of interest to the driver.  The device
2436  * has been initialized by the firmware at this point.
2437  */
2438 static int
2439 get_params__post_init(struct adapter *sc)
2440 {
2441         int rc;
2442         uint32_t param[7], val[7];
2443         struct fw_caps_config_cmd caps;
2444
2445         param[0] = FW_PARAM_PFVF(IQFLINT_START);
2446         param[1] = FW_PARAM_PFVF(EQ_START);
2447         param[2] = FW_PARAM_PFVF(FILTER_START);
2448         param[3] = FW_PARAM_PFVF(FILTER_END);
2449         param[4] = FW_PARAM_PFVF(L2T_START);
2450         param[5] = FW_PARAM_PFVF(L2T_END);
2451         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2452         if (rc != 0) {
2453                 device_printf(sc->dev,
2454                     "failed to query parameters (post_init): %d.\n", rc);
2455                 return (rc);
2456         }
2457
2458         sc->sge.iq_start = val[0];
2459         sc->sge.eq_start = val[1];
2460         sc->tids.ftid_base = val[2];
2461         sc->tids.nftids = val[3] - val[2] + 1;
2462         sc->vres.l2t.start = val[4];
2463         sc->vres.l2t.size = val[5] - val[4] + 1;
2464         KASSERT(sc->vres.l2t.size <= L2T_SIZE,
2465             ("%s: L2 table size (%u) larger than expected (%u)",
2466             __func__, sc->vres.l2t.size, L2T_SIZE));
2467
2468         /* get capabilites */
2469         bzero(&caps, sizeof(caps));
2470         caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2471             F_FW_CMD_REQUEST | F_FW_CMD_READ);
2472         caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2473         rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2474         if (rc != 0) {
2475                 device_printf(sc->dev,
2476                     "failed to get card capabilities: %d.\n", rc);
2477                 return (rc);
2478         }
2479
2480         if (caps.toecaps) {
2481                 /* query offload-related parameters */
2482                 param[0] = FW_PARAM_DEV(NTID);
2483                 param[1] = FW_PARAM_PFVF(SERVER_START);
2484                 param[2] = FW_PARAM_PFVF(SERVER_END);
2485                 param[3] = FW_PARAM_PFVF(TDDP_START);
2486                 param[4] = FW_PARAM_PFVF(TDDP_END);
2487                 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2488                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2489                 if (rc != 0) {
2490                         device_printf(sc->dev,
2491                             "failed to query TOE parameters: %d.\n", rc);
2492                         return (rc);
2493                 }
2494                 sc->tids.ntids = val[0];
2495                 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
2496                 sc->tids.stid_base = val[1];
2497                 sc->tids.nstids = val[2] - val[1] + 1;
2498                 sc->vres.ddp.start = val[3];
2499                 sc->vres.ddp.size = val[4] - val[3] + 1;
2500                 sc->params.ofldq_wr_cred = val[5];
2501                 sc->params.offload = 1;
2502         }
2503         if (caps.rdmacaps) {
2504                 param[0] = FW_PARAM_PFVF(STAG_START);
2505                 param[1] = FW_PARAM_PFVF(STAG_END);
2506                 param[2] = FW_PARAM_PFVF(RQ_START);
2507                 param[3] = FW_PARAM_PFVF(RQ_END);
2508                 param[4] = FW_PARAM_PFVF(PBL_START);
2509                 param[5] = FW_PARAM_PFVF(PBL_END);
2510                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2511                 if (rc != 0) {
2512                         device_printf(sc->dev,
2513                             "failed to query RDMA parameters(1): %d.\n", rc);
2514                         return (rc);
2515                 }
2516                 sc->vres.stag.start = val[0];
2517                 sc->vres.stag.size = val[1] - val[0] + 1;
2518                 sc->vres.rq.start = val[2];
2519                 sc->vres.rq.size = val[3] - val[2] + 1;
2520                 sc->vres.pbl.start = val[4];
2521                 sc->vres.pbl.size = val[5] - val[4] + 1;
2522
2523                 param[0] = FW_PARAM_PFVF(SQRQ_START);
2524                 param[1] = FW_PARAM_PFVF(SQRQ_END);
2525                 param[2] = FW_PARAM_PFVF(CQ_START);
2526                 param[3] = FW_PARAM_PFVF(CQ_END);
2527                 param[4] = FW_PARAM_PFVF(OCQ_START);
2528                 param[5] = FW_PARAM_PFVF(OCQ_END);
2529                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2530                 if (rc != 0) {
2531                         device_printf(sc->dev,
2532                             "failed to query RDMA parameters(2): %d.\n", rc);
2533                         return (rc);
2534                 }
2535                 sc->vres.qp.start = val[0];
2536                 sc->vres.qp.size = val[1] - val[0] + 1;
2537                 sc->vres.cq.start = val[2];
2538                 sc->vres.cq.size = val[3] - val[2] + 1;
2539                 sc->vres.ocq.start = val[4];
2540                 sc->vres.ocq.size = val[5] - val[4] + 1;
2541         }
2542         if (caps.iscsicaps) {
2543                 param[0] = FW_PARAM_PFVF(ISCSI_START);
2544                 param[1] = FW_PARAM_PFVF(ISCSI_END);
2545                 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2546                 if (rc != 0) {
2547                         device_printf(sc->dev,
2548                             "failed to query iSCSI parameters: %d.\n", rc);
2549                         return (rc);
2550                 }
2551                 sc->vres.iscsi.start = val[0];
2552                 sc->vres.iscsi.size = val[1] - val[0] + 1;
2553         }
2554
2555         /*
2556          * We've got the params we wanted to query via the firmware.  Now grab
2557          * some others directly from the chip.
2558          */
2559         rc = t4_read_chip_settings(sc);
2560
2561         return (rc);
2562 }
2563
2564 static int
2565 set_params__post_init(struct adapter *sc)
2566 {
2567         uint32_t param, val;
2568
2569         /* ask for encapsulated CPLs */
2570         param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
2571         val = 1;
2572         (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2573
2574         return (0);
2575 }
2576
2577 #undef FW_PARAM_PFVF
2578 #undef FW_PARAM_DEV
2579
2580 static void
2581 t4_set_desc(struct adapter *sc)
2582 {
2583         char buf[128];
2584         struct adapter_params *p = &sc->params;
2585
2586         snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
2587             "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
2588             chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
2589
2590         device_set_desc_copy(sc->dev, buf);
2591 }
2592
2593 static void
2594 build_medialist(struct port_info *pi)
2595 {
2596         struct ifmedia *media = &pi->media;
2597         int data, m;
2598
2599         PORT_LOCK(pi);
2600
2601         ifmedia_removeall(media);
2602
2603         m = IFM_ETHER | IFM_FDX;
2604         data = (pi->port_type << 8) | pi->mod_type;
2605
2606         switch(pi->port_type) {
2607         case FW_PORT_TYPE_BT_XFI:
2608                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2609                 break;
2610
2611         case FW_PORT_TYPE_BT_XAUI:
2612                 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2613                 /* fall through */
2614
2615         case FW_PORT_TYPE_BT_SGMII:
2616                 ifmedia_add(media, m | IFM_1000_T, data, NULL);
2617                 ifmedia_add(media, m | IFM_100_TX, data, NULL);
2618                 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL);
2619                 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2620                 break;
2621
2622         case FW_PORT_TYPE_CX4:
2623                 ifmedia_add(media, m | IFM_10G_CX4, data, NULL);
2624                 ifmedia_set(media, m | IFM_10G_CX4);
2625                 break;
2626
2627         case FW_PORT_TYPE_QSFP_10G:
2628         case FW_PORT_TYPE_SFP:
2629         case FW_PORT_TYPE_FIBER_XFI:
2630         case FW_PORT_TYPE_FIBER_XAUI:
2631                 switch (pi->mod_type) {
2632
2633                 case FW_PORT_MOD_TYPE_LR:
2634                         ifmedia_add(media, m | IFM_10G_LR, data, NULL);
2635                         ifmedia_set(media, m | IFM_10G_LR);
2636                         break;
2637
2638                 case FW_PORT_MOD_TYPE_SR:
2639                         ifmedia_add(media, m | IFM_10G_SR, data, NULL);
2640                         ifmedia_set(media, m | IFM_10G_SR);
2641                         break;
2642
2643                 case FW_PORT_MOD_TYPE_LRM:
2644                         ifmedia_add(media, m | IFM_10G_LRM, data, NULL);
2645                         ifmedia_set(media, m | IFM_10G_LRM);
2646                         break;
2647
2648                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2649                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2650                         ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL);
2651                         ifmedia_set(media, m | IFM_10G_TWINAX);
2652                         break;
2653
2654                 case FW_PORT_MOD_TYPE_NONE:
2655                         m &= ~IFM_FDX;
2656                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2657                         ifmedia_set(media, m | IFM_NONE);
2658                         break;
2659
2660                 case FW_PORT_MOD_TYPE_NA:
2661                 case FW_PORT_MOD_TYPE_ER:
2662                 default:
2663                         device_printf(pi->dev,
2664                             "unknown port_type (%d), mod_type (%d)\n",
2665                             pi->port_type, pi->mod_type);
2666                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2667                         ifmedia_set(media, m | IFM_UNKNOWN);
2668                         break;
2669                 }
2670                 break;
2671
2672         case FW_PORT_TYPE_QSFP:
2673                 switch (pi->mod_type) {
2674
2675                 case FW_PORT_MOD_TYPE_LR:
2676                         ifmedia_add(media, m | IFM_40G_LR4, data, NULL);
2677                         ifmedia_set(media, m | IFM_40G_LR4);
2678                         break;
2679
2680                 case FW_PORT_MOD_TYPE_SR:
2681                         ifmedia_add(media, m | IFM_40G_SR4, data, NULL);
2682                         ifmedia_set(media, m | IFM_40G_SR4);
2683                         break;
2684
2685                 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2686                 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2687                         ifmedia_add(media, m | IFM_40G_CR4, data, NULL);
2688                         ifmedia_set(media, m | IFM_40G_CR4);
2689                         break;
2690
2691                 case FW_PORT_MOD_TYPE_NONE:
2692                         m &= ~IFM_FDX;
2693                         ifmedia_add(media, m | IFM_NONE, data, NULL);
2694                         ifmedia_set(media, m | IFM_NONE);
2695                         break;
2696
2697                 default:
2698                         device_printf(pi->dev,
2699                             "unknown port_type (%d), mod_type (%d)\n",
2700                             pi->port_type, pi->mod_type);
2701                         ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2702                         ifmedia_set(media, m | IFM_UNKNOWN);
2703                         break;
2704                 }
2705                 break;
2706
2707         default:
2708                 device_printf(pi->dev,
2709                     "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
2710                     pi->mod_type);
2711                 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2712                 ifmedia_set(media, m | IFM_UNKNOWN);
2713                 break;
2714         }
2715
2716         PORT_UNLOCK(pi);
2717 }
2718
2719 #define FW_MAC_EXACT_CHUNK      7
2720
2721 /*
2722  * Program the port's XGMAC based on parameters in ifnet.  The caller also
2723  * indicates which parameters should be programmed (the rest are left alone).
2724  */
2725 static int
2726 update_mac_settings(struct port_info *pi, int flags)
2727 {
2728         int rc;
2729         struct ifnet *ifp = pi->ifp;
2730         struct adapter *sc = pi->adapter;
2731         int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
2732
2733         ASSERT_SYNCHRONIZED_OP(sc);
2734         KASSERT(flags, ("%s: not told what to update.", __func__));
2735
2736         if (flags & XGMAC_MTU)
2737                 mtu = ifp->if_mtu;
2738
2739         if (flags & XGMAC_PROMISC)
2740                 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
2741
2742         if (flags & XGMAC_ALLMULTI)
2743                 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
2744
2745         if (flags & XGMAC_VLANEX)
2746                 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
2747
2748         rc = -t4_set_rxmode(sc, sc->mbox, pi->viid, mtu, promisc, allmulti, 1,
2749             vlanex, false);
2750         if (rc) {
2751                 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, rc);
2752                 return (rc);
2753         }
2754
2755         if (flags & XGMAC_UCADDR) {
2756                 uint8_t ucaddr[ETHER_ADDR_LEN];
2757
2758                 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
2759                 rc = t4_change_mac(sc, sc->mbox, pi->viid, pi->xact_addr_filt,
2760                     ucaddr, true, true);
2761                 if (rc < 0) {
2762                         rc = -rc;
2763                         if_printf(ifp, "change_mac failed: %d\n", rc);
2764                         return (rc);
2765                 } else {
2766                         pi->xact_addr_filt = rc;
2767                         rc = 0;
2768                 }
2769         }
2770
2771         if (flags & XGMAC_MCADDRS) {
2772                 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
2773                 int del = 1;
2774                 uint64_t hash = 0;
2775                 struct ifmultiaddr *ifma;
2776                 int i = 0, j;
2777
2778                 if_maddr_rlock(ifp);
2779                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2780                         if (ifma->ifma_addr->sa_family != AF_LINK)
2781                                 continue;
2782                         mcaddr[i++] =
2783                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
2784
2785                         if (i == FW_MAC_EXACT_CHUNK) {
2786                                 rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid,
2787                                     del, i, mcaddr, NULL, &hash, 0);
2788                                 if (rc < 0) {
2789                                         rc = -rc;
2790                                         for (j = 0; j < i; j++) {
2791                                                 if_printf(ifp,
2792                                                     "failed to add mc address"
2793                                                     " %02x:%02x:%02x:"
2794                                                     "%02x:%02x:%02x rc=%d\n",
2795                                                     mcaddr[j][0], mcaddr[j][1],
2796                                                     mcaddr[j][2], mcaddr[j][3],
2797                                                     mcaddr[j][4], mcaddr[j][5],
2798                                                     rc);
2799                                         }
2800                                         goto mcfail;
2801                                 }
2802                                 del = 0;
2803                                 i = 0;
2804                         }
2805                 }
2806                 if (i > 0) {
2807                         rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid,
2808                             del, i, mcaddr, NULL, &hash, 0);
2809                         if (rc < 0) {
2810                                 rc = -rc;
2811                                 for (j = 0; j < i; j++) {
2812                                         if_printf(ifp,
2813                                             "failed to add mc address"
2814                                             " %02x:%02x:%02x:"
2815                                             "%02x:%02x:%02x rc=%d\n",
2816                                             mcaddr[j][0], mcaddr[j][1],
2817                                             mcaddr[j][2], mcaddr[j][3],
2818                                             mcaddr[j][4], mcaddr[j][5],
2819                                             rc);
2820                                 }
2821                                 goto mcfail;
2822                         }
2823                 }
2824
2825                 rc = -t4_set_addr_hash(sc, sc->mbox, pi->viid, 0, hash, 0);
2826                 if (rc != 0)
2827                         if_printf(ifp, "failed to set mc address hash: %d", rc);
2828 mcfail:
2829                 if_maddr_runlock(ifp);
2830         }
2831
2832         return (rc);
2833 }
2834
2835 int
2836 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
2837     char *wmesg)
2838 {
2839         int rc, pri;
2840
2841 #ifdef WITNESS
2842         /* the caller thinks it's ok to sleep, but is it really? */
2843         if (flags & SLEEP_OK)
2844                 pause("t4slptst", 1);
2845 #endif
2846
2847         if (INTR_OK)
2848                 pri = PCATCH;
2849         else
2850                 pri = 0;
2851
2852         ADAPTER_LOCK(sc);
2853         for (;;) {
2854
2855                 if (pi && IS_DOOMED(pi)) {
2856                         rc = ENXIO;
2857                         goto done;
2858                 }
2859
2860                 if (!IS_BUSY(sc)) {
2861                         rc = 0;
2862                         break;
2863                 }
2864
2865                 if (!(flags & SLEEP_OK)) {
2866                         rc = EBUSY;
2867                         goto done;
2868                 }
2869
2870                 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
2871                         rc = EINTR;
2872                         goto done;
2873                 }
2874         }
2875
2876         KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
2877         SET_BUSY(sc);
2878 #ifdef INVARIANTS
2879         sc->last_op = wmesg;
2880         sc->last_op_thr = curthread;
2881 #endif
2882
2883 done:
2884         if (!(flags & HOLD_LOCK) || rc)
2885                 ADAPTER_UNLOCK(sc);
2886
2887         return (rc);
2888 }
2889
2890 void
2891 end_synchronized_op(struct adapter *sc, int flags)
2892 {
2893
2894         if (flags & LOCK_HELD)
2895                 ADAPTER_LOCK_ASSERT_OWNED(sc);
2896         else
2897                 ADAPTER_LOCK(sc);
2898
2899         KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
2900         CLR_BUSY(sc);
2901         wakeup(&sc->flags);
2902         ADAPTER_UNLOCK(sc);
2903 }
2904
2905 static int
2906 cxgbe_init_synchronized(struct port_info *pi)
2907 {
2908         struct adapter *sc = pi->adapter;
2909         struct ifnet *ifp = pi->ifp;
2910         int rc = 0;
2911
2912         ASSERT_SYNCHRONIZED_OP(sc);
2913
2914         if (isset(&sc->open_device_map, pi->port_id)) {
2915                 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
2916                     ("mismatch between open_device_map and if_drv_flags"));
2917                 return (0);     /* already running */
2918         }
2919
2920         if (!(sc->flags & FULL_INIT_DONE) &&
2921             ((rc = adapter_full_init(sc)) != 0))
2922                 return (rc);    /* error message displayed already */
2923
2924         if (!(pi->flags & PORT_INIT_DONE) &&
2925             ((rc = port_full_init(pi)) != 0))
2926                 return (rc); /* error message displayed already */
2927
2928         rc = update_mac_settings(pi, XGMAC_ALL);
2929         if (rc)
2930                 goto done;      /* error message displayed already */
2931
2932         rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
2933         if (rc != 0) {
2934                 if_printf(ifp, "start_link failed: %d\n", rc);
2935                 goto done;
2936         }
2937
2938         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
2939         if (rc != 0) {
2940                 if_printf(ifp, "enable_vi failed: %d\n", rc);
2941                 goto done;
2942         }
2943
2944         /*
2945          * The first iq of the first port to come up is used for tracing.
2946          */
2947         if (sc->traceq < 0) {
2948                 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
2949                 t4_write_reg(sc, is_t4(sc) ?  A_MPS_TRC_RSS_CONTROL :
2950                     A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
2951                     V_QUEUENUMBER(sc->traceq));
2952                 pi->flags |= HAS_TRACEQ;
2953         }
2954
2955         /* all ok */
2956         setbit(&sc->open_device_map, pi->port_id);
2957         PORT_LOCK(pi);
2958         ifp->if_drv_flags |= IFF_DRV_RUNNING;
2959         PORT_UNLOCK(pi);
2960
2961         callout_reset(&pi->tick, hz, cxgbe_tick, pi);
2962 done:
2963         if (rc != 0)
2964                 cxgbe_uninit_synchronized(pi);
2965
2966         return (rc);
2967 }
2968
2969 /*
2970  * Idempotent.
2971  */
2972 static int
2973 cxgbe_uninit_synchronized(struct port_info *pi)
2974 {
2975         struct adapter *sc = pi->adapter;
2976         struct ifnet *ifp = pi->ifp;
2977         int rc;
2978
2979         ASSERT_SYNCHRONIZED_OP(sc);
2980
2981         /*
2982          * Disable the VI so that all its data in either direction is discarded
2983          * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
2984          * tick) intact as the TP can deliver negative advice or data that it's
2985          * holding in its RAM (for an offloaded connection) even after the VI is
2986          * disabled.
2987          */
2988         rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
2989         if (rc) {
2990                 if_printf(ifp, "disable_vi failed: %d\n", rc);
2991                 return (rc);
2992         }
2993
2994         clrbit(&sc->open_device_map, pi->port_id);
2995         PORT_LOCK(pi);
2996         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2997         PORT_UNLOCK(pi);
2998
2999         pi->link_cfg.link_ok = 0;
3000         pi->link_cfg.speed = 0;
3001         pi->linkdnrc = -1;
3002         t4_os_link_changed(sc, pi->port_id, 0, -1);
3003
3004         return (0);
3005 }
3006
3007 /*
3008  * It is ok for this function to fail midway and return right away.  t4_detach
3009  * will walk the entire sc->irq list and clean up whatever is valid.
3010  */
3011 static int
3012 setup_intr_handlers(struct adapter *sc)
3013 {
3014         int rc, rid, p, q;
3015         char s[8];
3016         struct irq *irq;
3017         struct port_info *pi;
3018         struct sge_rxq *rxq;
3019 #ifdef TCP_OFFLOAD
3020         struct sge_ofld_rxq *ofld_rxq;
3021 #endif
3022
3023         /*
3024          * Setup interrupts.
3025          */
3026         irq = &sc->irq[0];
3027         rid = sc->intr_type == INTR_INTX ? 0 : 1;
3028         if (sc->intr_count == 1) {
3029                 KASSERT(!(sc->flags & INTR_DIRECT),
3030                     ("%s: single interrupt && INTR_DIRECT?", __func__));
3031
3032                 rc = t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all");
3033                 if (rc != 0)
3034                         return (rc);
3035         } else {
3036                 /* Multiple interrupts. */
3037                 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
3038                     ("%s: too few intr.", __func__));
3039
3040                 /* The first one is always error intr */
3041                 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
3042                 if (rc != 0)
3043                         return (rc);
3044                 irq++;
3045                 rid++;
3046
3047                 /* The second one is always the firmware event queue */
3048                 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq,
3049                     "evt");
3050                 if (rc != 0)
3051                         return (rc);
3052                 irq++;
3053                 rid++;
3054
3055                 /*
3056                  * Note that if INTR_DIRECT is not set then either the NIC rx
3057                  * queues or (exclusive or) the TOE rx queueus will be taking
3058                  * direct interrupts.
3059                  *
3060                  * There is no need to check for is_offload(sc) as nofldrxq
3061                  * will be 0 if offload is disabled.
3062                  */
3063                 for_each_port(sc, p) {
3064                         pi = sc->port[p];
3065
3066 #ifdef TCP_OFFLOAD
3067                         /*
3068                          * Skip over the NIC queues if they aren't taking direct
3069                          * interrupts.
3070                          */
3071                         if (!(sc->flags & INTR_DIRECT) &&
3072                             pi->nofldrxq > pi->nrxq)
3073                                 goto ofld_queues;
3074 #endif
3075                         rxq = &sc->sge.rxq[pi->first_rxq];
3076                         for (q = 0; q < pi->nrxq; q++, rxq++) {
3077                                 snprintf(s, sizeof(s), "%d.%d", p, q);
3078                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
3079                                     s);
3080                                 if (rc != 0)
3081                                         return (rc);
3082                                 irq++;
3083                                 rid++;
3084                         }
3085
3086 #ifdef TCP_OFFLOAD
3087                         /*
3088                          * Skip over the offload queues if they aren't taking
3089                          * direct interrupts.
3090                          */
3091                         if (!(sc->flags & INTR_DIRECT))
3092                                 continue;
3093 ofld_queues:
3094                         ofld_rxq = &sc->sge.ofld_rxq[pi->first_ofld_rxq];
3095                         for (q = 0; q < pi->nofldrxq; q++, ofld_rxq++) {
3096                                 snprintf(s, sizeof(s), "%d,%d", p, q);
3097                                 rc = t4_alloc_irq(sc, irq, rid, t4_intr,
3098                                     ofld_rxq, s);
3099                                 if (rc != 0)
3100                                         return (rc);
3101                                 irq++;
3102                                 rid++;
3103                         }
3104 #endif
3105                 }
3106         }
3107
3108         return (0);
3109 }
3110
3111 static int
3112 adapter_full_init(struct adapter *sc)
3113 {
3114         int rc, i;
3115
3116         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3117         KASSERT((sc->flags & FULL_INIT_DONE) == 0,
3118             ("%s: FULL_INIT_DONE already", __func__));
3119
3120         /*
3121          * queues that belong to the adapter (not any particular port).
3122          */
3123         rc = t4_setup_adapter_queues(sc);
3124         if (rc != 0)
3125                 goto done;
3126
3127         for (i = 0; i < nitems(sc->tq); i++) {
3128                 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
3129                     taskqueue_thread_enqueue, &sc->tq[i]);
3130                 if (sc->tq[i] == NULL) {
3131                         device_printf(sc->dev,
3132                             "failed to allocate task queue %d\n", i);
3133                         rc = ENOMEM;
3134                         goto done;
3135                 }
3136                 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
3137                     device_get_nameunit(sc->dev), i);
3138         }
3139
3140         t4_intr_enable(sc);
3141         sc->flags |= FULL_INIT_DONE;
3142 done:
3143         if (rc != 0)
3144                 adapter_full_uninit(sc);
3145
3146         return (rc);
3147 }
3148
3149 static int
3150 adapter_full_uninit(struct adapter *sc)
3151 {
3152         int i;
3153
3154         ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3155
3156         t4_teardown_adapter_queues(sc);
3157
3158         for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
3159                 taskqueue_free(sc->tq[i]);
3160                 sc->tq[i] = NULL;
3161         }
3162
3163         sc->flags &= ~FULL_INIT_DONE;
3164
3165         return (0);
3166 }
3167
3168 static int
3169 port_full_init(struct port_info *pi)
3170 {
3171         struct adapter *sc = pi->adapter;
3172         struct ifnet *ifp = pi->ifp;
3173         uint16_t *rss;
3174         struct sge_rxq *rxq;
3175         int rc, i, j;
3176
3177         ASSERT_SYNCHRONIZED_OP(sc);
3178         KASSERT((pi->flags & PORT_INIT_DONE) == 0,
3179             ("%s: PORT_INIT_DONE already", __func__));
3180
3181         sysctl_ctx_init(&pi->ctx);
3182         pi->flags |= PORT_SYSCTL_CTX;
3183
3184         /*
3185          * Allocate tx/rx/fl queues for this port.
3186          */
3187         rc = t4_setup_port_queues(pi);
3188         if (rc != 0)
3189                 goto done;      /* error message displayed already */
3190
3191         /*
3192          * Setup RSS for this port.  Save a copy of the RSS table for later use.
3193          */
3194         rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
3195         for (i = 0; i < pi->rss_size;) {
3196                 for_each_rxq(pi, j, rxq) {
3197                         rss[i++] = rxq->iq.abs_id;
3198                         if (i == pi->rss_size)
3199                                 break;
3200                 }
3201         }
3202
3203         rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
3204             pi->rss_size);
3205         if (rc != 0) {
3206                 if_printf(ifp, "rss_config failed: %d\n", rc);
3207                 goto done;
3208         }
3209
3210         pi->rss = rss;
3211         pi->flags |= PORT_INIT_DONE;
3212 done:
3213         if (rc != 0)
3214                 port_full_uninit(pi);
3215
3216         return (rc);
3217 }
3218
3219 /*
3220  * Idempotent.
3221  */
3222 static int
3223 port_full_uninit(struct port_info *pi)
3224 {
3225         struct adapter *sc = pi->adapter;
3226         int i;
3227         struct sge_rxq *rxq;
3228         struct sge_txq *txq;
3229 #ifdef TCP_OFFLOAD
3230         struct sge_ofld_rxq *ofld_rxq;
3231         struct sge_wrq *ofld_txq;
3232 #endif
3233
3234         if (pi->flags & PORT_INIT_DONE) {
3235
3236                 /* Need to quiesce queues.  XXX: ctrl queues? */
3237
3238                 for_each_txq(pi, i, txq) {
3239                         quiesce_eq(sc, &txq->eq);
3240                 }
3241
3242 #ifdef TCP_OFFLOAD
3243                 for_each_ofld_txq(pi, i, ofld_txq) {
3244                         quiesce_eq(sc, &ofld_txq->eq);
3245                 }
3246 #endif
3247
3248                 for_each_rxq(pi, i, rxq) {
3249                         quiesce_iq(sc, &rxq->iq);
3250                         quiesce_fl(sc, &rxq->fl);
3251                 }
3252
3253 #ifdef TCP_OFFLOAD
3254                 for_each_ofld_rxq(pi, i, ofld_rxq) {
3255                         quiesce_iq(sc, &ofld_rxq->iq);
3256                         quiesce_fl(sc, &ofld_rxq->fl);
3257                 }
3258 #endif
3259                 free(pi->rss, M_CXGBE);
3260         }
3261
3262         t4_teardown_port_queues(pi);
3263         pi->flags &= ~PORT_INIT_DONE;
3264
3265         return (0);
3266 }
3267
3268 static void
3269 quiesce_eq(struct adapter *sc, struct sge_eq *eq)
3270 {
3271         EQ_LOCK(eq);
3272         eq->flags |= EQ_DOOMED;
3273
3274         /*
3275          * Wait for the response to a credit flush if one's
3276          * pending.
3277          */
3278         while (eq->flags & EQ_CRFLUSHED)
3279                 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0);
3280         EQ_UNLOCK(eq);
3281
3282         callout_drain(&eq->tx_callout); /* XXX: iffy */
3283         pause("callout", 10);           /* Still iffy */
3284
3285         taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task);
3286 }
3287
3288 static void
3289 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
3290 {
3291         (void) sc;      /* unused */
3292
3293         /* Synchronize with the interrupt handler */
3294         while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
3295                 pause("iqfree", 1);
3296 }
3297
3298 static void
3299 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
3300 {
3301         mtx_lock(&sc->sfl_lock);
3302         FL_LOCK(fl);
3303         fl->flags |= FL_DOOMED;
3304         FL_UNLOCK(fl);
3305         mtx_unlock(&sc->sfl_lock);
3306
3307         callout_drain(&sc->sfl_callout);
3308         KASSERT((fl->flags & FL_STARVING) == 0,
3309             ("%s: still starving", __func__));
3310 }
3311
3312 static int
3313 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
3314     driver_intr_t *handler, void *arg, char *name)
3315 {
3316         int rc;
3317
3318         irq->rid = rid;
3319         irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
3320             RF_SHAREABLE | RF_ACTIVE);
3321         if (irq->res == NULL) {
3322                 device_printf(sc->dev,
3323                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
3324                 return (ENOMEM);
3325         }
3326
3327         rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
3328             NULL, handler, arg, &irq->tag);
3329         if (rc != 0) {
3330                 device_printf(sc->dev,
3331                     "failed to setup interrupt for rid %d, name %s: %d\n",
3332                     rid, name, rc);
3333         } else if (name)
3334                 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
3335
3336         return (rc);
3337 }
3338
3339 static int
3340 t4_free_irq(struct adapter *sc, struct irq *irq)
3341 {
3342         if (irq->tag)
3343                 bus_teardown_intr(sc->dev, irq->res, irq->tag);
3344         if (irq->res)
3345                 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
3346
3347         bzero(irq, sizeof(*irq));
3348
3349         return (0);
3350 }
3351
3352 static void
3353 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
3354     unsigned int end)
3355 {
3356         uint32_t *p = (uint32_t *)(buf + start);
3357
3358         for ( ; start <= end; start += sizeof(uint32_t))
3359                 *p++ = t4_read_reg(sc, start);
3360 }
3361
3362 static void
3363 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
3364 {
3365         int i, n;
3366         const unsigned int *reg_ranges;
3367         static const unsigned int t4_reg_ranges[] = {
3368                 0x1008, 0x1108,
3369                 0x1180, 0x11b4,
3370                 0x11fc, 0x123c,
3371                 0x1300, 0x173c,
3372                 0x1800, 0x18fc,
3373                 0x3000, 0x30d8,
3374                 0x30e0, 0x5924,
3375                 0x5960, 0x59d4,
3376                 0x5a00, 0x5af8,
3377                 0x6000, 0x6098,
3378                 0x6100, 0x6150,
3379                 0x6200, 0x6208,
3380                 0x6240, 0x6248,
3381                 0x6280, 0x6338,
3382                 0x6370, 0x638c,
3383                 0x6400, 0x643c,
3384                 0x6500, 0x6524,
3385                 0x6a00, 0x6a38,
3386                 0x6a60, 0x6a78,
3387                 0x6b00, 0x6b84,
3388                 0x6bf0, 0x6c84,
3389                 0x6cf0, 0x6d84,
3390                 0x6df0, 0x6e84,
3391                 0x6ef0, 0x6f84,
3392                 0x6ff0, 0x7084,
3393                 0x70f0, 0x7184,
3394                 0x71f0, 0x7284,
3395                 0x72f0, 0x7384,
3396                 0x73f0, 0x7450,
3397                 0x7500, 0x7530,
3398                 0x7600, 0x761c,
3399                 0x7680, 0x76cc,
3400                 0x7700, 0x7798,
3401                 0x77c0, 0x77fc,
3402                 0x7900, 0x79fc,
3403                 0x7b00, 0x7c38,
3404                 0x7d00, 0x7efc,
3405                 0x8dc0, 0x8e1c,
3406                 0x8e30, 0x8e78,
3407                 0x8ea0, 0x8f6c,
3408                 0x8fc0, 0x9074,
3409                 0x90fc, 0x90fc,
3410                 0x9400, 0x9458,
3411                 0x9600, 0x96bc,
3412                 0x9800, 0x9808,
3413                 0x9820, 0x983c,
3414                 0x9850, 0x9864,
3415                 0x9c00, 0x9c6c,
3416                 0x9c80, 0x9cec,
3417                 0x9d00, 0x9d6c,
3418                 0x9d80, 0x9dec,
3419                 0x9e00, 0x9e6c,
3420                 0x9e80, 0x9eec,
3421                 0x9f00, 0x9f6c,
3422                 0x9f80, 0x9fec,
3423                 0xd004, 0xd03c,
3424                 0xdfc0, 0xdfe0,
3425                 0xe000, 0xea7c,
3426                 0xf000, 0x11110,
3427                 0x11118, 0x11190,
3428                 0x19040, 0x1906c,
3429                 0x19078, 0x19080,
3430                 0x1908c, 0x19124,
3431                 0x19150, 0x191b0,
3432                 0x191d0, 0x191e8,
3433                 0x19238, 0x1924c,
3434                 0x193f8, 0x19474,
3435                 0x19490, 0x194f8,
3436                 0x19800, 0x19f30,
3437                 0x1a000, 0x1a06c,
3438                 0x1a0b0, 0x1a120,
3439                 0x1a128, 0x1a138,
3440                 0x1a190, 0x1a1c4,
3441                 0x1a1fc, 0x1a1fc,
3442                 0x1e040, 0x1e04c,
3443                 0x1e284, 0x1e28c,
3444                 0x1e2c0, 0x1e2c0,
3445                 0x1e2e0, 0x1e2e0,
3446                 0x1e300, 0x1e384,
3447                 0x1e3c0, 0x1e3c8,
3448                 0x1e440, 0x1e44c,
3449                 0x1e684, 0x1e68c,
3450                 0x1e6c0, 0x1e6c0,
3451                 0x1e6e0, 0x1e6e0,
3452                 0x1e700, 0x1e784,
3453                 0x1e7c0, 0x1e7c8,
3454                 0x1e840, 0x1e84c,
3455                 0x1ea84, 0x1ea8c,
3456                 0x1eac0, 0x1eac0,
3457                 0x1eae0, 0x1eae0,
3458                 0x1eb00, 0x1eb84,
3459                 0x1ebc0, 0x1ebc8,
3460                 0x1ec40, 0x1ec4c,
3461                 0x1ee84, 0x1ee8c,
3462                 0x1eec0, 0x1eec0,
3463                 0x1eee0, 0x1eee0,
3464                 0x1ef00, 0x1ef84,
3465                 0x1efc0, 0x1efc8,
3466                 0x1f040, 0x1f04c,
3467                 0x1f284, 0x1f28c,
3468                 0x1f2c0, 0x1f2c0,
3469                 0x1f2e0, 0x1f2e0,
3470                 0x1f300, 0x1f384,
3471                 0x1f3c0, 0x1f3c8,
3472                 0x1f440, 0x1f44c,
3473                 0x1f684, 0x1f68c,
3474                 0x1f6c0, 0x1f6c0,
3475                 0x1f6e0, 0x1f6e0,
3476                 0x1f700, 0x1f784,
3477                 0x1f7c0, 0x1f7c8,
3478                 0x1f840, 0x1f84c,
3479                 0x1fa84, 0x1fa8c,
3480                 0x1fac0, 0x1fac0,
3481                 0x1fae0, 0x1fae0,
3482                 0x1fb00, 0x1fb84,
3483                 0x1fbc0, 0x1fbc8,
3484                 0x1fc40, 0x1fc4c,
3485                 0x1fe84, 0x1fe8c,
3486                 0x1fec0, 0x1fec0,
3487                 0x1fee0, 0x1fee0,
3488                 0x1ff00, 0x1ff84,
3489                 0x1ffc0, 0x1ffc8,
3490                 0x20000, 0x2002c,
3491                 0x20100, 0x2013c,
3492                 0x20190, 0x201c8,
3493                 0x20200, 0x20318,
3494                 0x20400, 0x20528,
3495                 0x20540, 0x20614,
3496                 0x21000, 0x21040,
3497                 0x2104c, 0x21060,
3498                 0x210c0, 0x210ec,
3499                 0x21200, 0x21268,
3500                 0x21270, 0x21284,
3501                 0x212fc, 0x21388,
3502                 0x21400, 0x21404,
3503                 0x21500, 0x21518,
3504                 0x2152c, 0x2153c,
3505                 0x21550, 0x21554,
3506                 0x21600, 0x21600,
3507                 0x21608, 0x21628,
3508                 0x21630, 0x2163c,
3509                 0x21700, 0x2171c,
3510                 0x21780, 0x2178c,
3511                 0x21800, 0x21c38,
3512                 0x21c80, 0x21d7c,
3513                 0x21e00, 0x21e04,
3514                 0x22000, 0x2202c,
3515                 0x22100, 0x2213c,
3516                 0x22190, 0x221c8,
3517                 0x22200, 0x22318,
3518                 0x22400, 0x22528,
3519                 0x22540, 0x22614,
3520                 0x23000, 0x23040,
3521                 0x2304c, 0x23060,
3522                 0x230c0, 0x230ec,
3523                 0x23200, 0x23268,
3524                 0x23270, 0x23284,
3525                 0x232fc, 0x23388,
3526                 0x23400, 0x23404,
3527                 0x23500, 0x23518,
3528                 0x2352c, 0x2353c,
3529                 0x23550, 0x23554,
3530                 0x23600, 0x23600,
3531                 0x23608, 0x23628,
3532                 0x23630, 0x2363c,
3533                 0x23700, 0x2371c,
3534                 0x23780, 0x2378c,
3535                 0x23800, 0x23c38,
3536                 0x23c80, 0x23d7c,
3537                 0x23e00, 0x23e04,
3538                 0x24000, 0x2402c,
3539                 0x24100, 0x2413c,
3540                 0x24190, 0x241c8,
3541                 0x24200, 0x24318,
3542                 0x24400, 0x24528,
3543                 0x24540, 0x24614,
3544                 0x25000, 0x25040,
3545                 0x2504c, 0x25060,
3546                 0x250c0, 0x250ec,
3547                 0x25200, 0x25268,
3548                 0x25270, 0x25284,
3549                 0x252fc, 0x25388,
3550                 0x25400, 0x25404,
3551                 0x25500, 0x25518,
3552                 0x2552c, 0x2553c,
3553                 0x25550, 0x25554,
3554                 0x25600, 0x25600,
3555                 0x25608, 0x25628,
3556                 0x25630, 0x2563c,
3557                 0x25700, 0x2571c,
3558                 0x25780, 0x2578c,
3559                 0x25800, 0x25c38,
3560                 0x25c80, 0x25d7c,
3561                 0x25e00, 0x25e04,
3562                 0x26000, 0x2602c,
3563                 0x26100, 0x2613c,
3564                 0x26190, 0x261c8,
3565                 0x26200, 0x26318,
3566                 0x26400, 0x26528,
3567                 0x26540, 0x26614,
3568                 0x27000, 0x27040,
3569                 0x2704c, 0x27060,
3570                 0x270c0, 0x270ec,
3571                 0x27200, 0x27268,
3572                 0x27270, 0x27284,
3573                 0x272fc, 0x27388,
3574                 0x27400, 0x27404,
3575                 0x27500, 0x27518,
3576                 0x2752c, 0x2753c,
3577                 0x27550, 0x27554,
3578                 0x27600, 0x27600,
3579                 0x27608, 0x27628,
3580                 0x27630, 0x2763c,
3581                 0x27700, 0x2771c,
3582                 0x27780, 0x2778c,
3583                 0x27800, 0x27c38,
3584                 0x27c80, 0x27d7c,
3585                 0x27e00, 0x27e04
3586         };
3587         static const unsigned int t5_reg_ranges[] = {
3588                 0x1008, 0x1148,
3589                 0x1180, 0x11b4,
3590                 0x11fc, 0x123c,
3591                 0x1280, 0x173c,
3592                 0x1800, 0x18fc,
3593                 0x3000, 0x3028,
3594                 0x3060, 0x30d8,
3595                 0x30e0, 0x30fc,
3596                 0x3140, 0x357c,
3597                 0x35a8, 0x35cc,
3598                 0x35ec, 0x35ec,
3599                 0x3600, 0x5624,
3600                 0x56cc, 0x575c,
3601                 0x580c, 0x5814,
3602                 0x5890, 0x58bc,
3603                 0x5940, 0x59dc,
3604                 0x59fc, 0x5a18,
3605                 0x5a60, 0x5a9c,
3606                 0x5b94, 0x5bfc,
3607                 0x6000, 0x6040,
3608                 0x6058, 0x614c,
3609                 0x7700, 0x7798,
3610                 0x77c0, 0x78fc,
3611                 0x7b00, 0x7c54,
3612                 0x7d00, 0x7efc,
3613                 0x8dc0, 0x8de0,
3614                 0x8df8, 0x8e84,
3615                 0x8ea0, 0x8f84,
3616                 0x8fc0, 0x90f8,
3617                 0x9400, 0x9470,
3618                 0x9600, 0x96f4,
3619                 0x9800, 0x9808,
3620                 0x9820, 0x983c,
3621                 0x9850, 0x9864,
3622                 0x9c00, 0x9c6c,
3623                 0x9c80, 0x9cec,
3624                 0x9d00, 0x9d6c,
3625                 0x9d80, 0x9dec,
3626                 0x9e00, 0x9e6c,
3627                 0x9e80, 0x9eec,
3628                 0x9f00, 0x9f6c,
3629                 0x9f80, 0xa020,
3630                 0xd004, 0xd03c,
3631                 0xdfc0, 0xdfe0,
3632                 0xe000, 0x11088,
3633                 0x1109c, 0x11110,
3634                 0x11118, 0x1117c,
3635                 0x11190, 0x11204,
3636                 0x19040, 0x1906c,
3637                 0x19078, 0x19080,
3638                 0x1908c, 0x19124,
3639                 0x19150, 0x191b0,
3640                 0x191d0, 0x191e8,
3641                 0x19238, 0x19290,
3642                 0x193f8, 0x19474,
3643                 0x19490, 0x194cc,
3644                 0x194f0, 0x194f8,
3645                 0x19c00, 0x19c60,
3646                 0x19c94, 0x19e10,
3647                 0x19e50, 0x19f34,
3648                 0x19f40, 0x19f50,
3649                 0x19f90, 0x19fe4,
3650                 0x1a000, 0x1a06c,
3651                 0x1a0b0, 0x1a120,
3652                 0x1a128, 0x1a138,
3653                 0x1a190, 0x1a1c4,
3654                 0x1a1fc, 0x1a1fc,
3655                 0x1e008, 0x1e00c,
3656                 0x1e040, 0x1e04c,
3657                 0x1e284, 0x1e290,
3658                 0x1e2c0, 0x1e2c0,
3659                 0x1e2e0, 0x1e2e0,
3660                 0x1e300, 0x1e384,
3661                 0x1e3c0, 0x1e3c8,
3662                 0x1e408, 0x1e40c,
3663                 0x1e440, 0x1e44c,
3664                 0x1e684, 0x1e690,
3665                 0x1e6c0, 0x1e6c0,
3666                 0x1e6e0, 0x1e6e0,
3667                 0x1e700, 0x1e784,
3668                 0x1e7c0, 0x1e7c8,
3669                 0x1e808, 0x1e80c,
3670                 0x1e840, 0x1e84c,
3671                 0x1ea84, 0x1ea90,
3672                 0x1eac0, 0x1eac0,
3673                 0x1eae0, 0x1eae0,
3674                 0x1eb00, 0x1eb84,
3675                 0x1ebc0, 0x1ebc8,
3676                 0x1ec08, 0x1ec0c,
3677                 0x1ec40, 0x1ec4c,
3678                 0x1ee84, 0x1ee90,
3679                 0x1eec0, 0x1eec0,
3680                 0x1eee0, 0x1eee0,
3681                 0x1ef00, 0x1ef84,
3682                 0x1efc0, 0x1efc8,
3683                 0x1f008, 0x1f00c,
3684                 0x1f040, 0x1f04c,
3685                 0x1f284, 0x1f290,
3686                 0x1f2c0, 0x1f2c0,
3687                 0x1f2e0, 0x1f2e0,
3688                 0x1f300, 0x1f384,
3689                 0x1f3c0, 0x1f3c8,
3690                 0x1f408, 0x1f40c,
3691                 0x1f440, 0x1f44c,
3692                 0x1f684, 0x1f690,
3693                 0x1f6c0, 0x1f6c0,
3694                 0x1f6e0, 0x1f6e0,
3695                 0x1f700, 0x1f784,
3696                 0x1f7c0, 0x1f7c8,
3697                 0x1f808, 0x1f80c,
3698                 0x1f840, 0x1f84c,
3699                 0x1fa84, 0x1fa90,
3700                 0x1fac0, 0x1fac0,
3701                 0x1fae0, 0x1fae0,
3702                 0x1fb00, 0x1fb84,
3703                 0x1fbc0, 0x1fbc8,
3704                 0x1fc08, 0x1fc0c,
3705                 0x1fc40, 0x1fc4c,
3706                 0x1fe84, 0x1fe90,
3707                 0x1fec0, 0x1fec0,
3708                 0x1fee0, 0x1fee0,
3709                 0x1ff00, 0x1ff84,
3710                 0x1ffc0, 0x1ffc8,
3711                 0x30000, 0x30030,
3712                 0x30100, 0x30144,
3713                 0x30190, 0x301d0,
3714                 0x30200, 0x30318,
3715                 0x30400, 0x3052c,
3716                 0x30540, 0x3061c,
3717                 0x30800, 0x30834,
3718                 0x308c0, 0x30908,
3719                 0x30910, 0x309ac,
3720                 0x30a00, 0x30a2c,
3721                 0x30a44, 0x30a50,
3722                 0x30a74, 0x30c24,
3723                 0x30d00, 0x30d00,
3724                 0x30d08, 0x30d14,
3725                 0x30d1c, 0x30d20,
3726                 0x30d3c, 0x30d50,
3727                 0x31200, 0x3120c,
3728                 0x31220, 0x31220,
3729                 0x31240, 0x31240,
3730                 0x31600, 0x3160c,
3731                 0x31a00, 0x31a1c,
3732                 0x31e00, 0x31e20,
3733                 0x31e38, 0x31e3c,
3734                 0x31e80, 0x31e80,
3735                 0x31e88, 0x31ea8,
3736                 0x31eb0, 0x31eb4,
3737                 0x31ec8, 0x31ed4,
3738                 0x31fb8, 0x32004,
3739                 0x32200, 0x32200,
3740                 0x32208, 0x32240,
3741                 0x32248, 0x32280,
3742                 0x32288, 0x322c0,
3743                 0x322c8, 0x322fc,
3744                 0x32600, 0x32630,
3745                 0x32a00, 0x32abc,
3746                 0x32b00, 0x32b70,
3747                 0x33000, 0x33048,
3748                 0x33060, 0x3309c,
3749                 0x330f0, 0x33148,
3750                 0x33160, 0x3319c,
3751                 0x331f0, 0x332e4,
3752                 0x332f8, 0x333e4,
3753                 0x333f8, 0x33448,
3754                 0x33460, 0x3349c,
3755                 0x334f0, 0x33548,
3756                 0x33560, 0x3359c,
3757                 0x335f0, 0x336e4,
3758                 0x336f8, 0x337e4,
3759                 0x337f8, 0x337fc,
3760                 0x33814, 0x33814,
3761                 0x3382c, 0x3382c,
3762                 0x33880, 0x3388c,
3763                 0x338e8, 0x338ec,
3764                 0x33900, 0x33948,
3765                 0x33960, 0x3399c,
3766                 0x339f0, 0x33ae4,
3767                 0x33af8, 0x33b10,
3768                 0x33b28, 0x33b28,
3769                 0x33b3c, 0x33b50,
3770                 0x33bf0, 0x33c10,
3771                 0x33c28, 0x33c28,
3772                 0x33c3c, 0x33c50,
3773                 0x33cf0, 0x33cfc,
3774                 0x34000, 0x34030,
3775                 0x34100, 0x34144,
3776                 0x34190, 0x341d0,
3777                 0x34200, 0x34318,
3778                 0x34400, 0x3452c,
3779                 0x34540, 0x3461c,
3780                 0x34800, 0x34834,
3781                 0x348c0, 0x34908,
3782                 0x34910, 0x349ac,
3783                 0x34a00, 0x34a2c,
3784                 0x34a44, 0x34a50,
3785                 0x34a74, 0x34c24,
3786                 0x34d00, 0x34d00,
3787                 0x34d08, 0x34d14,
3788                 0x34d1c, 0x34d20,
3789                 0x34d3c, 0x34d50,
3790                 0x35200, 0x3520c,
3791                 0x35220, 0x35220,
3792                 0x35240, 0x35240,
3793                 0x35600, 0x3560c,
3794                 0x35a00, 0x35a1c,
3795                 0x35e00, 0x35e20,
3796                 0x35e38, 0x35e3c,
3797                 0x35e80, 0x35e80,
3798                 0x35e88, 0x35ea8,
3799                 0x35eb0, 0x35eb4,
3800                 0x35ec8, 0x35ed4,
3801                 0x35fb8, 0x36004,
3802                 0x36200, 0x36200,
3803                 0x36208, 0x36240,
3804                 0x36248, 0x36280,
3805                 0x36288, 0x362c0,
3806                 0x362c8, 0x362fc,
3807                 0x36600, 0x36630,
3808                 0x36a00, 0x36abc,
3809                 0x36b00, 0x36b70,
3810                 0x37000, 0x37048,
3811                 0x37060, 0x3709c,
3812                 0x370f0, 0x37148,
3813                 0x37160, 0x3719c,
3814                 0x371f0, 0x372e4,
3815                 0x372f8, 0x373e4,
3816                 0x373f8, 0x37448,
3817                 0x37460, 0x3749c,
3818                 0x374f0, 0x37548,
3819                 0x37560, 0x3759c,
3820                 0x375f0, 0x376e4,
3821                 0x376f8, 0x377e4,
3822                 0x377f8, 0x377fc,
3823                 0x37814, 0x37814,
3824                 0x3782c, 0x3782c,
3825                 0x37880, 0x3788c,
3826                 0x378e8, 0x378ec,
3827                 0x37900, 0x37948,
3828                 0x37960, 0x3799c,
3829                 0x379f0, 0x37ae4,
3830                 0x37af8, 0x37b10,
3831                 0x37b28, 0x37b28,
3832                 0x37b3c, 0x37b50,
3833                 0x37bf0, 0x37c10,
3834                 0x37c28, 0x37c28,
3835                 0x37c3c, 0x37c50,
3836                 0x37cf0, 0x37cfc,
3837                 0x38000, 0x38030,
3838                 0x38100, 0x38144,
3839                 0x38190, 0x381d0,
3840                 0x38200, 0x38318,
3841                 0x38400, 0x3852c,
3842                 0x38540, 0x3861c,
3843                 0x38800, 0x38834,
3844                 0x388c0, 0x38908,
3845                 0x38910, 0x389ac,
3846                 0x38a00, 0x38a2c,
3847                 0x38a44, 0x38a50,
3848                 0x38a74, 0x38c24,
3849                 0x38d00, 0x38d00,
3850                 0x38d08, 0x38d14,
3851                 0x38d1c, 0x38d20,
3852                 0x38d3c, 0x38d50,
3853                 0x39200, 0x3920c,
3854                 0x39220, 0x39220,
3855                 0x39240, 0x39240,
3856                 0x39600, 0x3960c,
3857                 0x39a00, 0x39a1c,
3858                 0x39e00, 0x39e20,
3859                 0x39e38, 0x39e3c,
3860                 0x39e80, 0x39e80,
3861                 0x39e88, 0x39ea8,
3862                 0x39eb0, 0x39eb4,
3863                 0x39ec8, 0x39ed4,
3864                 0x39fb8, 0x3a004,
3865                 0x3a200, 0x3a200,
3866                 0x3a208, 0x3a240,
3867                 0x3a248, 0x3a280,
3868                 0x3a288, 0x3a2c0,
3869                 0x3a2c8, 0x3a2fc,
3870                 0x3a600, 0x3a630,
3871                 0x3aa00, 0x3aabc,
3872                 0x3ab00, 0x3ab70,
3873                 0x3b000, 0x3b048,
3874                 0x3b060, 0x3b09c,
3875                 0x3b0f0, 0x3b148,
3876                 0x3b160, 0x3b19c,
3877                 0x3b1f0, 0x3b2e4,
3878                 0x3b2f8, 0x3b3e4,
3879                 0x3b3f8, 0x3b448,
3880                 0x3b460, 0x3b49c,
3881                 0x3b4f0, 0x3b548,
3882                 0x3b560, 0x3b59c,
3883                 0x3b5f0, 0x3b6e4,
3884                 0x3b6f8, 0x3b7e4,
3885                 0x3b7f8, 0x3b7fc,
3886                 0x3b814, 0x3b814,
3887                 0x3b82c, 0x3b82c,
3888                 0x3b880, 0x3b88c,
3889                 0x3b8e8, 0x3b8ec,
3890                 0x3b900, 0x3b948,
3891                 0x3b960, 0x3b99c,
3892                 0x3b9f0, 0x3bae4,
3893                 0x3baf8, 0x3bb10,
3894                 0x3bb28, 0x3bb28,
3895                 0x3bb3c, 0x3bb50,
3896                 0x3bbf0, 0x3bc10,
3897                 0x3bc28, 0x3bc28,
3898                 0x3bc3c, 0x3bc50,
3899                 0x3bcf0, 0x3bcfc,
3900                 0x3c000, 0x3c030,
3901                 0x3c100, 0x3c144,
3902                 0x3c190, 0x3c1d0,
3903                 0x3c200, 0x3c318,
3904                 0x3c400, 0x3c52c,
3905                 0x3c540, 0x3c61c,
3906                 0x3c800, 0x3c834,
3907                 0x3c8c0, 0x3c908,
3908                 0x3c910, 0x3c9ac,
3909                 0x3ca00, 0x3ca2c,
3910                 0x3ca44, 0x3ca50,
3911                 0x3ca74, 0x3cc24,
3912                 0x3cd00, 0x3cd00,
3913                 0x3cd08, 0x3cd14,
3914                 0x3cd1c, 0x3cd20,
3915                 0x3cd3c, 0x3cd50,
3916                 0x3d200, 0x3d20c,
3917                 0x3d220, 0x3d220,
3918                 0x3d240, 0x3d240,
3919                 0x3d600, 0x3d60c,
3920                 0x3da00, 0x3da1c,
3921                 0x3de00, 0x3de20,
3922                 0x3de38, 0x3de3c,
3923                 0x3de80, 0x3de80,
3924                 0x3de88, 0x3dea8,
3925                 0x3deb0, 0x3deb4,
3926                 0x3dec8, 0x3ded4,
3927                 0x3dfb8, 0x3e004,
3928                 0x3e200, 0x3e200,
3929                 0x3e208, 0x3e240,
3930                 0x3e248, 0x3e280,
3931                 0x3e288, 0x3e2c0,
3932                 0x3e2c8, 0x3e2fc,
3933                 0x3e600, 0x3e630,
3934                 0x3ea00, 0x3eabc,
3935                 0x3eb00, 0x3eb70,
3936                 0x3f000, 0x3f048,
3937                 0x3f060, 0x3f09c,
3938                 0x3f0f0, 0x3f148,
3939                 0x3f160, 0x3f19c,
3940                 0x3f1f0, 0x3f2e4,
3941                 0x3f2f8, 0x3f3e4,
3942                 0x3f3f8, 0x3f448,
3943                 0x3f460, 0x3f49c,
3944                 0x3f4f0, 0x3f548,
3945                 0x3f560, 0x3f59c,
3946                 0x3f5f0, 0x3f6e4,
3947                 0x3f6f8, 0x3f7e4,
3948                 0x3f7f8, 0x3f7fc,
3949                 0x3f814, 0x3f814,
3950                 0x3f82c, 0x3f82c,
3951                 0x3f880, 0x3f88c,
3952                 0x3f8e8, 0x3f8ec,
3953                 0x3f900, 0x3f948,
3954                 0x3f960, 0x3f99c,
3955                 0x3f9f0, 0x3fae4,
3956                 0x3faf8, 0x3fb10,
3957                 0x3fb28, 0x3fb28,
3958                 0x3fb3c, 0x3fb50,
3959                 0x3fbf0, 0x3fc10,
3960                 0x3fc28, 0x3fc28,
3961                 0x3fc3c, 0x3fc50,
3962                 0x3fcf0, 0x3fcfc,
3963                 0x40000, 0x4000c,
3964                 0x40040, 0x40068,
3965                 0x4007c, 0x40144,
3966                 0x40180, 0x4018c,
3967                 0x40200, 0x40298,
3968                 0x402ac, 0x4033c,
3969                 0x403f8, 0x403fc,
3970                 0x41304, 0x413c4,
3971                 0x41400, 0x4141c,
3972                 0x41480, 0x414d0,
3973                 0x44000, 0x44078,
3974                 0x440c0, 0x44278,
3975                 0x442c0, 0x44478,
3976                 0x444c0, 0x44678,
3977                 0x446c0, 0x44878,
3978                 0x448c0, 0x449fc,
3979                 0x45000, 0x45068,
3980                 0x45080, 0x45084,
3981                 0x450a0, 0x450b0,
3982                 0x45200, 0x45268,
3983                 0x45280, 0x45284,
3984                 0x452a0, 0x452b0,
3985                 0x460c0, 0x460e4,
3986                 0x47000, 0x4708c,
3987                 0x47200, 0x47250,
3988                 0x47400, 0x47420,
3989                 0x47600, 0x47618,
3990                 0x47800, 0x47814,
3991                 0x48000, 0x4800c,
3992                 0x48040, 0x48068,
3993                 0x4807c, 0x48144,
3994                 0x48180, 0x4818c,
3995                 0x48200, 0x48298,
3996                 0x482ac, 0x4833c,
3997                 0x483f8, 0x483fc,
3998                 0x49304, 0x493c4,
3999                 0x49400, 0x4941c,
4000                 0x49480, 0x494d0,
4001                 0x4c000, 0x4c078,
4002                 0x4c0c0, 0x4c278,
4003                 0x4c2c0, 0x4c478,
4004                 0x4c4c0, 0x4c678,
4005                 0x4c6c0, 0x4c878,
4006                 0x4c8c0, 0x4c9fc,
4007                 0x4d000, 0x4d068,
4008                 0x4d080, 0x4d084,
4009                 0x4d0a0, 0x4d0b0,
4010                 0x4d200, 0x4d268,
4011                 0x4d280, 0x4d284,
4012                 0x4d2a0, 0x4d2b0,
4013                 0x4e0c0, 0x4e0e4,
4014                 0x4f000, 0x4f08c,
4015                 0x4f200, 0x4f250,
4016                 0x4f400, 0x4f420,
4017                 0x4f600, 0x4f618,
4018                 0x4f800, 0x4f814,
4019                 0x50000, 0x500cc,
4020                 0x50400, 0x50400,
4021                 0x50800, 0x508cc,
4022                 0x50c00, 0x50c00,
4023                 0x51000, 0x5101c,
4024                 0x51300, 0x51308,
4025         };
4026
4027         if (is_t4(sc)) {
4028                 reg_ranges = &t4_reg_ranges[0];
4029                 n = nitems(t4_reg_ranges);
4030         } else {
4031                 reg_ranges = &t5_reg_ranges[0];
4032                 n = nitems(t5_reg_ranges);
4033         }
4034
4035         regs->version = chip_id(sc) | chip_rev(sc) << 10;
4036         for (i = 0; i < n; i += 2)
4037                 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
4038 }
4039
4040 static void
4041 cxgbe_tick(void *arg)
4042 {
4043         struct port_info *pi = arg;
4044         struct adapter *sc = pi->adapter;
4045         struct ifnet *ifp = pi->ifp;
4046         struct sge_txq *txq;
4047         int i, drops;
4048         struct port_stats *s = &pi->stats;
4049
4050         PORT_LOCK(pi);
4051         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4052                 PORT_UNLOCK(pi);
4053                 return; /* without scheduling another callout */
4054         }
4055
4056         t4_get_port_stats(sc, pi->tx_chan, s);
4057
4058         ifp->if_opackets = s->tx_frames - s->tx_pause;
4059         ifp->if_ipackets = s->rx_frames - s->rx_pause;
4060         ifp->if_obytes = s->tx_octets - s->tx_pause * 64;
4061         ifp->if_ibytes = s->rx_octets - s->rx_pause * 64;
4062         ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause;
4063         ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause;
4064         ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
4065             s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
4066             s->rx_trunc3;
4067         for (i = 0; i < 4; i++) {
4068                 if (pi->rx_chan_map & (1 << i)) {
4069                         uint32_t v;
4070
4071                         /*
4072                          * XXX: indirect reads from the same ADDR/DATA pair can
4073                          * race with each other.
4074                          */
4075                         t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
4076                             1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4077                         ifp->if_iqdrops += v;
4078                 }
4079         }
4080
4081         drops = s->tx_drop;
4082         for_each_txq(pi, i, txq)
4083                 drops += txq->br->br_drops;
4084         ifp->if_snd.ifq_drops = drops;
4085
4086         ifp->if_oerrors = s->tx_error_frames;
4087         ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long +
4088             s->rx_fcs_err + s->rx_len_err;
4089
4090         callout_schedule(&pi->tick, hz);
4091         PORT_UNLOCK(pi);
4092 }
4093
4094 static void
4095 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4096 {
4097         struct ifnet *vlan;
4098
4099         if (arg != ifp || ifp->if_type != IFT_ETHER)
4100                 return;
4101
4102         vlan = VLAN_DEVAT(ifp, vid);
4103         VLAN_SETCOOKIE(vlan, ifp);
4104 }
4105
4106 static int
4107 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4108 {
4109
4110 #ifdef INVARIANTS
4111         panic("%s: opcode 0x%02x on iq %p with payload %p",
4112             __func__, rss->opcode, iq, m);
4113 #else
4114         log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
4115             __func__, rss->opcode, iq, m);
4116         m_freem(m);
4117 #endif
4118         return (EDOOFUS);
4119 }
4120
4121 int
4122 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
4123 {
4124         uintptr_t *loc, new;
4125
4126         if (opcode >= nitems(sc->cpl_handler))
4127                 return (EINVAL);
4128
4129         new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
4130         loc = (uintptr_t *) &sc->cpl_handler[opcode];
4131         atomic_store_rel_ptr(loc, new);
4132
4133         return (0);
4134 }
4135
4136 static int
4137 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
4138 {
4139
4140 #ifdef INVARIANTS
4141         panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
4142 #else
4143         log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
4144             __func__, iq, ctrl);
4145 #endif
4146         return (EDOOFUS);
4147 }
4148
4149 int
4150 t4_register_an_handler(struct adapter *sc, an_handler_t h)
4151 {
4152         uintptr_t *loc, new;
4153
4154         new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
4155         loc = (uintptr_t *) &sc->an_handler;
4156         atomic_store_rel_ptr(loc, new);
4157
4158         return (0);
4159 }
4160
4161 static int
4162 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
4163 {
4164         const struct cpl_fw6_msg *cpl =
4165             __containerof(rpl, struct cpl_fw6_msg, data[0]);
4166
4167 #ifdef INVARIANTS
4168         panic("%s: fw_msg type %d", __func__, cpl->type);
4169 #else
4170         log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
4171 #endif
4172         return (EDOOFUS);
4173 }
4174
4175 int
4176 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
4177 {
4178         uintptr_t *loc, new;
4179
4180         if (type >= nitems(sc->fw_msg_handler))
4181                 return (EINVAL);
4182
4183         /*
4184          * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
4185          * handler dispatch table.  Reject any attempt to install a handler for
4186          * this subtype.
4187          */
4188         if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
4189                 return (EINVAL);
4190
4191         new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
4192         loc = (uintptr_t *) &sc->fw_msg_handler[type];
4193         atomic_store_rel_ptr(loc, new);
4194
4195         return (0);
4196 }
4197
4198 static int
4199 t4_sysctls(struct adapter *sc)
4200 {
4201         struct sysctl_ctx_list *ctx;
4202         struct sysctl_oid *oid;
4203         struct sysctl_oid_list *children, *c0;
4204         static char *caps[] = {
4205                 "\20\1PPP\2QFC\3DCBX",                  /* caps[0] linkcaps */
4206                 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL"        /* caps[1] niccaps */
4207                     "\6HASHFILTER\7ETHOFLD",
4208                 "\20\1TOE",                             /* caps[2] toecaps */
4209                 "\20\1RDDP\2RDMAC",                     /* caps[3] rdmacaps */
4210                 "\20\1INITIATOR_PDU\2TARGET_PDU"        /* caps[4] iscsicaps */
4211                     "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
4212                     "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
4213                 "\20\1INITIATOR\2TARGET\3CTRL_OFLD"     /* caps[5] fcoecaps */
4214                     "\4PO_INITIAOR\5PO_TARGET"
4215         };
4216         static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
4217
4218         ctx = device_get_sysctl_ctx(sc->dev);
4219
4220         /*
4221          * dev.t4nex.X.
4222          */
4223         oid = device_get_sysctl_tree(sc->dev);
4224         c0 = children = SYSCTL_CHILDREN(oid);
4225
4226         sc->sc_do_rxcopy = 1;
4227         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
4228             &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
4229
4230         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
4231             sc->params.nports, "# of ports");
4232
4233         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
4234             NULL, chip_rev(sc), "chip hardware revision");
4235
4236         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
4237             CTLFLAG_RD, &sc->fw_version, 0, "firmware version");
4238
4239         SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
4240             CTLFLAG_RD, &sc->cfg_file, 0, "configuration file");
4241
4242         SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
4243             sc->cfcsum, "config file checksum");
4244
4245         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
4246             CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
4247             sysctl_bitfield, "A", "available doorbells");
4248
4249         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
4250             CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
4251             sysctl_bitfield, "A", "available link capabilities");
4252
4253         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
4254             CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
4255             sysctl_bitfield, "A", "available NIC capabilities");
4256
4257         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
4258             CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
4259             sysctl_bitfield, "A", "available TCP offload capabilities");
4260
4261         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
4262             CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
4263             sysctl_bitfield, "A", "available RDMA capabilities");
4264
4265         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
4266             CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
4267             sysctl_bitfield, "A", "available iSCSI capabilities");
4268
4269         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
4270             CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
4271             sysctl_bitfield, "A", "available FCoE capabilities");
4272
4273         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
4274             sc->params.vpd.cclk, "core clock frequency (in KHz)");
4275
4276         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
4277             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
4278             sizeof(sc->sge.timer_val), sysctl_int_array, "A",
4279             "interrupt holdoff timer values (us)");
4280
4281         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
4282             CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
4283             sizeof(sc->sge.counter_val), sysctl_int_array, "A",
4284             "interrupt holdoff packet counter values");
4285
4286         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
4287             NULL, sc->tids.nftids, "number of filters");
4288
4289         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
4290             CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
4291             "chip temperature (in Celsius)");
4292
4293         t4_sge_sysctls(sc, ctx, children);
4294
4295         sc->lro_timeout = 100;
4296         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
4297             &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
4298
4299 #ifdef SBUF_DRAIN
4300         /*
4301          * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
4302          */
4303         oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
4304             CTLFLAG_RD | CTLFLAG_SKIP, NULL,
4305             "logs and miscellaneous information");
4306         children = SYSCTL_CHILDREN(oid);
4307
4308         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
4309             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4310             sysctl_cctrl, "A", "congestion control");
4311
4312         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
4313             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4314             sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
4315
4316         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
4317             CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
4318             sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
4319
4320         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
4321             CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
4322             sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
4323
4324         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
4325             CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
4326             sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
4327
4328         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
4329             CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
4330             sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
4331
4332         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
4333             CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
4334             sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
4335
4336         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
4337             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4338             sysctl_cim_la, "A", "CIM logic analyzer");
4339
4340         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
4341             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4342             sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
4343
4344         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
4345             CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
4346             sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
4347
4348         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
4349             CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
4350             sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
4351
4352         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
4353             CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
4354             sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
4355
4356         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
4357             CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
4358             sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
4359
4360         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
4361             CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
4362             sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
4363
4364         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
4365             CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
4366             sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
4367
4368         if (is_t5(sc)) {
4369                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
4370                     CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
4371                     sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
4372
4373                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
4374                     CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
4375                     sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
4376         }
4377
4378         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
4379             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4380             sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
4381
4382         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
4383             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4384             sysctl_cim_qcfg, "A", "CIM queue configuration");
4385
4386         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
4387             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4388             sysctl_cpl_stats, "A", "CPL statistics");
4389
4390         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
4391             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4392             sysctl_ddp_stats, "A", "DDP statistics");
4393
4394         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
4395             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4396             sysctl_devlog, "A", "firmware's device log");
4397
4398         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
4399             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4400             sysctl_fcoe_stats, "A", "FCoE statistics");
4401
4402         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
4403             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4404             sysctl_hw_sched, "A", "hardware scheduler ");
4405
4406         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
4407             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4408             sysctl_l2t, "A", "hardware L2 table");
4409
4410         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
4411             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4412             sysctl_lb_stats, "A", "loopback statistics");
4413
4414         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
4415             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4416             sysctl_meminfo, "A", "memory regions");
4417
4418         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
4419             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4420             sysctl_mps_tcam, "A", "MPS TCAM entries");
4421
4422         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
4423             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4424             sysctl_path_mtus, "A", "path MTUs");
4425
4426         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
4427             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4428             sysctl_pm_stats, "A", "PM statistics");
4429
4430         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
4431             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4432             sysctl_rdma_stats, "A", "RDMA statistics");
4433
4434         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
4435             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4436             sysctl_tcp_stats, "A", "TCP statistics");
4437
4438         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
4439             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4440             sysctl_tids, "A", "TID information");
4441
4442         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
4443             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4444             sysctl_tp_err_stats, "A", "TP error statistics");
4445
4446         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
4447             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4448             sysctl_tp_la, "A", "TP logic analyzer");
4449
4450         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
4451             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4452             sysctl_tx_rate, "A", "Tx rate");
4453
4454         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
4455             CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4456             sysctl_ulprx_la, "A", "ULPRX logic analyzer");
4457
4458         if (is_t5(sc)) {
4459                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
4460                     CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4461                     sysctl_wcwr_stats, "A", "write combined work requests");
4462         }
4463 #endif
4464
4465 #ifdef TCP_OFFLOAD
4466         if (is_offload(sc)) {
4467                 /*
4468                  * dev.t4nex.X.toe.
4469                  */
4470                 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
4471                     NULL, "TOE parameters");
4472                 children = SYSCTL_CHILDREN(oid);
4473
4474                 sc->tt.sndbuf = 256 * 1024;
4475                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
4476                     &sc->tt.sndbuf, 0, "max hardware send buffer size");
4477
4478                 sc->tt.ddp = 0;
4479                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
4480                     &sc->tt.ddp, 0, "DDP allowed");
4481
4482                 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
4483                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
4484                     &sc->tt.indsz, 0, "DDP max indicate size allowed");
4485
4486                 sc->tt.ddp_thres =
4487                     G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
4488                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
4489                     &sc->tt.ddp_thres, 0, "DDP threshold");
4490
4491                 sc->tt.rx_coalesce = 1;
4492                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
4493                     CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
4494         }
4495 #endif
4496
4497
4498         return (0);
4499 }
4500
4501 static int
4502 cxgbe_sysctls(struct port_info *pi)
4503 {
4504         struct sysctl_ctx_list *ctx;
4505         struct sysctl_oid *oid;
4506         struct sysctl_oid_list *children;
4507
4508         ctx = device_get_sysctl_ctx(pi->dev);
4509
4510         /*
4511          * dev.cxgbe.X.
4512          */
4513         oid = device_get_sysctl_tree(pi->dev);
4514         children = SYSCTL_CHILDREN(oid);
4515
4516         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
4517            CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
4518         if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
4519                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
4520                     CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
4521                     "PHY temperature (in Celsius)");
4522                 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
4523                     CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
4524                     "PHY firmware version");
4525         }
4526         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
4527             &pi->nrxq, 0, "# of rx queues");
4528         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
4529             &pi->ntxq, 0, "# of tx queues");
4530         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
4531             &pi->first_rxq, 0, "index of first rx queue");
4532         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
4533             &pi->first_txq, 0, "index of first tx queue");
4534         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
4535             CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU",
4536             "Reserve queue 0 for non-flowid packets");
4537
4538 #ifdef TCP_OFFLOAD
4539         if (is_offload(pi->adapter)) {
4540                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
4541                     &pi->nofldrxq, 0,
4542                     "# of rx queues for offloaded TCP connections");
4543                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
4544                     &pi->nofldtxq, 0,
4545                     "# of tx queues for offloaded TCP connections");
4546                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
4547                     CTLFLAG_RD, &pi->first_ofld_rxq, 0,
4548                     "index of first TOE rx queue");
4549                 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
4550                     CTLFLAG_RD, &pi->first_ofld_txq, 0,
4551                     "index of first TOE tx queue");
4552         }
4553 #endif
4554
4555         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
4556             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
4557             "holdoff timer index");
4558         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
4559             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
4560             "holdoff packet counter index");
4561
4562         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
4563             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
4564             "rx queue size");
4565         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
4566             CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
4567             "tx queue size");
4568
4569         /*
4570          * dev.cxgbe.X.stats.
4571          */
4572         oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
4573             NULL, "port statistics");
4574         children = SYSCTL_CHILDREN(oid);
4575
4576 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
4577         SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
4578             CTLTYPE_U64 | CTLFLAG_RD, pi->adapter, reg, \
4579             sysctl_handle_t4_reg64, "QU", desc)
4580
4581         SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
4582             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
4583         SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
4584             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
4585         SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
4586             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
4587         SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
4588             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
4589         SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
4590             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
4591         SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
4592             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
4593         SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
4594             "# of tx frames in this range",
4595             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
4596         SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
4597             "# of tx frames in this range",
4598             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
4599         SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
4600             "# of tx frames in this range",
4601             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
4602         SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
4603             "# of tx frames in this range",
4604             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
4605         SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
4606             "# of tx frames in this range",
4607             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
4608         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
4609             "# of tx frames in this range",
4610             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
4611         SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
4612             "# of tx frames in this range",
4613             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
4614         SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
4615             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
4616         SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
4617             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
4618         SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
4619             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
4620         SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
4621             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
4622         SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
4623             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
4624         SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
4625             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
4626         SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
4627             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
4628         SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
4629             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
4630         SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
4631             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
4632         SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
4633             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
4634
4635         SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
4636             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
4637         SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
4638             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
4639         SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
4640             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
4641         SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
4642             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
4643         SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
4644             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
4645         SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
4646             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
4647         SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
4648             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
4649         SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
4650             "# of frames received with bad FCS",
4651             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
4652         SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
4653             "# of frames received with length error",
4654             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
4655         SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
4656             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
4657         SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
4658             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
4659         SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
4660             "# of rx frames in this range",
4661             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
4662         SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
4663             "# of rx frames in this range",
4664             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
4665         SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
4666             "# of rx frames in this range",
4667             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
4668         SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
4669             "# of rx frames in this range",
4670             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
4671         SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
4672             "# of rx frames in this range",
4673             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
4674         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
4675             "# of rx frames in this range",
4676             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
4677         SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
4678             "# of rx frames in this range",
4679             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
4680         SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
4681             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
4682         SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
4683             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
4684         SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
4685             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
4686         SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
4687             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
4688         SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
4689             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
4690         SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
4691             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
4692         SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
4693             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
4694         SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
4695             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
4696         SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
4697             PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
4698
4699 #undef SYSCTL_ADD_T4_REG64
4700
4701 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
4702         SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
4703             &pi->stats.name, desc)
4704
4705         /* We get these from port_stats and they may be stale by upto 1s */
4706         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
4707             "# drops due to buffer-group 0 overflows");
4708         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
4709             "# drops due to buffer-group 1 overflows");
4710         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
4711             "# drops due to buffer-group 2 overflows");
4712         SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
4713             "# drops due to buffer-group 3 overflows");
4714         SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
4715             "# of buffer-group 0 truncated packets");
4716         SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
4717             "# of buffer-group 1 truncated packets");
4718         SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
4719             "# of buffer-group 2 truncated packets");
4720         SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
4721             "# of buffer-group 3 truncated packets");
4722
4723 #undef SYSCTL_ADD_T4_PORTSTAT
4724
4725         return (0);
4726 }
4727
4728 static int
4729 sysctl_int_array(SYSCTL_HANDLER_ARGS)
4730 {
4731         int rc, *i;
4732         struct sbuf sb;
4733
4734         sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
4735         for (i = arg1; arg2; arg2 -= sizeof(int), i++)
4736                 sbuf_printf(&sb, "%d ", *i);
4737         sbuf_trim(&sb);
4738         sbuf_finish(&sb);
4739         rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
4740         sbuf_delete(&sb);
4741         return (rc);
4742 }
4743
4744 static int
4745 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
4746 {
4747         int rc;
4748         struct sbuf *sb;
4749
4750         rc = sysctl_wire_old_buffer(req, 0);
4751         if (rc != 0)
4752                 return(rc);
4753
4754         sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
4755         if (sb == NULL)
4756                 return (ENOMEM);
4757
4758         sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
4759         rc = sbuf_finish(sb);
4760         sbuf_delete(sb);
4761
4762         return (rc);
4763 }
4764
4765 static int
4766 sysctl_btphy(SYSCTL_HANDLER_ARGS)
4767 {
4768         struct port_info *pi = arg1;
4769         int op = arg2;
4770         struct adapter *sc = pi->adapter;
4771         u_int v;
4772         int rc;
4773
4774         rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt");
4775         if (rc)
4776                 return (rc);
4777         /* XXX: magic numbers */
4778         rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
4779             &v);
4780         end_synchronized_op(sc, 0);
4781         if (rc)
4782                 return (rc);
4783         if (op == 0)
4784                 v /= 256;
4785
4786         rc = sysctl_handle_int(oidp, &v, 0, req);
4787         return (rc);
4788 }
4789
4790 static int
4791 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
4792 {
4793         struct port_info *pi = arg1;
4794         int rc, val;
4795
4796         val = pi->rsrv_noflowq;
4797         rc = sysctl_handle_int(oidp, &val, 0, req);
4798         if (rc != 0 || req->newptr == NULL)
4799                 return (rc);
4800
4801         if ((val >= 1) && (pi->ntxq > 1))
4802                 pi->rsrv_noflowq = 1;
4803         else
4804                 pi->rsrv_noflowq = 0;
4805
4806         return (rc);
4807 }
4808
4809 static int
4810 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
4811 {
4812         struct port_info *pi = arg1;
4813         struct adapter *sc = pi->adapter;
4814         int idx, rc, i;
4815         struct sge_rxq *rxq;
4816 #ifdef TCP_OFFLOAD
4817         struct sge_ofld_rxq *ofld_rxq;
4818 #endif
4819         uint8_t v;
4820
4821         idx = pi->tmr_idx;
4822
4823         rc = sysctl_handle_int(oidp, &idx, 0, req);
4824         if (rc != 0 || req->newptr == NULL)
4825                 return (rc);
4826
4827         if (idx < 0 || idx >= SGE_NTIMERS)
4828                 return (EINVAL);
4829
4830         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
4831             "t4tmr");
4832         if (rc)
4833                 return (rc);
4834
4835         v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
4836         for_each_rxq(pi, i, rxq) {
4837 #ifdef atomic_store_rel_8
4838                 atomic_store_rel_8(&rxq->iq.intr_params, v);
4839 #else
4840                 rxq->iq.intr_params = v;
4841 #endif
4842         }
4843 #ifdef TCP_OFFLOAD
4844         for_each_ofld_rxq(pi, i, ofld_rxq) {
4845 #ifdef atomic_store_rel_8
4846                 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
4847 #else
4848                 ofld_rxq->iq.intr_params = v;
4849 #endif
4850         }
4851 #endif
4852         pi->tmr_idx = idx;
4853
4854         end_synchronized_op(sc, LOCK_HELD);
4855         return (0);
4856 }
4857
4858 static int
4859 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
4860 {
4861         struct port_info *pi = arg1;
4862         struct adapter *sc = pi->adapter;
4863         int idx, rc;
4864
4865         idx = pi->pktc_idx;
4866
4867         rc = sysctl_handle_int(oidp, &idx, 0, req);
4868         if (rc != 0 || req->newptr == NULL)
4869                 return (rc);
4870
4871         if (idx < -1 || idx >= SGE_NCOUNTERS)
4872                 return (EINVAL);
4873
4874         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
4875             "t4pktc");
4876         if (rc)
4877                 return (rc);
4878
4879         if (pi->flags & PORT_INIT_DONE)
4880                 rc = EBUSY; /* cannot be changed once the queues are created */
4881         else
4882                 pi->pktc_idx = idx;
4883
4884         end_synchronized_op(sc, LOCK_HELD);
4885         return (rc);
4886 }
4887
4888 static int
4889 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
4890 {
4891         struct port_info *pi = arg1;
4892         struct adapter *sc = pi->adapter;
4893         int qsize, rc;
4894
4895         qsize = pi->qsize_rxq;
4896
4897         rc = sysctl_handle_int(oidp, &qsize, 0, req);
4898         if (rc != 0 || req->newptr == NULL)
4899                 return (rc);
4900
4901         if (qsize < 128 || (qsize & 7))
4902                 return (EINVAL);
4903
4904         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
4905             "t4rxqs");
4906         if (rc)
4907                 return (rc);
4908
4909         if (pi->flags & PORT_INIT_DONE)
4910                 rc = EBUSY; /* cannot be changed once the queues are created */
4911         else
4912                 pi->qsize_rxq = qsize;
4913
4914         end_synchronized_op(sc, LOCK_HELD);
4915         return (rc);
4916 }
4917
4918 static int
4919 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
4920 {
4921         struct port_info *pi = arg1;
4922         struct adapter *sc = pi->adapter;
4923         int qsize, rc;
4924
4925         qsize = pi->qsize_txq;
4926
4927         rc = sysctl_handle_int(oidp, &qsize, 0, req);
4928         if (rc != 0 || req->newptr == NULL)
4929                 return (rc);
4930
4931         /* bufring size must be powerof2 */
4932         if (qsize < 128 || !powerof2(qsize))
4933                 return (EINVAL);
4934
4935         rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
4936             "t4txqs");
4937         if (rc)
4938                 return (rc);
4939
4940         if (pi->flags & PORT_INIT_DONE)
4941                 rc = EBUSY; /* cannot be changed once the queues are created */
4942         else
4943                 pi->qsize_txq = qsize;
4944
4945         end_synchronized_op(sc, LOCK_HELD);
4946         return (rc);
4947 }
4948
4949 static int
4950 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
4951 {
4952         struct adapter *sc = arg1;
4953         int reg = arg2;
4954         uint64_t val;
4955
4956         val = t4_read_reg64(sc, reg);
4957
4958         return (sysctl_handle_64(oidp, &val, 0, req));
4959 }
4960
4961 static int
4962 sysctl_temperature(SYSCTL_HANDLER_ARGS)
4963 {
4964         struct adapter *sc = arg1;
4965         int rc, t;
4966         uint32_t param, val;
4967
4968         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
4969         if (rc)
4970                 return (rc);
4971         param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
4972             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
4973             V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
4974         rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
4975         end_synchronized_op(sc, 0);
4976         if (rc)
4977                 return (rc);
4978
4979         /* unknown is returned as 0 but we display -1 in that case */
4980         t = val == 0 ? -1 : val;
4981
4982         rc = sysctl_handle_int(oidp, &t, 0, req);
4983         return (rc);
4984 }
4985
4986 #ifdef SBUF_DRAIN
4987 static int
4988 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
4989 {
4990         struct adapter *sc = arg1;
4991         struct sbuf *sb;
4992         int rc, i;
4993         uint16_t incr[NMTUS][NCCTRL_WIN];
4994         static const char *dec_fac[] = {
4995                 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
4996                 "0.9375"
4997         };
4998
4999         rc = sysctl_wire_old_buffer(req, 0);
5000         if (rc != 0)
5001                 return (rc);
5002
5003         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5004         if (sb == NULL)
5005                 return (ENOMEM);
5006
5007         t4_read_cong_tbl(sc, incr);
5008
5009         for (i = 0; i < NCCTRL_WIN; ++i) {
5010                 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
5011                     incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
5012                     incr[5][i], incr[6][i], incr[7][i]);
5013                 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
5014                     incr[8][i], incr[9][i], incr[10][i], incr[11][i],
5015                     incr[12][i], incr[13][i], incr[14][i], incr[15][i],
5016                     sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
5017         }
5018
5019         rc = sbuf_finish(sb);
5020         sbuf_delete(sb);
5021
5022         return (rc);
5023 }
5024
5025 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
5026         "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI",   /* ibq's */
5027         "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
5028         "SGE0-RX", "SGE1-RX"    /* additional obq's (T5 onwards) */
5029 };
5030
5031 static int
5032 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
5033 {
5034         struct adapter *sc = arg1;
5035         struct sbuf *sb;
5036         int rc, i, n, qid = arg2;
5037         uint32_t *buf, *p;
5038         char *qtype;
5039         u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
5040
5041         KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
5042             ("%s: bad qid %d\n", __func__, qid));
5043
5044         if (qid < CIM_NUM_IBQ) {
5045                 /* inbound queue */
5046                 qtype = "IBQ";
5047                 n = 4 * CIM_IBQ_SIZE;
5048                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5049                 rc = t4_read_cim_ibq(sc, qid, buf, n);
5050         } else {
5051                 /* outbound queue */
5052                 qtype = "OBQ";
5053                 qid -= CIM_NUM_IBQ;
5054                 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
5055                 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5056                 rc = t4_read_cim_obq(sc, qid, buf, n);
5057         }
5058
5059         if (rc < 0) {
5060                 rc = -rc;
5061                 goto done;
5062         }
5063         n = rc * sizeof(uint32_t);      /* rc has # of words actually read */
5064
5065         rc = sysctl_wire_old_buffer(req, 0);
5066         if (rc != 0)
5067                 goto done;
5068
5069         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5070         if (sb == NULL) {
5071                 rc = ENOMEM;
5072                 goto done;
5073         }
5074
5075         sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
5076         for (i = 0, p = buf; i < n; i += 16, p += 4)
5077                 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
5078                     p[2], p[3]);
5079
5080         rc = sbuf_finish(sb);
5081         sbuf_delete(sb);
5082 done:
5083         free(buf, M_CXGBE);
5084         return (rc);
5085 }
5086
5087 static int
5088 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
5089 {
5090         struct adapter *sc = arg1;
5091         u_int cfg;
5092         struct sbuf *sb;
5093         uint32_t *buf, *p;
5094         int rc;
5095
5096         rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
5097         if (rc != 0)
5098                 return (rc);
5099
5100         rc = sysctl_wire_old_buffer(req, 0);
5101         if (rc != 0)
5102                 return (rc);
5103
5104         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5105         if (sb == NULL)
5106                 return (ENOMEM);
5107
5108         buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
5109             M_ZERO | M_WAITOK);
5110
5111         rc = -t4_cim_read_la(sc, buf, NULL);
5112         if (rc != 0)
5113                 goto done;
5114
5115         sbuf_printf(sb, "Status   Data      PC%s",
5116             cfg & F_UPDBGLACAPTPCONLY ? "" :
5117             "     LS0Stat  LS0Addr             LS0Data");
5118
5119         KASSERT((sc->params.cim_la_size & 7) == 0,
5120             ("%s: p will walk off the end of buf", __func__));
5121
5122         for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
5123                 if (cfg & F_UPDBGLACAPTPCONLY) {
5124                         sbuf_printf(sb, "\n  %02x   %08x %08x", p[5] & 0xff,
5125                             p[6], p[7]);
5126                         sbuf_printf(sb, "\n  %02x   %02x%06x %02x%06x",
5127                             (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
5128                             p[4] & 0xff, p[5] >> 8);
5129                         sbuf_printf(sb, "\n  %02x   %x%07x %x%07x",
5130                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5131                             p[1] & 0xf, p[2] >> 4);
5132                 } else {
5133                         sbuf_printf(sb,
5134                             "\n  %02x   %x%07x %x%07x %08x %08x "
5135                             "%08x%08x%08x%08x",
5136                             (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5137                             p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
5138                             p[6], p[7]);
5139                 }
5140         }
5141
5142         rc = sbuf_finish(sb);
5143         sbuf_delete(sb);
5144 done:
5145         free(buf, M_CXGBE);
5146         return (rc);
5147 }
5148
5149 static int
5150 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
5151 {
5152         struct adapter *sc = arg1;
5153         u_int i;
5154         struct sbuf *sb;
5155         uint32_t *buf, *p;
5156         int rc;
5157
5158         rc = sysctl_wire_old_buffer(req, 0);
5159         if (rc != 0)
5160                 return (rc);
5161
5162         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5163         if (sb == NULL)
5164                 return (ENOMEM);
5165
5166         buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
5167             M_ZERO | M_WAITOK);
5168
5169         t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
5170         p = buf;
5171
5172         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5173                 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
5174                     p[1], p[0]);
5175         }
5176
5177         sbuf_printf(sb, "\n\nCnt ID Tag UE       Data       RDY VLD");
5178         for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5179                 sbuf_printf(sb, "\n%3u %2u  %x   %u %08x%08x  %u   %u",
5180                     (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
5181                     (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
5182                     (p[1] >> 2) | ((p[2] & 3) << 30),
5183                     (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
5184                     p[0] & 1);
5185         }
5186
5187         rc = sbuf_finish(sb);
5188         sbuf_delete(sb);
5189         free(buf, M_CXGBE);
5190         return (rc);
5191 }
5192
5193 static int
5194 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
5195 {
5196         struct adapter *sc = arg1;
5197         u_int i;
5198         struct sbuf *sb;
5199         uint32_t *buf, *p;
5200         int rc;
5201
5202         rc = sysctl_wire_old_buffer(req, 0);
5203         if (rc != 0)
5204                 return (rc);
5205
5206         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5207         if (sb == NULL)
5208                 return (ENOMEM);
5209
5210         buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
5211             M_ZERO | M_WAITOK);
5212
5213         t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
5214         p = buf;
5215
5216         sbuf_printf(sb, "Cntl ID DataBE   Addr                 Data");
5217         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5218                 sbuf_printf(sb, "\n %02x  %02x  %04x  %08x %08x%08x%08x%08x",
5219                     (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
5220                     p[4], p[3], p[2], p[1], p[0]);
5221         }
5222
5223         sbuf_printf(sb, "\n\nCntl ID               Data");
5224         for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5225                 sbuf_printf(sb, "\n %02x  %02x %08x%08x%08x%08x",
5226                     (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
5227         }
5228
5229         rc = sbuf_finish(sb);
5230         sbuf_delete(sb);
5231         free(buf, M_CXGBE);
5232         return (rc);
5233 }
5234
5235 static int
5236 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
5237 {
5238         struct adapter *sc = arg1;
5239         struct sbuf *sb;
5240         int rc, i;
5241         uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5242         uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5243         uint16_t thres[CIM_NUM_IBQ];
5244         uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
5245         uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
5246         u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
5247
5248         if (is_t4(sc)) {
5249                 cim_num_obq = CIM_NUM_OBQ;
5250                 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
5251                 obq_rdaddr = A_UP_OBQ_0_REALADDR;
5252         } else {
5253                 cim_num_obq = CIM_NUM_OBQ_T5;
5254                 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
5255                 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
5256         }
5257         nq = CIM_NUM_IBQ + cim_num_obq;
5258
5259         rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
5260         if (rc == 0)
5261                 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
5262         if (rc != 0)
5263                 return (rc);
5264
5265         t4_read_cimq_cfg(sc, base, size, thres);
5266
5267         rc = sysctl_wire_old_buffer(req, 0);
5268         if (rc != 0)
5269                 return (rc);
5270
5271         sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5272         if (sb == NULL)
5273                 return (ENOMEM);
5274
5275         sbuf_printf(sb, "Queue  Base  Size Thres RdPtr WrPtr  SOP  EOP Avail");
5276
5277         for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
5278                 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x  %4x %4u %4u %5u",
5279                     qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
5280                     G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5281                     G_QUEREMFLITS(p[2]) * 16);
5282         for ( ; i < nq; i++, p += 4, wr += 2)
5283                 sbuf_printf(sb, "\n%7s %5x %5u %12x  %4x %4u %4u %5u", qname[i],
5284                     base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
5285                     wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5286                     G_QUEREMFLITS(p[2]) * 16);
5287
5288         rc = sbuf_finish(sb);
5289         sbuf_delete(sb);
5290
5291         return (rc);
5292 }
5293
5294 static int
5295 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
5296 {
5297         struct adapter *sc = arg1;
5298         struct sbuf *sb;
5299         int rc;
5300         struct tp_cpl_stats stats;
5301
5302         rc = sysctl_wire_old_buffer(req, 0);
5303         if (rc != 0)
5304                 return (rc);
5305
5306         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5307         if (sb == NULL)
5308                 return (ENOMEM);
5309
5310         t4_tp_get_cpl_stats(sc, &stats);
5311
5312         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
5313             "channel 3\n");
5314         sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
5315                    stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
5316         sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
5317                    stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
5318
5319         rc = sbuf_finish(sb);
5320         sbuf_delete(sb);
5321
5322         return (rc);
5323 }
5324
5325 static int
5326 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
5327 {
5328         struct adapter *sc = arg1;
5329         struct sbuf *sb;
5330         int rc;
5331         struct tp_usm_stats stats;
5332
5333         rc = sysctl_wire_old_buffer(req, 0);
5334         if (rc != 0)
5335                 return(rc);
5336
5337         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5338         if (sb == NULL)
5339                 return (ENOMEM);
5340
5341         t4_get_usm_stats(sc, &stats);
5342
5343         sbuf_printf(sb, "Frames: %u\n", stats.frames);
5344         sbuf_printf(sb, "Octets: %ju\n", stats.octets);
5345         sbuf_printf(sb, "Drops:  %u", stats.drops);
5346
5347         rc = sbuf_finish(sb);
5348         sbuf_delete(sb);
5349
5350         return (rc);
5351 }
5352
5353 const char *devlog_level_strings[] = {
5354         [FW_DEVLOG_LEVEL_EMERG]         = "EMERG",
5355         [FW_DEVLOG_LEVEL_CRIT]          = "CRIT",
5356         [FW_DEVLOG_LEVEL_ERR]           = "ERR",
5357         [FW_DEVLOG_LEVEL_NOTICE]        = "NOTICE",
5358         [FW_DEVLOG_LEVEL_INFO]          = "INFO",
5359         [FW_DEVLOG_LEVEL_DEBUG]         = "DEBUG"
5360 };
5361
5362 const char *devlog_facility_strings[] = {
5363         [FW_DEVLOG_FACILITY_CORE]       = "CORE",
5364         [FW_DEVLOG_FACILITY_SCHED]      = "SCHED",
5365         [FW_DEVLOG_FACILITY_TIMER]      = "TIMER",
5366         [FW_DEVLOG_FACILITY_RES]        = "RES",
5367         [FW_DEVLOG_FACILITY_HW]         = "HW",
5368         [FW_DEVLOG_FACILITY_FLR]        = "FLR",
5369         [FW_DEVLOG_FACILITY_DMAQ]       = "DMAQ",
5370         [FW_DEVLOG_FACILITY_PHY]        = "PHY",
5371         [FW_DEVLOG_FACILITY_MAC]        = "MAC",
5372         [FW_DEVLOG_FACILITY_PORT]       = "PORT",
5373         [FW_DEVLOG_FACILITY_VI]         = "VI",
5374         [FW_DEVLOG_FACILITY_FILTER]     = "FILTER",
5375         [FW_DEVLOG_FACILITY_ACL]        = "ACL",
5376         [FW_DEVLOG_FACILITY_TM]         = "TM",
5377         [FW_DEVLOG_FACILITY_QFC]        = "QFC",
5378         [FW_DEVLOG_FACILITY_DCB]        = "DCB",
5379         [FW_DEVLOG_FACILITY_ETH]        = "ETH",
5380         [FW_DEVLOG_FACILITY_OFLD]       = "OFLD",
5381         [FW_DEVLOG_FACILITY_RI]         = "RI",
5382         [FW_DEVLOG_FACILITY_ISCSI]      = "ISCSI",
5383         [FW_DEVLOG_FACILITY_FCOE]       = "FCOE",
5384         [FW_DEVLOG_FACILITY_FOISCSI]    = "FOISCSI",
5385         [FW_DEVLOG_FACILITY_FOFCOE]     = "FOFCOE"
5386 };
5387
5388 static int
5389 sysctl_devlog(SYSCTL_HANDLER_ARGS)
5390 {
5391         struct adapter *sc = arg1;
5392         struct devlog_params *dparams = &sc->params.devlog;
5393         struct fw_devlog_e *buf, *e;
5394         int i, j, rc, nentries, first = 0, m;
5395         struct sbuf *sb;
5396         uint64_t ftstamp = UINT64_MAX;
5397
5398         if (dparams->start == 0) {
5399                 dparams->memtype = FW_MEMTYPE_EDC0;
5400                 dparams->start = 0x84000;
5401                 dparams->size = 32768;
5402         }
5403
5404         nentries = dparams->size / sizeof(struct fw_devlog_e);
5405
5406         buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
5407         if (buf == NULL)
5408                 return (ENOMEM);
5409
5410         m = fwmtype_to_hwmtype(dparams->memtype);
5411         rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
5412         if (rc != 0)
5413                 goto done;
5414
5415         for (i = 0; i < nentries; i++) {
5416                 e = &buf[i];
5417
5418                 if (e->timestamp == 0)
5419                         break;  /* end */
5420
5421                 e->timestamp = be64toh(e->timestamp);
5422                 e->seqno = be32toh(e->seqno);
5423                 for (j = 0; j < 8; j++)
5424                         e->params[j] = be32toh(e->params[j]);
5425
5426                 if (e->timestamp < ftstamp) {
5427                         ftstamp = e->timestamp;
5428                         first = i;
5429                 }
5430         }
5431
5432         if (buf[first].timestamp == 0)
5433                 goto done;      /* nothing in the log */
5434
5435         rc = sysctl_wire_old_buffer(req, 0);
5436         if (rc != 0)
5437                 goto done;
5438
5439         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5440         if (sb == NULL) {
5441                 rc = ENOMEM;
5442                 goto done;
5443         }
5444         sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
5445             "Seq#", "Tstamp", "Level", "Facility", "Message");
5446
5447         i = first;
5448         do {
5449                 e = &buf[i];
5450                 if (e->timestamp == 0)
5451                         break;  /* end */
5452
5453                 sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
5454                     e->seqno, e->timestamp,
5455                     (e->level < nitems(devlog_level_strings) ?
5456                         devlog_level_strings[e->level] : "UNKNOWN"),
5457                     (e->facility < nitems(devlog_facility_strings) ?
5458                         devlog_facility_strings[e->facility] : "UNKNOWN"));
5459                 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
5460                     e->params[2], e->params[3], e->params[4],
5461                     e->params[5], e->params[6], e->params[7]);
5462
5463                 if (++i == nentries)
5464                         i = 0;
5465         } while (i != first);
5466
5467         rc = sbuf_finish(sb);
5468         sbuf_delete(sb);
5469 done:
5470         free(buf, M_CXGBE);
5471         return (rc);
5472 }
5473
5474 static int
5475 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
5476 {
5477         struct adapter *sc = arg1;
5478         struct sbuf *sb;
5479         int rc;
5480         struct tp_fcoe_stats stats[4];
5481
5482         rc = sysctl_wire_old_buffer(req, 0);
5483         if (rc != 0)
5484                 return (rc);
5485
5486         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5487         if (sb == NULL)
5488                 return (ENOMEM);
5489
5490         t4_get_fcoe_stats(sc, 0, &stats[0]);
5491         t4_get_fcoe_stats(sc, 1, &stats[1]);
5492         t4_get_fcoe_stats(sc, 2, &stats[2]);
5493         t4_get_fcoe_stats(sc, 3, &stats[3]);
5494
5495         sbuf_printf(sb, "                   channel 0        channel 1        "
5496             "channel 2        channel 3\n");
5497         sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
5498             stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
5499             stats[3].octetsDDP);
5500         sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
5501             stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
5502         sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
5503             stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
5504             stats[3].framesDrop);
5505
5506         rc = sbuf_finish(sb);
5507         sbuf_delete(sb);
5508
5509         return (rc);
5510 }
5511
5512 static int
5513 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
5514 {
5515         struct adapter *sc = arg1;
5516         struct sbuf *sb;
5517         int rc, i;
5518         unsigned int map, kbps, ipg, mode;
5519         unsigned int pace_tab[NTX_SCHED];
5520
5521         rc = sysctl_wire_old_buffer(req, 0);
5522         if (rc != 0)
5523                 return (rc);
5524
5525         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5526         if (sb == NULL)
5527                 return (ENOMEM);
5528
5529         map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
5530         mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
5531         t4_read_pace_tbl(sc, pace_tab);
5532
5533         sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
5534             "Class IPG (0.1 ns)   Flow IPG (us)");
5535
5536         for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
5537                 t4_get_tx_sched(sc, i, &kbps, &ipg);
5538                 sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
5539                     (mode & (1 << i)) ? "flow" : "class", map & 3);
5540                 if (kbps)
5541                         sbuf_printf(sb, "%9u     ", kbps);
5542                 else
5543                         sbuf_printf(sb, " disabled     ");
5544
5545                 if (ipg)
5546                         sbuf_printf(sb, "%13u        ", ipg);
5547                 else
5548                         sbuf_printf(sb, "     disabled        ");
5549
5550                 if (pace_tab[i])
5551                         sbuf_printf(sb, "%10u", pace_tab[i]);
5552                 else
5553                         sbuf_printf(sb, "  disabled");
5554         }
5555
5556         rc = sbuf_finish(sb);
5557         sbuf_delete(sb);
5558
5559         return (rc);
5560 }
5561
5562 static int
5563 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
5564 {
5565         struct adapter *sc = arg1;
5566         struct sbuf *sb;
5567         int rc, i, j;
5568         uint64_t *p0, *p1;
5569         struct lb_port_stats s[2];
5570         static const char *stat_name[] = {
5571                 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
5572                 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
5573                 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
5574                 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
5575                 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
5576                 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
5577                 "BG2FramesTrunc:", "BG3FramesTrunc:"
5578         };
5579
5580         rc = sysctl_wire_old_buffer(req, 0);
5581         if (rc != 0)
5582                 return (rc);
5583
5584         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5585         if (sb == NULL)
5586                 return (ENOMEM);
5587
5588         memset(s, 0, sizeof(s));
5589
5590         for (i = 0; i < 4; i += 2) {
5591                 t4_get_lb_stats(sc, i, &s[0]);
5592                 t4_get_lb_stats(sc, i + 1, &s[1]);
5593
5594                 p0 = &s[0].octets;
5595                 p1 = &s[1].octets;
5596                 sbuf_printf(sb, "%s                       Loopback %u"
5597                     "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
5598
5599                 for (j = 0; j < nitems(stat_name); j++)
5600                         sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
5601                                    *p0++, *p1++);
5602         }
5603
5604         rc = sbuf_finish(sb);
5605         sbuf_delete(sb);
5606
5607         return (rc);
5608 }
5609
5610 static int
5611 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
5612 {
5613         int rc = 0;
5614         struct port_info *pi = arg1;
5615         struct sbuf *sb;
5616         static const char *linkdnreasons[] = {
5617                 "non-specific", "remote fault", "autoneg failed", "reserved3",
5618                 "PHY overheated", "unknown", "rx los", "reserved7"
5619         };
5620
5621         rc = sysctl_wire_old_buffer(req, 0);
5622         if (rc != 0)
5623                 return(rc);
5624         sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
5625         if (sb == NULL)
5626                 return (ENOMEM);
5627
5628         if (pi->linkdnrc < 0)
5629                 sbuf_printf(sb, "n/a");
5630         else if (pi->linkdnrc < nitems(linkdnreasons))
5631                 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
5632         else
5633                 sbuf_printf(sb, "%d", pi->linkdnrc);
5634
5635         rc = sbuf_finish(sb);
5636         sbuf_delete(sb);
5637
5638         return (rc);
5639 }
5640
5641 struct mem_desc {
5642         unsigned int base;
5643         unsigned int limit;
5644         unsigned int idx;
5645 };
5646
5647 static int
5648 mem_desc_cmp(const void *a, const void *b)
5649 {
5650         return ((const struct mem_desc *)a)->base -
5651                ((const struct mem_desc *)b)->base;
5652 }
5653
5654 static void
5655 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
5656     unsigned int to)
5657 {
5658         unsigned int size;
5659
5660         size = to - from + 1;
5661         if (size == 0)
5662                 return;
5663
5664         /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
5665         sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
5666 }
5667
5668 static int
5669 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
5670 {
5671         struct adapter *sc = arg1;
5672         struct sbuf *sb;
5673         int rc, i, n;
5674         uint32_t lo, hi, used, alloc;
5675         static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
5676         static const char *region[] = {
5677                 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
5678                 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
5679                 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
5680                 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
5681                 "RQUDP region:", "PBL region:", "TXPBL region:",
5682                 "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
5683                 "On-chip queues:"
5684         };
5685         struct mem_desc avail[4];
5686         struct mem_desc mem[nitems(region) + 3];        /* up to 3 holes */
5687         struct mem_desc *md = mem;
5688
5689         rc = sysctl_wire_old_buffer(req, 0);
5690         if (rc != 0)
5691                 return (rc);
5692
5693         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5694         if (sb == NULL)
5695                 return (ENOMEM);
5696
5697         for (i = 0; i < nitems(mem); i++) {
5698                 mem[i].limit = 0;
5699                 mem[i].idx = i;
5700         }
5701
5702         /* Find and sort the populated memory ranges */
5703         i = 0;
5704         lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
5705         if (lo & F_EDRAM0_ENABLE) {
5706                 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
5707                 avail[i].base = G_EDRAM0_BASE(hi) << 20;
5708                 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
5709                 avail[i].idx = 0;
5710                 i++;
5711         }
5712         if (lo & F_EDRAM1_ENABLE) {
5713                 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
5714                 avail[i].base = G_EDRAM1_BASE(hi) << 20;
5715                 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
5716                 avail[i].idx = 1;
5717                 i++;
5718         }
5719         if (lo & F_EXT_MEM_ENABLE) {
5720                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
5721                 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
5722                 avail[i].limit = avail[i].base +
5723                     (G_EXT_MEM_SIZE(hi) << 20);
5724                 avail[i].idx = is_t4(sc) ? 2 : 3;       /* Call it MC for T4 */
5725                 i++;
5726         }
5727         if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
5728                 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
5729                 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
5730                 avail[i].limit = avail[i].base +
5731                     (G_EXT_MEM1_SIZE(hi) << 20);
5732                 avail[i].idx = 4;
5733                 i++;
5734         }
5735         if (!i)                                    /* no memory available */
5736                 return 0;
5737         qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
5738
5739         (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
5740         (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
5741         (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
5742         (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
5743         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
5744         (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
5745         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
5746         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
5747         (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
5748
5749         /* the next few have explicit upper bounds */
5750         md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
5751         md->limit = md->base - 1 +
5752                     t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
5753                     G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
5754         md++;
5755
5756         md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
5757         md->limit = md->base - 1 +
5758                     t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
5759                     G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
5760         md++;
5761
5762         if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
5763                 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
5764                 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
5765                 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
5766         } else {
5767                 md->base = 0;
5768                 md->idx = nitems(region);  /* hide it */
5769         }
5770         md++;
5771
5772 #define ulp_region(reg) \
5773         md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
5774         (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
5775
5776         ulp_region(RX_ISCSI);
5777         ulp_region(RX_TDDP);
5778         ulp_region(TX_TPT);
5779         ulp_region(RX_STAG);
5780         ulp_region(RX_RQ);
5781         ulp_region(RX_RQUDP);
5782         ulp_region(RX_PBL);
5783         ulp_region(TX_PBL);
5784 #undef ulp_region
5785
5786         md->base = 0;
5787         md->idx = nitems(region);
5788         if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
5789                 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
5790                 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
5791                     A_SGE_DBVFIFO_SIZE))) << 2) - 1;
5792         }
5793         md++;
5794
5795         md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
5796         md->limit = md->base + sc->tids.ntids - 1;
5797         md++;
5798         md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
5799         md->limit = md->base + sc->tids.ntids - 1;
5800         md++;
5801
5802         md->base = sc->vres.ocq.start;
5803         if (sc->vres.ocq.size)
5804                 md->limit = md->base + sc->vres.ocq.size - 1;
5805         else
5806                 md->idx = nitems(region);  /* hide it */
5807         md++;
5808
5809         /* add any address-space holes, there can be up to 3 */
5810         for (n = 0; n < i - 1; n++)
5811                 if (avail[n].limit < avail[n + 1].base)
5812                         (md++)->base = avail[n].limit;
5813         if (avail[n].limit)
5814                 (md++)->base = avail[n].limit;
5815
5816         n = md - mem;
5817         qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
5818
5819         for (lo = 0; lo < i; lo++)
5820                 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
5821                                 avail[lo].limit - 1);
5822
5823         sbuf_printf(sb, "\n");
5824         for (i = 0; i < n; i++) {
5825                 if (mem[i].idx >= nitems(region))
5826                         continue;                        /* skip holes */
5827                 if (!mem[i].limit)
5828                         mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
5829                 mem_region_show(sb, region[mem[i].idx], mem[i].base,
5830                                 mem[i].limit);
5831         }
5832
5833         sbuf_printf(sb, "\n");
5834         lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
5835         hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
5836         mem_region_show(sb, "uP RAM:", lo, hi);
5837
5838         lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
5839         hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
5840         mem_region_show(sb, "uP Extmem2:", lo, hi);
5841
5842         lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
5843         sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
5844                    G_PMRXMAXPAGE(lo),
5845                    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
5846                    (lo & F_PMRXNUMCHN) ? 2 : 1);
5847
5848         lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
5849         hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
5850         sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
5851                    G_PMTXMAXPAGE(lo),
5852                    hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
5853                    hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
5854         sbuf_printf(sb, "%u p-structs\n",
5855                    t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
5856
5857         for (i = 0; i < 4; i++) {
5858                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
5859                 if (is_t4(sc)) {
5860                         used = G_USED(lo);
5861                         alloc = G_ALLOC(lo);
5862                 } else {
5863                         used = G_T5_USED(lo);
5864                         alloc = G_T5_ALLOC(lo);
5865                 }
5866                 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
5867                            i, used, alloc);
5868         }
5869         for (i = 0; i < 4; i++) {
5870                 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
5871                 if (is_t4(sc)) {
5872                         used = G_USED(lo);
5873                         alloc = G_ALLOC(lo);
5874                 } else {
5875                         used = G_T5_USED(lo);
5876                         alloc = G_T5_ALLOC(lo);
5877                 }
5878                 sbuf_printf(sb,
5879                            "\nLoopback %d using %u pages out of %u allocated",
5880                            i, used, alloc);
5881         }
5882
5883         rc = sbuf_finish(sb);
5884         sbuf_delete(sb);
5885
5886         return (rc);
5887 }
5888
5889 static inline void
5890 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
5891 {
5892         *mask = x | y;
5893         y = htobe64(y);
5894         memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
5895 }
5896
5897 static int
5898 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
5899 {
5900         struct adapter *sc = arg1;
5901         struct sbuf *sb;
5902         int rc, i, n;
5903
5904         rc = sysctl_wire_old_buffer(req, 0);
5905         if (rc != 0)
5906                 return (rc);
5907
5908         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5909         if (sb == NULL)
5910                 return (ENOMEM);
5911
5912         sbuf_printf(sb,
5913             "Idx  Ethernet address     Mask     Vld Ports PF"
5914             "  VF              Replication             P0 P1 P2 P3  ML");
5915         n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
5916             NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
5917         for (i = 0; i < n; i++) {
5918                 uint64_t tcamx, tcamy, mask;
5919                 uint32_t cls_lo, cls_hi;
5920                 uint8_t addr[ETHER_ADDR_LEN];
5921
5922                 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
5923                 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
5924                 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
5925                 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
5926
5927                 if (tcamx & tcamy)
5928                         continue;
5929
5930                 tcamxy2valmask(tcamx, tcamy, addr, &mask);
5931                 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
5932                            "  %c   %#x%4u%4d", i, addr[0], addr[1], addr[2],
5933                            addr[3], addr[4], addr[5], (uintmax_t)mask,
5934                            (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
5935                            G_PORTMAP(cls_hi), G_PF(cls_lo),
5936                            (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
5937
5938                 if (cls_lo & F_REPLICATE) {
5939                         struct fw_ldst_cmd ldst_cmd;
5940
5941                         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
5942                         ldst_cmd.op_to_addrspace =
5943                             htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
5944                                 F_FW_CMD_REQUEST | F_FW_CMD_READ |
5945                                 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
5946                         ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
5947                         ldst_cmd.u.mps.fid_ctl =
5948                             htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
5949                                 V_FW_LDST_CMD_CTL(i));
5950
5951                         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
5952                             "t4mps");
5953                         if (rc)
5954                                 break;
5955                         rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
5956                             sizeof(ldst_cmd), &ldst_cmd);
5957                         end_synchronized_op(sc, 0);
5958
5959                         if (rc != 0) {
5960                                 sbuf_printf(sb,
5961                                     " ------------ error %3u ------------", rc);
5962                                 rc = 0;
5963                         } else {
5964                                 sbuf_printf(sb, " %08x %08x %08x %08x",
5965                                     be32toh(ldst_cmd.u.mps.rplc127_96),
5966                                     be32toh(ldst_cmd.u.mps.rplc95_64),
5967                                     be32toh(ldst_cmd.u.mps.rplc63_32),
5968                                     be32toh(ldst_cmd.u.mps.rplc31_0));
5969                         }
5970                 } else
5971                         sbuf_printf(sb, "%36s", "");
5972
5973                 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
5974                     G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
5975                     G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
5976         }
5977
5978         if (rc)
5979                 (void) sbuf_finish(sb);
5980         else
5981                 rc = sbuf_finish(sb);
5982         sbuf_delete(sb);
5983
5984         return (rc);
5985 }
5986
5987 static int
5988 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
5989 {
5990         struct adapter *sc = arg1;
5991         struct sbuf *sb;
5992         int rc;
5993         uint16_t mtus[NMTUS];
5994
5995         rc = sysctl_wire_old_buffer(req, 0);
5996         if (rc != 0)
5997                 return (rc);
5998
5999         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6000         if (sb == NULL)
6001                 return (ENOMEM);
6002
6003         t4_read_mtu_tbl(sc, mtus, NULL);
6004
6005         sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
6006             mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
6007             mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
6008             mtus[14], mtus[15]);
6009
6010         rc = sbuf_finish(sb);
6011         sbuf_delete(sb);
6012
6013         return (rc);
6014 }
6015
6016 static int
6017 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
6018 {
6019         struct adapter *sc = arg1;
6020         struct sbuf *sb;
6021         int rc, i;
6022         uint32_t cnt[PM_NSTATS];
6023         uint64_t cyc[PM_NSTATS];
6024         static const char *rx_stats[] = {
6025                 "Read:", "Write bypass:", "Write mem:", "Flush:"
6026         };
6027         static const char *tx_stats[] = {
6028                 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
6029         };
6030
6031         rc = sysctl_wire_old_buffer(req, 0);
6032         if (rc != 0)
6033                 return (rc);
6034
6035         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6036         if (sb == NULL)
6037                 return (ENOMEM);
6038
6039         t4_pmtx_get_stats(sc, cnt, cyc);
6040         sbuf_printf(sb, "                Tx pcmds             Tx bytes");
6041         for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
6042                 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
6043                     cyc[i]);
6044
6045         t4_pmrx_get_stats(sc, cnt, cyc);
6046         sbuf_printf(sb, "\n                Rx pcmds             Rx bytes");
6047         for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
6048                 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
6049                     cyc[i]);
6050
6051         rc = sbuf_finish(sb);
6052         sbuf_delete(sb);
6053
6054         return (rc);
6055 }
6056
6057 static int
6058 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
6059 {
6060         struct adapter *sc = arg1;
6061         struct sbuf *sb;
6062         int rc;
6063         struct tp_rdma_stats stats;
6064
6065         rc = sysctl_wire_old_buffer(req, 0);
6066         if (rc != 0)
6067                 return (rc);
6068
6069         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6070         if (sb == NULL)
6071                 return (ENOMEM);
6072
6073         t4_tp_get_rdma_stats(sc, &stats);
6074         sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
6075         sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
6076
6077         rc = sbuf_finish(sb);
6078         sbuf_delete(sb);
6079
6080         return (rc);
6081 }
6082
6083 static int
6084 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
6085 {
6086         struct adapter *sc = arg1;
6087         struct sbuf *sb;
6088         int rc;
6089         struct tp_tcp_stats v4, v6;
6090
6091         rc = sysctl_wire_old_buffer(req, 0);
6092         if (rc != 0)
6093                 return (rc);
6094
6095         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6096         if (sb == NULL)
6097                 return (ENOMEM);
6098
6099         t4_tp_get_tcp_stats(sc, &v4, &v6);
6100         sbuf_printf(sb,
6101             "                                IP                 IPv6\n");
6102         sbuf_printf(sb, "OutRsts:      %20u %20u\n",
6103             v4.tcpOutRsts, v6.tcpOutRsts);
6104         sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
6105             v4.tcpInSegs, v6.tcpInSegs);
6106         sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
6107             v4.tcpOutSegs, v6.tcpOutSegs);
6108         sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
6109             v4.tcpRetransSegs, v6.tcpRetransSegs);
6110
6111         rc = sbuf_finish(sb);
6112         sbuf_delete(sb);
6113
6114         return (rc);
6115 }
6116
6117 static int
6118 sysctl_tids(SYSCTL_HANDLER_ARGS)
6119 {
6120         struct adapter *sc = arg1;
6121         struct sbuf *sb;
6122         int rc;
6123         struct tid_info *t = &sc->tids;
6124
6125         rc = sysctl_wire_old_buffer(req, 0);
6126         if (rc != 0)
6127                 return (rc);
6128
6129         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6130         if (sb == NULL)
6131                 return (ENOMEM);
6132
6133         if (t->natids) {
6134                 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
6135                     t->atids_in_use);
6136         }
6137
6138         if (t->ntids) {
6139                 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6140                         uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
6141
6142                         if (b) {
6143                                 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
6144                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6145                                     t->ntids - 1);
6146                         } else {
6147                                 sbuf_printf(sb, "TID range: %u-%u",
6148                                     t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6149                                     t->ntids - 1);
6150                         }
6151                 } else
6152                         sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
6153                 sbuf_printf(sb, ", in use: %u\n",
6154                     atomic_load_acq_int(&t->tids_in_use));
6155         }
6156
6157         if (t->nstids) {
6158                 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
6159                     t->stid_base + t->nstids - 1, t->stids_in_use);
6160         }
6161
6162         if (t->nftids) {
6163                 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
6164                     t->ftid_base + t->nftids - 1);
6165         }
6166
6167         sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
6168             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
6169             t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
6170
6171         rc = sbuf_finish(sb);
6172         sbuf_delete(sb);
6173
6174         return (rc);
6175 }
6176
6177 static int
6178 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
6179 {
6180         struct adapter *sc = arg1;
6181         struct sbuf *sb;
6182         int rc;
6183         struct tp_err_stats stats;
6184
6185         rc = sysctl_wire_old_buffer(req, 0);
6186         if (rc != 0)
6187                 return (rc);
6188
6189         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6190         if (sb == NULL)
6191                 return (ENOMEM);
6192
6193         t4_tp_get_err_stats(sc, &stats);
6194
6195         sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
6196                       "channel 3\n");
6197         sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
6198             stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
6199             stats.macInErrs[3]);
6200         sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
6201             stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
6202             stats.hdrInErrs[3]);
6203         sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
6204             stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
6205             stats.tcpInErrs[3]);
6206         sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
6207             stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
6208             stats.tcp6InErrs[3]);
6209         sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
6210             stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
6211             stats.tnlCongDrops[3]);
6212         sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
6213             stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
6214             stats.tnlTxDrops[3]);
6215         sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
6216             stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
6217             stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
6218         sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
6219             stats.ofldChanDrops[0], stats.ofldChanDrops[1],
6220             stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
6221         sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
6222             stats.ofldNoNeigh, stats.ofldCongDefer);
6223
6224         rc = sbuf_finish(sb);
6225         sbuf_delete(sb);
6226
6227         return (rc);
6228 }
6229
6230 struct field_desc {
6231         const char *name;
6232         u_int start;
6233         u_int width;
6234 };
6235
6236 static void
6237 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
6238 {
6239         char buf[32];
6240         int line_size = 0;
6241
6242         while (f->name) {
6243                 uint64_t mask = (1ULL << f->width) - 1;
6244                 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
6245                     ((uintmax_t)v >> f->start) & mask);
6246
6247                 if (line_size + len >= 79) {
6248                         line_size = 8;
6249                         sbuf_printf(sb, "\n        ");
6250                 }
6251                 sbuf_printf(sb, "%s ", buf);
6252                 line_size += len + 1;
6253                 f++;
6254         }
6255         sbuf_printf(sb, "\n");
6256 }
6257
6258 static struct field_desc tp_la0[] = {
6259         { "RcfOpCodeOut", 60, 4 },
6260         { "State", 56, 4 },
6261         { "WcfState", 52, 4 },
6262         { "RcfOpcSrcOut", 50, 2 },
6263         { "CRxError", 49, 1 },
6264         { "ERxError", 48, 1 },
6265         { "SanityFailed", 47, 1 },
6266         { "SpuriousMsg", 46, 1 },
6267         { "FlushInputMsg", 45, 1 },
6268         { "FlushInputCpl", 44, 1 },
6269         { "RssUpBit", 43, 1 },
6270         { "RssFilterHit", 42, 1 },
6271         { "Tid", 32, 10 },
6272         { "InitTcb", 31, 1 },
6273         { "LineNumber", 24, 7 },
6274         { "Emsg", 23, 1 },
6275         { "EdataOut", 22, 1 },
6276         { "Cmsg", 21, 1 },
6277         { "CdataOut", 20, 1 },
6278         { "EreadPdu", 19, 1 },
6279         { "CreadPdu", 18, 1 },
6280         { "TunnelPkt", 17, 1 },
6281         { "RcfPeerFin", 16, 1 },
6282         { "RcfReasonOut", 12, 4 },
6283         { "TxCchannel", 10, 2 },
6284         { "RcfTxChannel", 8, 2 },
6285         { "RxEchannel", 6, 2 },
6286         { "RcfRxChannel", 5, 1 },
6287         { "RcfDataOutSrdy", 4, 1 },
6288         { "RxDvld", 3, 1 },
6289         { "RxOoDvld", 2, 1 },
6290         { "RxCongestion", 1, 1 },
6291         { "TxCongestion", 0, 1 },
6292         { NULL }
6293 };
6294
6295 static struct field_desc tp_la1[] = {
6296         { "CplCmdIn", 56, 8 },
6297         { "CplCmdOut", 48, 8 },
6298         { "ESynOut", 47, 1 },
6299         { "EAckOut", 46, 1 },
6300         { "EFinOut", 45, 1 },
6301         { "ERstOut", 44, 1 },
6302         { "SynIn", 43, 1 },
6303         { "AckIn", 42, 1 },
6304         { "FinIn", 41, 1 },
6305         { "RstIn", 40, 1 },
6306         { "DataIn", 39, 1 },
6307         { "DataInVld", 38, 1 },
6308         { "PadIn", 37, 1 },
6309         { "RxBufEmpty", 36, 1 },
6310         { "RxDdp", 35, 1 },
6311         { "RxFbCongestion", 34, 1 },
6312         { "TxFbCongestion", 33, 1 },
6313         { "TxPktSumSrdy", 32, 1 },
6314         { "RcfUlpType", 28, 4 },
6315         { "Eread", 27, 1 },
6316         { "Ebypass", 26, 1 },
6317         { "Esave", 25, 1 },
6318         { "Static0", 24, 1 },
6319         { "Cread", 23, 1 },
6320         { "Cbypass", 22, 1 },
6321         { "Csave", 21, 1 },
6322         { "CPktOut", 20, 1 },
6323         { "RxPagePoolFull", 18, 2 },
6324         { "RxLpbkPkt", 17, 1 },
6325         { "TxLpbkPkt", 16, 1 },
6326         { "RxVfValid", 15, 1 },
6327         { "SynLearned", 14, 1 },
6328         { "SetDelEntry", 13, 1 },
6329         { "SetInvEntry", 12, 1 },
6330         { "CpcmdDvld", 11, 1 },
6331         { "CpcmdSave", 10, 1 },
6332         { "RxPstructsFull", 8, 2 },
6333         { "EpcmdDvld", 7, 1 },
6334         { "EpcmdFlush", 6, 1 },
6335         { "EpcmdTrimPrefix", 5, 1 },
6336         { "EpcmdTrimPostfix", 4, 1 },
6337         { "ERssIp4Pkt", 3, 1 },
6338         { "ERssIp6Pkt", 2, 1 },
6339         { "ERssTcpUdpPkt", 1, 1 },
6340         { "ERssFceFipPkt", 0, 1 },
6341         { NULL }
6342 };
6343
6344 static struct field_desc tp_la2[] = {
6345         { "CplCmdIn", 56, 8 },
6346         { "MpsVfVld", 55, 1 },
6347         { "MpsPf", 52, 3 },
6348         { "MpsVf", 44, 8 },
6349         { "SynIn", 43, 1 },
6350         { "AckIn", 42, 1 },
6351         { "FinIn", 41, 1 },
6352         { "RstIn", 40, 1 },
6353         { "DataIn", 39, 1 },
6354         { "DataInVld", 38, 1 },
6355         { "PadIn", 37, 1 },
6356         { "RxBufEmpty", 36, 1 },
6357         { "RxDdp", 35, 1 },
6358         { "RxFbCongestion", 34, 1 },
6359         { "TxFbCongestion", 33, 1 },
6360         { "TxPktSumSrdy", 32, 1 },
6361         { "RcfUlpType", 28, 4 },
6362         { "Eread", 27, 1 },
6363         { "Ebypass", 26, 1 },
6364         { "Esave", 25, 1 },
6365         { "Static0", 24, 1 },
6366         { "Cread", 23, 1 },
6367         { "Cbypass", 22, 1 },
6368         { "Csave", 21, 1 },
6369         { "CPktOut", 20, 1 },
6370         { "RxPagePoolFull", 18, 2 },
6371         { "RxLpbkPkt", 17, 1 },
6372         { "TxLpbkPkt", 16, 1 },
6373         { "RxVfValid", 15, 1 },
6374         { "SynLearned", 14, 1 },
6375         { "SetDelEntry", 13, 1 },
6376         { "SetInvEntry", 12, 1 },
6377         { "CpcmdDvld", 11, 1 },
6378         { "CpcmdSave", 10, 1 },
6379         { "RxPstructsFull", 8, 2 },
6380         { "EpcmdDvld", 7, 1 },
6381         { "EpcmdFlush", 6, 1 },
6382         { "EpcmdTrimPrefix", 5, 1 },
6383         { "EpcmdTrimPostfix", 4, 1 },
6384         { "ERssIp4Pkt", 3, 1 },
6385         { "ERssIp6Pkt", 2, 1 },
6386         { "ERssTcpUdpPkt", 1, 1 },
6387         { "ERssFceFipPkt", 0, 1 },
6388         { NULL }
6389 };
6390
6391 static void
6392 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
6393 {
6394
6395         field_desc_show(sb, *p, tp_la0);
6396 }
6397
6398 static void
6399 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
6400 {
6401
6402         if (idx)
6403                 sbuf_printf(sb, "\n");
6404         field_desc_show(sb, p[0], tp_la0);
6405         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6406                 field_desc_show(sb, p[1], tp_la0);
6407 }
6408
6409 static void
6410 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
6411 {
6412
6413         if (idx)
6414                 sbuf_printf(sb, "\n");
6415         field_desc_show(sb, p[0], tp_la0);
6416         if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6417                 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
6418 }
6419
6420 static int
6421 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
6422 {
6423         struct adapter *sc = arg1;
6424         struct sbuf *sb;
6425         uint64_t *buf, *p;
6426         int rc;
6427         u_int i, inc;
6428         void (*show_func)(struct sbuf *, uint64_t *, int);
6429
6430         rc = sysctl_wire_old_buffer(req, 0);
6431         if (rc != 0)
6432                 return (rc);
6433
6434         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6435         if (sb == NULL)
6436                 return (ENOMEM);
6437
6438         buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
6439
6440         t4_tp_read_la(sc, buf, NULL);
6441         p = buf;
6442
6443         switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
6444         case 2:
6445                 inc = 2;
6446                 show_func = tp_la_show2;
6447                 break;
6448         case 3:
6449                 inc = 2;
6450                 show_func = tp_la_show3;
6451                 break;
6452         default:
6453                 inc = 1;
6454                 show_func = tp_la_show;
6455         }
6456
6457         for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
6458                 (*show_func)(sb, p, i);
6459
6460         rc = sbuf_finish(sb);
6461         sbuf_delete(sb);
6462         free(buf, M_CXGBE);
6463         return (rc);
6464 }
6465
6466 static int
6467 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
6468 {
6469         struct adapter *sc = arg1;
6470         struct sbuf *sb;
6471         int rc;
6472         u64 nrate[NCHAN], orate[NCHAN];
6473
6474         rc = sysctl_wire_old_buffer(req, 0);
6475         if (rc != 0)
6476                 return (rc);
6477
6478         sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6479         if (sb == NULL)
6480                 return (ENOMEM);
6481
6482         t4_get_chan_txrate(sc, nrate, orate);
6483         sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
6484                  "channel 3\n");
6485         sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
6486             nrate[0], nrate[1], nrate[2], nrate[3]);
6487         sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
6488             orate[0], orate[1], orate[2], orate[3]);
6489
6490         rc = sbuf_finish(sb);
6491         sbuf_delete(sb);
6492
6493         return (rc);
6494 }
6495
6496 static int
6497 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
6498 {
6499         struct adapter *sc = arg1;
6500         struct sbuf *sb;
6501         uint32_t *buf, *p;
6502         int rc, i;
6503
6504         rc = sysctl_wire_old_buffer(req, 0);
6505         if (rc != 0)
6506                 return (rc);
6507
6508         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6509         if (sb == NULL)
6510                 return (ENOMEM);
6511
6512         buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
6513             M_ZERO | M_WAITOK);
6514
6515         t4_ulprx_read_la(sc, buf);
6516         p = buf;
6517
6518         sbuf_printf(sb, "      Pcmd        Type   Message"
6519             "                Data");
6520         for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
6521                 sbuf_printf(sb, "\n%08x%08x  %4x  %08x  %08x%08x%08x%08x",
6522                     p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
6523         }
6524
6525         rc = sbuf_finish(sb);
6526         sbuf_delete(sb);
6527         free(buf, M_CXGBE);
6528         return (rc);
6529 }
6530
6531 static int
6532 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
6533 {
6534         struct adapter *sc = arg1;
6535         struct sbuf *sb;
6536         int rc, v;
6537
6538         rc = sysctl_wire_old_buffer(req, 0);
6539         if (rc != 0)
6540                 return (rc);
6541
6542         sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6543         if (sb == NULL)
6544                 return (ENOMEM);
6545
6546         v = t4_read_reg(sc, A_SGE_STAT_CFG);
6547         if (G_STATSOURCE_T5(v) == 7) {
6548                 if (G_STATMODE(v) == 0) {
6549                         sbuf_printf(sb, "total %d, incomplete %d",
6550                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6551                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6552                 } else if (G_STATMODE(v) == 1) {
6553                         sbuf_printf(sb, "total %d, data overflow %d",
6554                             t4_read_reg(sc, A_SGE_STAT_TOTAL),
6555                             t4_read_reg(sc, A_SGE_STAT_MATCH));
6556                 }
6557         }
6558         rc = sbuf_finish(sb);
6559         sbuf_delete(sb);
6560
6561         return (rc);
6562 }
6563 #endif
6564
6565 static inline void
6566 txq_start(struct ifnet *ifp, struct sge_txq *txq)
6567 {
6568         struct buf_ring *br;
6569         struct mbuf *m;
6570
6571         TXQ_LOCK_ASSERT_OWNED(txq);
6572
6573         br = txq->br;
6574         m = txq->m ? txq->m : drbr_dequeue(ifp, br);
6575         if (m)
6576                 t4_eth_tx(ifp, txq, m);
6577 }
6578
6579 void
6580 t4_tx_callout(void *arg)
6581 {
6582         struct sge_eq *eq = arg;
6583         struct adapter *sc;
6584
6585         if (EQ_TRYLOCK(eq) == 0)
6586                 goto reschedule;
6587
6588         if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) {
6589                 EQ_UNLOCK(eq);
6590 reschedule:
6591                 if (__predict_true(!(eq->flags && EQ_DOOMED)))
6592                         callout_schedule(&eq->tx_callout, 1);
6593                 return;
6594         }
6595
6596         EQ_LOCK_ASSERT_OWNED(eq);
6597
6598         if (__predict_true((eq->flags & EQ_DOOMED) == 0)) {
6599
6600                 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
6601                         struct sge_txq *txq = arg;
6602                         struct port_info *pi = txq->ifp->if_softc;
6603
6604                         sc = pi->adapter;
6605                 } else {
6606                         struct sge_wrq *wrq = arg;
6607
6608                         sc = wrq->adapter;
6609                 }
6610
6611                 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task);
6612         }
6613
6614         EQ_UNLOCK(eq);
6615 }
6616
6617 void
6618 t4_tx_task(void *arg, int count)
6619 {
6620         struct sge_eq *eq = arg;
6621
6622         EQ_LOCK(eq);
6623         if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
6624                 struct sge_txq *txq = arg;
6625                 txq_start(txq->ifp, txq);
6626         } else {
6627                 struct sge_wrq *wrq = arg;
6628                 t4_wrq_tx_locked(wrq->adapter, wrq, NULL);
6629         }
6630         EQ_UNLOCK(eq);
6631 }
6632
6633 static uint32_t
6634 fconf_to_mode(uint32_t fconf)
6635 {
6636         uint32_t mode;
6637
6638         mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
6639             T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
6640
6641         if (fconf & F_FRAGMENTATION)
6642                 mode |= T4_FILTER_IP_FRAGMENT;
6643
6644         if (fconf & F_MPSHITTYPE)
6645                 mode |= T4_FILTER_MPS_HIT_TYPE;
6646
6647         if (fconf & F_MACMATCH)
6648                 mode |= T4_FILTER_MAC_IDX;
6649
6650         if (fconf & F_ETHERTYPE)
6651                 mode |= T4_FILTER_ETH_TYPE;
6652
6653         if (fconf & F_PROTOCOL)
6654                 mode |= T4_FILTER_IP_PROTO;
6655
6656         if (fconf & F_TOS)
6657                 mode |= T4_FILTER_IP_TOS;
6658
6659         if (fconf & F_VLAN)
6660                 mode |= T4_FILTER_VLAN;
6661
6662         if (fconf & F_VNIC_ID)
6663                 mode |= T4_FILTER_VNIC;
6664
6665         if (fconf & F_PORT)
6666                 mode |= T4_FILTER_PORT;
6667
6668         if (fconf & F_FCOE)
6669                 mode |= T4_FILTER_FCoE;
6670
6671         return (mode);
6672 }
6673
6674 static uint32_t
6675 mode_to_fconf(uint32_t mode)
6676 {
6677         uint32_t fconf = 0;
6678
6679         if (mode & T4_FILTER_IP_FRAGMENT)
6680                 fconf |= F_FRAGMENTATION;
6681
6682         if (mode & T4_FILTER_MPS_HIT_TYPE)
6683                 fconf |= F_MPSHITTYPE;
6684
6685         if (mode & T4_FILTER_MAC_IDX)
6686                 fconf |= F_MACMATCH;
6687
6688         if (mode & T4_FILTER_ETH_TYPE)
6689                 fconf |= F_ETHERTYPE;
6690
6691         if (mode & T4_FILTER_IP_PROTO)
6692                 fconf |= F_PROTOCOL;
6693
6694         if (mode & T4_FILTER_IP_TOS)
6695                 fconf |= F_TOS;
6696
6697         if (mode & T4_FILTER_VLAN)
6698                 fconf |= F_VLAN;
6699
6700         if (mode & T4_FILTER_VNIC)
6701                 fconf |= F_VNIC_ID;
6702
6703         if (mode & T4_FILTER_PORT)
6704                 fconf |= F_PORT;
6705
6706         if (mode & T4_FILTER_FCoE)
6707                 fconf |= F_FCOE;
6708
6709         return (fconf);
6710 }
6711
6712 static uint32_t
6713 fspec_to_fconf(struct t4_filter_specification *fs)
6714 {
6715         uint32_t fconf = 0;
6716
6717         if (fs->val.frag || fs->mask.frag)
6718                 fconf |= F_FRAGMENTATION;
6719
6720         if (fs->val.matchtype || fs->mask.matchtype)
6721                 fconf |= F_MPSHITTYPE;
6722
6723         if (fs->val.macidx || fs->mask.macidx)
6724                 fconf |= F_MACMATCH;
6725
6726         if (fs->val.ethtype || fs->mask.ethtype)
6727                 fconf |= F_ETHERTYPE;
6728
6729         if (fs->val.proto || fs->mask.proto)
6730                 fconf |= F_PROTOCOL;
6731
6732         if (fs->val.tos || fs->mask.tos)
6733                 fconf |= F_TOS;
6734
6735         if (fs->val.vlan_vld || fs->mask.vlan_vld)
6736                 fconf |= F_VLAN;
6737
6738         if (fs->val.vnic_vld || fs->mask.vnic_vld)
6739                 fconf |= F_VNIC_ID;
6740
6741         if (fs->val.iport || fs->mask.iport)
6742                 fconf |= F_PORT;
6743
6744         if (fs->val.fcoe || fs->mask.fcoe)
6745                 fconf |= F_FCOE;
6746
6747         return (fconf);
6748 }
6749
6750 static int
6751 get_filter_mode(struct adapter *sc, uint32_t *mode)
6752 {
6753         int rc;
6754         uint32_t fconf;
6755
6756         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
6757             "t4getfm");
6758         if (rc)
6759                 return (rc);
6760
6761         t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
6762             A_TP_VLAN_PRI_MAP);
6763
6764         if (sc->params.tp.vlan_pri_map != fconf) {
6765                 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
6766                     device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
6767                     fconf);
6768                 sc->params.tp.vlan_pri_map = fconf;
6769         }
6770
6771         *mode = fconf_to_mode(sc->params.tp.vlan_pri_map);
6772
6773         end_synchronized_op(sc, LOCK_HELD);
6774         return (0);
6775 }
6776
6777 static int
6778 set_filter_mode(struct adapter *sc, uint32_t mode)
6779 {
6780         uint32_t fconf;
6781         int rc;
6782
6783         fconf = mode_to_fconf(mode);
6784
6785         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
6786             "t4setfm");
6787         if (rc)
6788                 return (rc);
6789
6790         if (sc->tids.ftids_in_use > 0) {
6791                 rc = EBUSY;
6792                 goto done;
6793         }
6794
6795 #ifdef TCP_OFFLOAD
6796         if (sc->offload_map) {
6797                 rc = EBUSY;
6798                 goto done;
6799         }
6800 #endif
6801
6802 #ifdef notyet
6803         rc = -t4_set_filter_mode(sc, fconf);
6804         if (rc == 0)
6805                 sc->filter_mode = fconf;
6806 #else
6807         rc = ENOTSUP;
6808 #endif
6809
6810 done:
6811         end_synchronized_op(sc, LOCK_HELD);
6812         return (rc);
6813 }
6814
6815 static inline uint64_t
6816 get_filter_hits(struct adapter *sc, uint32_t fid)
6817 {
6818         uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
6819         uint64_t hits;
6820
6821         memwin_info(sc, 0, &mw_base, NULL);
6822         off = position_memwin(sc, 0,
6823             tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
6824         if (is_t4(sc)) {
6825                 hits = t4_read_reg64(sc, mw_base + off + 16);
6826                 hits = be64toh(hits);
6827         } else {
6828                 hits = t4_read_reg(sc, mw_base + off + 24);
6829                 hits = be32toh(hits);
6830         }
6831
6832         return (hits);
6833 }
6834
6835 static int
6836 get_filter(struct adapter *sc, struct t4_filter *t)
6837 {
6838         int i, rc, nfilters = sc->tids.nftids;
6839         struct filter_entry *f;
6840
6841         rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
6842             "t4getf");
6843         if (rc)
6844                 return (rc);
6845
6846         if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
6847             t->idx >= nfilters) {
6848                 t->idx = 0xffffffff;
6849                 goto done;
6850         }
6851
6852         f = &sc->tids.ftid_tab[t->idx];
6853         for (i = t->idx; i < nfilters; i++, f++) {
6854                 if (f->valid) {
6855                         t->idx = i;
6856                         t->l2tidx = f->l2t ? f->l2t->idx : 0;
6857                         t->smtidx = f->smtidx;
6858                         if (f->fs.hitcnts)
6859                                 t->hits = get_filter_hits(sc, t->idx);
6860                         else
6861                                 t->hits = UINT64_MAX;
6862                         t->fs = f->fs;
6863
6864                         goto done;
6865                 }
6866         }
6867
6868         t->idx = 0xffffffff;
6869 done:
6870         end_synchronized_op(sc, LOCK_HELD);
6871         return (0);
6872 }
6873
6874 static int
6875 set_filter(struct adapter *sc, struct t4_filter *t)
6876 {
6877         unsigned int nfilters, nports;
6878         struct filter_entry *f;
6879         int i, rc;
6880
6881         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
6882         if (rc)
6883                 return (rc);
6884
6885         nfilters = sc->tids.nftids;
6886         nports = sc->params.nports;
6887
6888         if (nfilters == 0) {
6889                 rc = ENOTSUP;
6890                 goto done;
6891         }
6892
6893         if (!(sc->flags & FULL_INIT_DONE)) {
6894                 rc = EAGAIN;
6895                 goto done;
6896         }
6897
6898         if (t->idx >= nfilters) {
6899                 rc = EINVAL;
6900                 goto done;
6901         }
6902
6903         /* Validate against the global filter mode */
6904         if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
6905             sc->params.tp.vlan_pri_map) {
6906                 rc = E2BIG;
6907                 goto done;
6908         }
6909
6910         if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
6911                 rc = EINVAL;
6912                 goto done;
6913         }
6914
6915         if (t->fs.val.iport >= nports) {
6916                 rc = EINVAL;
6917                 goto done;
6918         }
6919
6920         /* Can't specify an iq if not steering to it */
6921         if (!t->fs.dirsteer && t->fs.iq) {
6922                 rc = EINVAL;
6923                 goto done;
6924         }
6925
6926         /* IPv6 filter idx must be 4 aligned */
6927         if (t->fs.type == 1 &&
6928             ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
6929                 rc = EINVAL;
6930                 goto done;
6931         }
6932
6933         if (sc->tids.ftid_tab == NULL) {
6934                 KASSERT(sc->tids.ftids_in_use == 0,
6935                     ("%s: no memory allocated but filters_in_use > 0",
6936                     __func__));
6937
6938                 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
6939                     nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
6940                 if (sc->tids.ftid_tab == NULL) {
6941                         rc = ENOMEM;
6942                         goto done;
6943                 }
6944                 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
6945         }
6946
6947         for (i = 0; i < 4; i++) {
6948                 f = &sc->tids.ftid_tab[t->idx + i];
6949
6950                 if (f->pending || f->valid) {
6951                         rc = EBUSY;
6952                         goto done;
6953                 }
6954                 if (f->locked) {
6955                         rc = EPERM;
6956                         goto done;
6957                 }
6958
6959                 if (t->fs.type == 0)
6960                         break;
6961         }
6962
6963         f = &sc->tids.ftid_tab[t->idx];
6964         f->fs = t->fs;
6965
6966         rc = set_filter_wr(sc, t->idx);
6967 done:
6968         end_synchronized_op(sc, 0);
6969
6970         if (rc == 0) {
6971                 mtx_lock(&sc->tids.ftid_lock);
6972                 for (;;) {
6973                         if (f->pending == 0) {
6974                                 rc = f->valid ? 0 : EIO;
6975                                 break;
6976                         }
6977
6978                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
6979                             PCATCH, "t4setfw", 0)) {
6980                                 rc = EINPROGRESS;
6981                                 break;
6982                         }
6983                 }
6984                 mtx_unlock(&sc->tids.ftid_lock);
6985         }
6986         return (rc);
6987 }
6988
6989 static int
6990 del_filter(struct adapter *sc, struct t4_filter *t)
6991 {
6992         unsigned int nfilters;
6993         struct filter_entry *f;
6994         int rc;
6995
6996         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
6997         if (rc)
6998                 return (rc);
6999
7000         nfilters = sc->tids.nftids;
7001
7002         if (nfilters == 0) {
7003                 rc = ENOTSUP;
7004                 goto done;
7005         }
7006
7007         if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
7008             t->idx >= nfilters) {
7009                 rc = EINVAL;
7010                 goto done;
7011         }
7012
7013         if (!(sc->flags & FULL_INIT_DONE)) {
7014                 rc = EAGAIN;
7015                 goto done;
7016         }
7017
7018         f = &sc->tids.ftid_tab[t->idx];
7019
7020         if (f->pending) {
7021                 rc = EBUSY;
7022                 goto done;
7023         }
7024         if (f->locked) {
7025                 rc = EPERM;
7026                 goto done;
7027         }
7028
7029         if (f->valid) {
7030                 t->fs = f->fs;  /* extra info for the caller */
7031                 rc = del_filter_wr(sc, t->idx);
7032         }
7033
7034 done:
7035         end_synchronized_op(sc, 0);
7036
7037         if (rc == 0) {
7038                 mtx_lock(&sc->tids.ftid_lock);
7039                 for (;;) {
7040                         if (f->pending == 0) {
7041                                 rc = f->valid ? EIO : 0;
7042                                 break;
7043                         }
7044
7045                         if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7046                             PCATCH, "t4delfw", 0)) {
7047                                 rc = EINPROGRESS;
7048                                 break;
7049                         }
7050                 }
7051                 mtx_unlock(&sc->tids.ftid_lock);
7052         }
7053
7054         return (rc);
7055 }
7056
7057 static void
7058 clear_filter(struct filter_entry *f)
7059 {
7060         if (f->l2t)
7061                 t4_l2t_release(f->l2t);
7062
7063         bzero(f, sizeof (*f));
7064 }
7065
7066 static int
7067 set_filter_wr(struct adapter *sc, int fidx)
7068 {
7069         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7070         struct wrqe *wr;
7071         struct fw_filter_wr *fwr;
7072         unsigned int ftid;
7073
7074         ASSERT_SYNCHRONIZED_OP(sc);
7075
7076         if (f->fs.newdmac || f->fs.newvlan) {
7077                 /* This filter needs an L2T entry; allocate one. */
7078                 f->l2t = t4_l2t_alloc_switching(sc->l2t);
7079                 if (f->l2t == NULL)
7080                         return (EAGAIN);
7081                 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
7082                     f->fs.dmac)) {
7083                         t4_l2t_release(f->l2t);
7084                         f->l2t = NULL;
7085                         return (ENOMEM);
7086                 }
7087         }
7088
7089         ftid = sc->tids.ftid_base + fidx;
7090
7091         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7092         if (wr == NULL)
7093                 return (ENOMEM);
7094
7095         fwr = wrtod(wr);
7096         bzero(fwr, sizeof (*fwr));
7097
7098         fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
7099         fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
7100         fwr->tid_to_iq =
7101             htobe32(V_FW_FILTER_WR_TID(ftid) |
7102                 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
7103                 V_FW_FILTER_WR_NOREPLY(0) |
7104                 V_FW_FILTER_WR_IQ(f->fs.iq));
7105         fwr->del_filter_to_l2tix =
7106             htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
7107                 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
7108                 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
7109                 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
7110                 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
7111                 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
7112                 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
7113                 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
7114                 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
7115                     f->fs.newvlan == VLAN_REWRITE) |
7116                 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
7117                     f->fs.newvlan == VLAN_REWRITE) |
7118                 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
7119                 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
7120                 V_FW_FILTER_WR_PRIO(f->fs.prio) |
7121                 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
7122         fwr->ethtype = htobe16(f->fs.val.ethtype);
7123         fwr->ethtypem = htobe16(f->fs.mask.ethtype);
7124         fwr->frag_to_ovlan_vldm =
7125             (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
7126                 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
7127                 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
7128                 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
7129                 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
7130                 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
7131         fwr->smac_sel = 0;
7132         fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
7133             V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
7134         fwr->maci_to_matchtypem =
7135             htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
7136                 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
7137                 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
7138                 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
7139                 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
7140                 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
7141                 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
7142                 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
7143         fwr->ptcl = f->fs.val.proto;
7144         fwr->ptclm = f->fs.mask.proto;
7145         fwr->ttyp = f->fs.val.tos;
7146         fwr->ttypm = f->fs.mask.tos;
7147         fwr->ivlan = htobe16(f->fs.val.vlan);
7148         fwr->ivlanm = htobe16(f->fs.mask.vlan);
7149         fwr->ovlan = htobe16(f->fs.val.vnic);
7150         fwr->ovlanm = htobe16(f->fs.mask.vnic);
7151         bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
7152         bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
7153         bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
7154         bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
7155         fwr->lp = htobe16(f->fs.val.dport);
7156         fwr->lpm = htobe16(f->fs.mask.dport);
7157         fwr->fp = htobe16(f->fs.val.sport);
7158         fwr->fpm = htobe16(f->fs.mask.sport);
7159         if (f->fs.newsmac)
7160                 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
7161
7162         f->pending = 1;
7163         sc->tids.ftids_in_use++;
7164
7165         t4_wrq_tx(sc, wr);
7166         return (0);
7167 }
7168
7169 static int
7170 del_filter_wr(struct adapter *sc, int fidx)
7171 {
7172         struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7173         struct wrqe *wr;
7174         struct fw_filter_wr *fwr;
7175         unsigned int ftid;
7176
7177         ftid = sc->tids.ftid_base + fidx;
7178
7179         wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
7180         if (wr == NULL)
7181                 return (ENOMEM);
7182         fwr = wrtod(wr);
7183         bzero(fwr, sizeof (*fwr));
7184
7185         t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
7186
7187         f->pending = 1;
7188         t4_wrq_tx(sc, wr);
7189         return (0);
7190 }
7191
7192 int
7193 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
7194 {
7195         struct adapter *sc = iq->adapter;
7196         const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
7197         unsigned int idx = GET_TID(rpl);
7198
7199         KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
7200             rss->opcode));
7201
7202         if (idx >= sc->tids.ftid_base &&
7203             (idx -= sc->tids.ftid_base) < sc->tids.nftids) {
7204                 unsigned int rc = G_COOKIE(rpl->cookie);
7205                 struct filter_entry *f = &sc->tids.ftid_tab[idx];
7206
7207                 mtx_lock(&sc->tids.ftid_lock);
7208                 if (rc == FW_FILTER_WR_FLT_ADDED) {
7209                         KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
7210                             __func__, idx));
7211                         f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
7212                         f->pending = 0;  /* asynchronous setup completed */
7213                         f->valid = 1;
7214                 } else {
7215                         if (rc != FW_FILTER_WR_FLT_DELETED) {
7216                                 /* Add or delete failed, display an error */
7217                                 log(LOG_ERR,
7218                                     "filter %u setup failed with error %u\n",
7219                                     idx, rc);
7220                         }
7221
7222                         clear_filter(f);
7223                         sc->tids.ftids_in_use--;
7224                 }
7225                 wakeup(&sc->tids.ftid_tab);
7226                 mtx_unlock(&sc->tids.ftid_lock);
7227         }
7228
7229         return (0);
7230 }
7231
7232 static int
7233 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
7234 {
7235         int rc;
7236
7237         if (cntxt->cid > M_CTXTQID)
7238                 return (EINVAL);
7239
7240         if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
7241             cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
7242                 return (EINVAL);
7243
7244         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
7245         if (rc)
7246                 return (rc);
7247
7248         if (sc->flags & FW_OK) {
7249                 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
7250                     &cntxt->data[0]);
7251                 if (rc == 0)
7252                         goto done;
7253         }
7254
7255         /*
7256          * Read via firmware failed or wasn't even attempted.  Read directly via
7257          * the backdoor.
7258          */
7259         rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
7260 done:
7261         end_synchronized_op(sc, 0);
7262         return (rc);
7263 }
7264
7265 static int
7266 load_fw(struct adapter *sc, struct t4_data *fw)
7267 {
7268         int rc;
7269         uint8_t *fw_data;
7270
7271         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
7272         if (rc)
7273                 return (rc);
7274
7275         if (sc->flags & FULL_INIT_DONE) {
7276                 rc = EBUSY;
7277                 goto done;
7278         }
7279
7280         fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
7281         if (fw_data == NULL) {
7282                 rc = ENOMEM;
7283                 goto done;
7284         }
7285
7286         rc = copyin(fw->data, fw_data, fw->len);
7287         if (rc == 0)
7288                 rc = -t4_load_fw(sc, fw_data, fw->len);
7289
7290         free(fw_data, M_CXGBE);
7291 done:
7292         end_synchronized_op(sc, 0);
7293         return (rc);
7294 }
7295
7296 static int
7297 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
7298 {
7299         uint32_t addr, off, remaining, i, n;
7300         uint32_t *buf, *b;
7301         uint32_t mw_base, mw_aperture;
7302         int rc;
7303         uint8_t *dst;
7304
7305         rc = validate_mem_range(sc, mr->addr, mr->len);
7306         if (rc != 0)
7307                 return (rc);
7308
7309         memwin_info(sc, win, &mw_base, &mw_aperture);
7310         buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
7311         addr = mr->addr;
7312         remaining = mr->len;
7313         dst = (void *)mr->data;
7314
7315         while (remaining) {
7316                 off = position_memwin(sc, win, addr);
7317
7318                 /* number of bytes that we'll copy in the inner loop */
7319                 n = min(remaining, mw_aperture - off);
7320                 for (i = 0; i < n; i += 4)
7321                         *b++ = t4_read_reg(sc, mw_base + off + i);
7322
7323                 rc = copyout(buf, dst, n);
7324                 if (rc != 0)
7325                         break;
7326
7327                 b = buf;
7328                 dst += n;
7329                 remaining -= n;
7330                 addr += n;
7331         }
7332
7333         free(buf, M_CXGBE);
7334         return (rc);
7335 }
7336
7337 static int
7338 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
7339 {
7340         int rc;
7341
7342         if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
7343                 return (EINVAL);
7344
7345         if (i2cd->len > 1) {
7346                 /* XXX: need fw support for longer reads in one go */
7347                 return (ENOTSUP);
7348         }
7349
7350         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
7351         if (rc)
7352                 return (rc);
7353         rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
7354             i2cd->offset, &i2cd->data[0]);
7355         end_synchronized_op(sc, 0);
7356
7357         return (rc);
7358 }
7359
7360 static int
7361 in_range(int val, int lo, int hi)
7362 {
7363
7364         return (val < 0 || (val <= hi && val >= lo));
7365 }
7366
7367 static int
7368 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
7369 {
7370         int fw_subcmd, fw_type, rc;
7371
7372         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
7373         if (rc)
7374                 return (rc);
7375
7376         if (!(sc->flags & FULL_INIT_DONE)) {
7377                 rc = EAGAIN;
7378                 goto done;
7379         }
7380
7381         /*
7382          * Translate the cxgbetool parameters into T4 firmware parameters.  (The
7383          * sub-command and type are in common locations.)
7384          */
7385         if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
7386                 fw_subcmd = FW_SCHED_SC_CONFIG;
7387         else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
7388                 fw_subcmd = FW_SCHED_SC_PARAMS;
7389         else {
7390                 rc = EINVAL;
7391                 goto done;
7392         }
7393         if (p->type == SCHED_CLASS_TYPE_PACKET)
7394                 fw_type = FW_SCHED_TYPE_PKTSCHED;
7395         else {
7396                 rc = EINVAL;
7397                 goto done;
7398         }
7399
7400         if (fw_subcmd == FW_SCHED_SC_CONFIG) {
7401                 /* Vet our parameters ..*/
7402                 if (p->u.config.minmax < 0) {
7403                         rc = EINVAL;
7404                         goto done;
7405                 }
7406
7407                 /* And pass the request to the firmware ...*/
7408                 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax);
7409                 goto done;
7410         }
7411
7412         if (fw_subcmd == FW_SCHED_SC_PARAMS) {
7413                 int fw_level;
7414                 int fw_mode;
7415                 int fw_rateunit;
7416                 int fw_ratemode;
7417
7418                 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
7419                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
7420                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
7421                         fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
7422                 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
7423                         fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
7424                 else {
7425                         rc = EINVAL;
7426                         goto done;
7427                 }
7428
7429                 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
7430                         fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
7431                 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
7432                         fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
7433                 else {
7434                         rc = EINVAL;
7435                         goto done;
7436                 }
7437
7438                 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
7439                         fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
7440                 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
7441                         fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
7442                 else {
7443                         rc = EINVAL;
7444                         goto done;
7445                 }
7446
7447                 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
7448                         fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
7449                 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
7450                         fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
7451                 else {
7452                         rc = EINVAL;
7453                         goto done;
7454                 }
7455
7456                 /* Vet our parameters ... */
7457                 if (!in_range(p->u.params.channel, 0, 3) ||
7458                     !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
7459                     !in_range(p->u.params.minrate, 0, 10000000) ||
7460                     !in_range(p->u.params.maxrate, 0, 10000000) ||
7461                     !in_range(p->u.params.weight, 0, 100)) {
7462                         rc = ERANGE;
7463                         goto done;
7464                 }
7465
7466                 /*
7467                  * Translate any unset parameters into the firmware's
7468                  * nomenclature and/or fail the call if the parameters
7469                  * are required ...
7470                  */
7471                 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
7472                     p->u.params.channel < 0 || p->u.params.cl < 0) {
7473                         rc = EINVAL;
7474                         goto done;
7475                 }
7476                 if (p->u.params.minrate < 0)
7477                         p->u.params.minrate = 0;
7478                 if (p->u.params.maxrate < 0) {
7479                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7480                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7481                                 rc = EINVAL;
7482                                 goto done;
7483                         } else
7484                                 p->u.params.maxrate = 0;
7485                 }
7486                 if (p->u.params.weight < 0) {
7487                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
7488                                 rc = EINVAL;
7489                                 goto done;
7490                         } else
7491                                 p->u.params.weight = 0;
7492                 }
7493                 if (p->u.params.pktsize < 0) {
7494                         if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7495                             p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7496                                 rc = EINVAL;
7497                                 goto done;
7498                         } else
7499                                 p->u.params.pktsize = 0;
7500                 }
7501
7502                 /* See what the firmware thinks of the request ... */
7503                 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
7504                     fw_rateunit, fw_ratemode, p->u.params.channel,
7505                     p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
7506                     p->u.params.weight, p->u.params.pktsize);
7507                 goto done;
7508         }
7509
7510         rc = EINVAL;
7511 done:
7512         end_synchronized_op(sc, 0);
7513         return (rc);
7514 }
7515
7516 static int
7517 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
7518 {
7519         struct port_info *pi = NULL;
7520         struct sge_txq *txq;
7521         uint32_t fw_mnem, fw_queue, fw_class;
7522         int i, rc;
7523
7524         rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
7525         if (rc)
7526                 return (rc);
7527
7528         if (!(sc->flags & FULL_INIT_DONE)) {
7529                 rc = EAGAIN;
7530                 goto done;
7531         }
7532
7533         if (p->port >= sc->params.nports) {
7534                 rc = EINVAL;
7535                 goto done;
7536         }
7537
7538         pi = sc->port[p->port];
7539         if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
7540                 rc = EINVAL;
7541                 goto done;
7542         }
7543
7544         /*
7545          * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
7546          * Scheduling Class in this case).
7547          */
7548         fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
7549             V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
7550         fw_class = p->cl < 0 ? 0xffffffff : p->cl;
7551
7552         /*
7553          * If op.queue is non-negative, then we're only changing the scheduling
7554          * on a single specified TX queue.
7555          */
7556         if (p->queue >= 0) {
7557                 txq = &sc->sge.txq[pi->first_txq + p->queue];
7558                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7559                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7560                     &fw_class);
7561                 goto done;
7562         }
7563
7564         /*
7565          * Change the scheduling on all the TX queues for the
7566          * interface.
7567          */
7568         for_each_txq(pi, i, txq) {
7569                 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7570                 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7571                     &fw_class);
7572                 if (rc)
7573                         goto done;
7574         }
7575
7576         rc = 0;
7577 done:
7578         end_synchronized_op(sc, 0);
7579         return (rc);
7580 }
7581
7582 int
7583 t4_os_find_pci_capability(struct adapter *sc, int cap)
7584 {
7585         int i;
7586
7587         return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
7588 }
7589
7590 int
7591 t4_os_pci_save_state(struct adapter *sc)
7592 {
7593         device_t dev;
7594         struct pci_devinfo *dinfo;
7595
7596         dev = sc->dev;
7597         dinfo = device_get_ivars(dev);
7598
7599         pci_cfg_save(dev, dinfo, 0);
7600         return (0);
7601 }
7602
7603 int
7604 t4_os_pci_restore_state(struct adapter *sc)
7605 {
7606         device_t dev;
7607         struct pci_devinfo *dinfo;
7608
7609         dev = sc->dev;
7610         dinfo = device_get_ivars(dev);
7611
7612         pci_cfg_restore(dev, dinfo);
7613         return (0);
7614 }
7615
7616 void
7617 t4_os_portmod_changed(const struct adapter *sc, int idx)
7618 {
7619         struct port_info *pi = sc->port[idx];
7620         static const char *mod_str[] = {
7621                 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
7622         };
7623
7624         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
7625                 if_printf(pi->ifp, "transceiver unplugged.\n");
7626         else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
7627                 if_printf(pi->ifp, "unknown transceiver inserted.\n");
7628         else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
7629                 if_printf(pi->ifp, "unsupported transceiver inserted.\n");
7630         else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
7631                 if_printf(pi->ifp, "%s transceiver inserted.\n",
7632                     mod_str[pi->mod_type]);
7633         } else {
7634                 if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
7635                     pi->mod_type);
7636         }
7637 }
7638
7639 void
7640 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
7641 {
7642         struct port_info *pi = sc->port[idx];
7643         struct ifnet *ifp = pi->ifp;
7644
7645         if (link_stat) {
7646                 pi->linkdnrc = -1;
7647                 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
7648                 if_link_state_change(ifp, LINK_STATE_UP);
7649         } else {
7650                 if (reason >= 0)
7651                         pi->linkdnrc = reason;
7652                 if_link_state_change(ifp, LINK_STATE_DOWN);
7653         }
7654 }
7655
7656 void
7657 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
7658 {
7659         struct adapter *sc;
7660
7661         sx_slock(&t4_list_lock);
7662         SLIST_FOREACH(sc, &t4_list, link) {
7663                 /*
7664                  * func should not make any assumptions about what state sc is
7665                  * in - the only guarantee is that sc->sc_lock is a valid lock.
7666                  */
7667                 func(sc, arg);
7668         }
7669         sx_sunlock(&t4_list_lock);
7670 }
7671
7672 static int
7673 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
7674 {
7675        return (0);
7676 }
7677
7678 static int
7679 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
7680 {
7681        return (0);
7682 }
7683
7684 static int
7685 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
7686     struct thread *td)
7687 {
7688         int rc;
7689         struct adapter *sc = dev->si_drv1;
7690
7691         rc = priv_check(td, PRIV_DRIVER);
7692         if (rc != 0)
7693                 return (rc);
7694
7695         switch (cmd) {
7696         case CHELSIO_T4_GETREG: {
7697                 struct t4_reg *edata = (struct t4_reg *)data;
7698
7699                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
7700                         return (EFAULT);
7701
7702                 if (edata->size == 4)
7703                         edata->val = t4_read_reg(sc, edata->addr);
7704                 else if (edata->size == 8)
7705                         edata->val = t4_read_reg64(sc, edata->addr);
7706                 else
7707                         return (EINVAL);
7708
7709                 break;
7710         }
7711         case CHELSIO_T4_SETREG: {
7712                 struct t4_reg *edata = (struct t4_reg *)data;
7713
7714                 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
7715                         return (EFAULT);
7716
7717                 if (edata->size == 4) {
7718                         if (edata->val & 0xffffffff00000000)
7719                                 return (EINVAL);
7720                         t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
7721                 } else if (edata->size == 8)
7722                         t4_write_reg64(sc, edata->addr, edata->val);
7723                 else
7724                         return (EINVAL);
7725                 break;
7726         }
7727         case CHELSIO_T4_REGDUMP: {
7728                 struct t4_regdump *regs = (struct t4_regdump *)data;
7729                 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
7730                 uint8_t *buf;
7731
7732                 if (regs->len < reglen) {
7733                         regs->len = reglen; /* hint to the caller */
7734                         return (ENOBUFS);
7735                 }
7736
7737                 regs->len = reglen;
7738                 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
7739                 t4_get_regs(sc, regs, buf);
7740                 rc = copyout(buf, regs->data, reglen);
7741                 free(buf, M_CXGBE);
7742                 break;
7743         }
7744         case CHELSIO_T4_GET_FILTER_MODE:
7745                 rc = get_filter_mode(sc, (uint32_t *)data);
7746                 break;
7747         case CHELSIO_T4_SET_FILTER_MODE:
7748                 rc = set_filter_mode(sc, *(uint32_t *)data);
7749                 break;
7750         case CHELSIO_T4_GET_FILTER:
7751                 rc = get_filter(sc, (struct t4_filter *)data);
7752                 break;
7753         case CHELSIO_T4_SET_FILTER:
7754                 rc = set_filter(sc, (struct t4_filter *)data);
7755                 break;
7756         case CHELSIO_T4_DEL_FILTER:
7757                 rc = del_filter(sc, (struct t4_filter *)data);
7758                 break;
7759         case CHELSIO_T4_GET_SGE_CONTEXT:
7760                 rc = get_sge_context(sc, (struct t4_sge_context *)data);
7761                 break;
7762         case CHELSIO_T4_LOAD_FW:
7763                 rc = load_fw(sc, (struct t4_data *)data);
7764                 break;
7765         case CHELSIO_T4_GET_MEM:
7766                 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
7767                 break;
7768         case CHELSIO_T4_GET_I2C:
7769                 rc = read_i2c(sc, (struct t4_i2c_data *)data);
7770                 break;
7771         case CHELSIO_T4_CLEAR_STATS: {
7772                 int i;
7773                 u_int port_id = *(uint32_t *)data;
7774                 struct port_info *pi;
7775
7776                 if (port_id >= sc->params.nports)
7777                         return (EINVAL);
7778                 pi = sc->port[port_id];
7779
7780                 /* MAC stats */
7781                 t4_clr_port_stats(sc, pi->tx_chan);
7782
7783                 if (pi->flags & PORT_INIT_DONE) {
7784                         struct sge_rxq *rxq;
7785                         struct sge_txq *txq;
7786                         struct sge_wrq *wrq;
7787
7788                         for_each_rxq(pi, i, rxq) {
7789 #if defined(INET) || defined(INET6)
7790                                 rxq->lro.lro_queued = 0;
7791                                 rxq->lro.lro_flushed = 0;
7792 #endif
7793                                 rxq->rxcsum = 0;
7794                                 rxq->vlan_extraction = 0;
7795                         }
7796
7797                         for_each_txq(pi, i, txq) {
7798                                 txq->txcsum = 0;
7799                                 txq->tso_wrs = 0;
7800                                 txq->vlan_insertion = 0;
7801                                 txq->imm_wrs = 0;
7802                                 txq->sgl_wrs = 0;
7803                                 txq->txpkt_wrs = 0;
7804                                 txq->txpkts_wrs = 0;
7805                                 txq->txpkts_pkts = 0;
7806                                 txq->br->br_drops = 0;
7807                                 txq->no_dmamap = 0;
7808                                 txq->no_desc = 0;
7809                         }
7810
7811 #ifdef TCP_OFFLOAD
7812                         /* nothing to clear for each ofld_rxq */
7813
7814                         for_each_ofld_txq(pi, i, wrq) {
7815                                 wrq->tx_wrs = 0;
7816                                 wrq->no_desc = 0;
7817                         }
7818 #endif
7819                         wrq = &sc->sge.ctrlq[pi->port_id];
7820                         wrq->tx_wrs = 0;
7821                         wrq->no_desc = 0;
7822                 }
7823                 break;
7824         }
7825         case CHELSIO_T4_SCHED_CLASS:
7826                 rc = set_sched_class(sc, (struct t4_sched_params *)data);
7827                 break;
7828         case CHELSIO_T4_SCHED_QUEUE:
7829                 rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
7830                 break;
7831         case CHELSIO_T4_GET_TRACER:
7832                 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
7833                 break;
7834         case CHELSIO_T4_SET_TRACER:
7835                 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
7836                 break;
7837         default:
7838                 rc = EINVAL;
7839         }
7840
7841         return (rc);
7842 }
7843
7844 #ifdef TCP_OFFLOAD
7845 static int
7846 toe_capability(struct port_info *pi, int enable)
7847 {
7848         int rc;
7849         struct adapter *sc = pi->adapter;
7850
7851         ASSERT_SYNCHRONIZED_OP(sc);
7852
7853         if (!is_offload(sc))
7854                 return (ENODEV);
7855
7856         if (enable) {
7857                 if (!(sc->flags & FULL_INIT_DONE)) {
7858                         rc = cxgbe_init_synchronized(pi);
7859                         if (rc)
7860                                 return (rc);
7861                 }
7862
7863                 if (isset(&sc->offload_map, pi->port_id))
7864                         return (0);
7865
7866                 if (!(sc->flags & TOM_INIT_DONE)) {
7867                         rc = t4_activate_uld(sc, ULD_TOM);
7868                         if (rc == EAGAIN) {
7869                                 log(LOG_WARNING,
7870                                     "You must kldload t4_tom.ko before trying "
7871                                     "to enable TOE on a cxgbe interface.\n");
7872                         }
7873                         if (rc != 0)
7874                                 return (rc);
7875                         KASSERT(sc->tom_softc != NULL,
7876                             ("%s: TOM activated but softc NULL", __func__));
7877                         KASSERT(sc->flags & TOM_INIT_DONE,
7878                             ("%s: TOM activated but flag not set", __func__));
7879                 }
7880
7881                 setbit(&sc->offload_map, pi->port_id);
7882         } else {
7883                 if (!isset(&sc->offload_map, pi->port_id))
7884                         return (0);
7885
7886                 KASSERT(sc->flags & TOM_INIT_DONE,
7887                     ("%s: TOM never initialized?", __func__));
7888                 clrbit(&sc->offload_map, pi->port_id);
7889         }
7890
7891         return (0);
7892 }
7893
7894 /*
7895  * Add an upper layer driver to the global list.
7896  */
7897 int
7898 t4_register_uld(struct uld_info *ui)
7899 {
7900         int rc = 0;
7901         struct uld_info *u;
7902
7903         sx_xlock(&t4_uld_list_lock);
7904         SLIST_FOREACH(u, &t4_uld_list, link) {
7905             if (u->uld_id == ui->uld_id) {
7906                     rc = EEXIST;
7907                     goto done;
7908             }
7909         }
7910
7911         SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
7912         ui->refcount = 0;
7913 done:
7914         sx_xunlock(&t4_uld_list_lock);
7915         return (rc);
7916 }
7917
7918 int
7919 t4_unregister_uld(struct uld_info *ui)
7920 {
7921         int rc = EINVAL;
7922         struct uld_info *u;
7923
7924         sx_xlock(&t4_uld_list_lock);
7925
7926         SLIST_FOREACH(u, &t4_uld_list, link) {
7927             if (u == ui) {
7928                     if (ui->refcount > 0) {
7929                             rc = EBUSY;
7930                             goto done;
7931                     }
7932
7933                     SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
7934                     rc = 0;
7935                     goto done;
7936             }
7937         }
7938 done:
7939         sx_xunlock(&t4_uld_list_lock);
7940         return (rc);
7941 }
7942
7943 int
7944 t4_activate_uld(struct adapter *sc, int id)
7945 {
7946         int rc = EAGAIN;
7947         struct uld_info *ui;
7948
7949         ASSERT_SYNCHRONIZED_OP(sc);
7950
7951         sx_slock(&t4_uld_list_lock);
7952
7953         SLIST_FOREACH(ui, &t4_uld_list, link) {
7954                 if (ui->uld_id == id) {
7955                         rc = ui->activate(sc);
7956                         if (rc == 0)
7957                                 ui->refcount++;
7958                         goto done;
7959                 }
7960         }
7961 done:
7962         sx_sunlock(&t4_uld_list_lock);
7963
7964         return (rc);
7965 }
7966
7967 int
7968 t4_deactivate_uld(struct adapter *sc, int id)
7969 {
7970         int rc = EINVAL;
7971         struct uld_info *ui;
7972
7973         ASSERT_SYNCHRONIZED_OP(sc);
7974
7975         sx_slock(&t4_uld_list_lock);
7976
7977         SLIST_FOREACH(ui, &t4_uld_list, link) {
7978                 if (ui->uld_id == id) {
7979                         rc = ui->deactivate(sc);
7980                         if (rc == 0)
7981                                 ui->refcount--;
7982                         goto done;
7983                 }
7984         }
7985 done:
7986         sx_sunlock(&t4_uld_list_lock);
7987
7988         return (rc);
7989 }
7990 #endif
7991
7992 /*
7993  * Come up with reasonable defaults for some of the tunables, provided they're
7994  * not set by the user (in which case we'll use the values as is).
7995  */
7996 static void
7997 tweak_tunables(void)
7998 {
7999         int nc = mp_ncpus;      /* our snapshot of the number of CPUs */
8000
8001         if (t4_ntxq10g < 1)
8002                 t4_ntxq10g = min(nc, NTXQ_10G);
8003
8004         if (t4_ntxq1g < 1)
8005                 t4_ntxq1g = min(nc, NTXQ_1G);
8006
8007         if (t4_nrxq10g < 1)
8008                 t4_nrxq10g = min(nc, NRXQ_10G);
8009
8010         if (t4_nrxq1g < 1)
8011                 t4_nrxq1g = min(nc, NRXQ_1G);
8012
8013 #ifdef TCP_OFFLOAD
8014         if (t4_nofldtxq10g < 1)
8015                 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
8016
8017         if (t4_nofldtxq1g < 1)
8018                 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
8019
8020         if (t4_nofldrxq10g < 1)
8021                 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
8022
8023         if (t4_nofldrxq1g < 1)
8024                 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
8025
8026         if (t4_toecaps_allowed == -1)
8027                 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
8028 #else
8029         if (t4_toecaps_allowed == -1)
8030                 t4_toecaps_allowed = 0;
8031 #endif
8032
8033         if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
8034                 t4_tmr_idx_10g = TMR_IDX_10G;
8035
8036         if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
8037                 t4_pktc_idx_10g = PKTC_IDX_10G;
8038
8039         if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
8040                 t4_tmr_idx_1g = TMR_IDX_1G;
8041
8042         if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
8043                 t4_pktc_idx_1g = PKTC_IDX_1G;
8044
8045         if (t4_qsize_txq < 128)
8046                 t4_qsize_txq = 128;
8047
8048         if (t4_qsize_rxq < 128)
8049                 t4_qsize_rxq = 128;
8050         while (t4_qsize_rxq & 7)
8051                 t4_qsize_rxq++;
8052
8053         t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
8054 }
8055
8056 static int
8057 mod_event(module_t mod, int cmd, void *arg)
8058 {
8059         int rc = 0;
8060         static int loaded = 0;
8061
8062         switch (cmd) {
8063         case MOD_LOAD:
8064                 if (atomic_fetchadd_int(&loaded, 1))
8065                         break;
8066                 t4_sge_modload();
8067                 sx_init(&t4_list_lock, "T4/T5 adapters");
8068                 SLIST_INIT(&t4_list);
8069 #ifdef TCP_OFFLOAD
8070                 sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
8071                 SLIST_INIT(&t4_uld_list);
8072 #endif
8073                 t4_tracer_modload();
8074                 tweak_tunables();
8075                 break;
8076
8077         case MOD_UNLOAD:
8078                 if (atomic_fetchadd_int(&loaded, -1) > 1)
8079                         break;
8080                 t4_tracer_modunload();
8081 #ifdef TCP_OFFLOAD
8082                 sx_slock(&t4_uld_list_lock);
8083                 if (!SLIST_EMPTY(&t4_uld_list)) {
8084                         rc = EBUSY;
8085                         sx_sunlock(&t4_uld_list_lock);
8086                         break;
8087                 }
8088                 sx_sunlock(&t4_uld_list_lock);
8089                 sx_destroy(&t4_uld_list_lock);
8090 #endif
8091                 sx_slock(&t4_list_lock);
8092                 if (!SLIST_EMPTY(&t4_list)) {
8093                         rc = EBUSY;
8094                         sx_sunlock(&t4_list_lock);
8095                         break;
8096                 }
8097                 sx_sunlock(&t4_list_lock);
8098                 sx_destroy(&t4_list_lock);
8099                 break;
8100         }
8101
8102         return (rc);
8103 }
8104
8105 static devclass_t t4_devclass, t5_devclass;
8106 static devclass_t cxgbe_devclass, cxl_devclass;
8107
8108 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
8109 MODULE_VERSION(t4nex, 1);
8110 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
8111
8112 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
8113 MODULE_VERSION(t5nex, 1);
8114 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
8115
8116 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
8117 MODULE_VERSION(cxgbe, 1);
8118
8119 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
8120 MODULE_VERSION(cxl, 1);