sys/dev/bce/if_bce.c

   1 /*-
   2  * Copyright (c) 2006-2009 Broadcom Corporation
   3  *      David Christensen <davidch@broadcom.com>.  All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  *
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  * 3. Neither the name of Broadcom Corporation nor the name of its contributors
  15  *    may be used to endorse or promote products derived from this software
  16  *    without specific prior written consent.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS'
  19  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
  22  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  23  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  24  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  25  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  26  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  27  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  28  * THE POSSIBILITY OF SUCH DAMAGE.
  29  */
  30
  31 #include <sys/cdefs.h>
  32 __FBSDID("$FreeBSD$");
  33
  34 /*
  35  * The following controllers are supported by this driver:
  36  *   BCM5706C A2, A3
  37  *   BCM5706S A2, A3
  38  *   BCM5708C B1, B2
  39  *   BCM5708S B1, B2
  40  *   BCM5709C A1, C0
  41  *       BCM5716C C0
  42  *
  43  * The following controllers are not supported by this driver:
  44  *   BCM5706C A0, A1 (pre-production)
  45  *   BCM5706S A0, A1 (pre-production)
  46  *   BCM5708C A0, B0 (pre-production)
  47  *   BCM5708S A0, B0 (pre-production)
  48  *   BCM5709C A0  B0, B1, B2 (pre-production)
  49  *   BCM5709S A0, A1, B0, B1, B2, C0 (pre-production)
  50  */
  51
  52 #include "opt_bce.h"
  53
  54 #include <dev/bce/if_bcereg.h>
  55 #include <dev/bce/if_bcefw.h>
  56
  57 /****************************************************************************/
  58 /* BCE Debug Options                                                        */
  59 /****************************************************************************/
  60 #ifdef BCE_DEBUG
  61         u32 bce_debug = BCE_WARN;
  62
  63         /*          0 = Never              */
  64         /*          1 = 1 in 2,147,483,648 */
  65         /*        256 = 1 in     8,388,608 */
  66         /*       2048 = 1 in     1,048,576 */
  67         /*      65536 = 1 in        32,768 */
  68         /*    1048576 = 1 in         2,048 */
  69         /*  268435456 = 1 in             8 */
  70         /*  536870912 = 1 in             4 */
  71         /* 1073741824 = 1 in             2 */
  72
  73         /* Controls how often the l2_fhdr frame error check will fail. */
  74         int l2fhdr_error_sim_control = 0;
  75
  76         /* Controls how often the unexpected attention check will fail. */
  77         int unexpected_attention_sim_control = 0;
  78
  79         /* Controls how often to simulate an mbuf allocation failure. */
  80         int mbuf_alloc_failed_sim_control = 0;
  81
  82         /* Controls how often to simulate a DMA mapping failure. */
  83         int dma_map_addr_failed_sim_control = 0;
  84
  85         /* Controls how often to simulate a bootcode failure. */
  86         int bootcode_running_failure_sim_control = 0;
  87 #endif
  88
  89 /****************************************************************************/
  90 /* BCE Build Time Options                                                   */
  91 /****************************************************************************/
  92 /* #define BCE_NVRAM_WRITE_SUPPORT 1 */
  93
  94
  95 /****************************************************************************/
  96 /* PCI Device ID Table                                                      */
  97 /*                                                                          */
  98 /* Used by bce_probe() to identify the devices supported by this driver.    */
  99 /****************************************************************************/
 100 #define BCE_DEVDESC_MAX         64
 101
 102 static struct bce_type bce_devs[] = {
 103         /* BCM5706C Controllers and OEM boards. */
 104         { BRCM_VENDORID, BRCM_DEVICEID_BCM5706,  HP_VENDORID, 0x3101,
 105                 "HP NC370T Multifunction Gigabit Server Adapter" },
 106         { BRCM_VENDORID, BRCM_DEVICEID_BCM5706,  HP_VENDORID, 0x3106,
 107                 "HP NC370i Multifunction Gigabit Server Adapter" },
 108         { BRCM_VENDORID, BRCM_DEVICEID_BCM5706,  HP_VENDORID, 0x3070,
 109                 "HP NC380T PCIe DP Multifunc Gig Server Adapter" },
 110         { BRCM_VENDORID, BRCM_DEVICEID_BCM5706,  HP_VENDORID, 0x1709,
 111                 "HP NC371i Multifunction Gigabit Server Adapter" },
 112         { BRCM_VENDORID, BRCM_DEVICEID_BCM5706,  PCI_ANY_ID,  PCI_ANY_ID,
 113                 "Broadcom NetXtreme II BCM5706 1000Base-T" },
 114
 115         /* BCM5706S controllers and OEM boards. */
 116         { BRCM_VENDORID, BRCM_DEVICEID_BCM5706S, HP_VENDORID, 0x3102,
 117                 "HP NC370F Multifunction Gigabit Server Adapter" },
 118         { BRCM_VENDORID, BRCM_DEVICEID_BCM5706S, PCI_ANY_ID,  PCI_ANY_ID,
 119                 "Broadcom NetXtreme II BCM5706 1000Base-SX" },
 120
 121         /* BCM5708C controllers and OEM boards. */
 122         { BRCM_VENDORID, BRCM_DEVICEID_BCM5708,  HP_VENDORID, 0x7037,
 123                 "HP NC373T PCIe Multifunction Gig Server Adapter" },
 124         { BRCM_VENDORID, BRCM_DEVICEID_BCM5708,  HP_VENDORID, 0x7038,
 125                 "HP NC373i Multifunction Gigabit Server Adapter" },
 126         { BRCM_VENDORID, BRCM_DEVICEID_BCM5708,  HP_VENDORID, 0x7045,
 127                 "HP NC374m PCIe Multifunction Adapter" },
 128         { BRCM_VENDORID, BRCM_DEVICEID_BCM5708,  PCI_ANY_ID,  PCI_ANY_ID,
 129                 "Broadcom NetXtreme II BCM5708 1000Base-T" },
 130
 131         /* BCM5708S controllers and OEM boards. */
 132         { BRCM_VENDORID, BRCM_DEVICEID_BCM5708S,  HP_VENDORID, 0x1706,
 133                 "HP NC373m Multifunction Gigabit Server Adapter" },
 134         { BRCM_VENDORID, BRCM_DEVICEID_BCM5708S,  HP_VENDORID, 0x703b,
 135                 "HP NC373i Multifunction Gigabit Server Adapter" },
 136         { BRCM_VENDORID, BRCM_DEVICEID_BCM5708S,  HP_VENDORID, 0x703d,
 137                 "HP NC373F PCIe Multifunc Giga Server Adapter" },
 138         { BRCM_VENDORID, BRCM_DEVICEID_BCM5708S,  PCI_ANY_ID,  PCI_ANY_ID,
 139                 "Broadcom NetXtreme II BCM5708 1000Base-SX" },
 140
 141         /* BCM5709C controllers and OEM boards. */
 142         { BRCM_VENDORID, BRCM_DEVICEID_BCM5709,  HP_VENDORID, 0x7055,
 143                 "HP NC382i DP Multifunction Gigabit Server Adapter" },
 144         { BRCM_VENDORID, BRCM_DEVICEID_BCM5709,  HP_VENDORID, 0x7059,
 145                 "HP NC382T PCIe DP Multifunction Gigabit Server Adapter" },
 146         { BRCM_VENDORID, BRCM_DEVICEID_BCM5709,  PCI_ANY_ID,  PCI_ANY_ID,
 147                 "Broadcom NetXtreme II BCM5709 1000Base-T" },
 148
 149         /* BCM5709S controllers and OEM boards. */
 150         { BRCM_VENDORID, BRCM_DEVICEID_BCM5709S,  HP_VENDORID, 0x171d,
 151                 "HP NC382m DP 1GbE Multifunction BL-c Adapter" },
 152         { BRCM_VENDORID, BRCM_DEVICEID_BCM5709S,  HP_VENDORID, 0x7056,
 153                 "HP NC382i DP Multifunction Gigabit Server Adapter" },
 154         { BRCM_VENDORID, BRCM_DEVICEID_BCM5709S,  PCI_ANY_ID,  PCI_ANY_ID,
 155                 "Broadcom NetXtreme II BCM5709 1000Base-SX" },
 156
 157         /* BCM5716 controllers and OEM boards. */
 158         { BRCM_VENDORID, BRCM_DEVICEID_BCM5716,  PCI_ANY_ID,  PCI_ANY_ID,
 159                 "Broadcom NetXtreme II BCM5716 1000Base-T" },
 160
 161         { 0, 0, 0, 0, NULL }
 162 };
 163
 164
 165 /****************************************************************************/
 166 /* Supported Flash NVRAM device data.                                       */
 167 /****************************************************************************/
 168 static struct flash_spec flash_table[] =
 169 {
 170 #define BUFFERED_FLAGS          (BCE_NV_BUFFERED | BCE_NV_TRANSLATE)
 171 #define NONBUFFERED_FLAGS       (BCE_NV_WREN)
 172
 173         /* Slow EEPROM */
 174         {0x00000000, 0x40830380, 0x009f0081, 0xa184a053, 0xaf000400,
 175          BUFFERED_FLAGS, SEEPROM_PAGE_BITS, SEEPROM_PAGE_SIZE,
 176          SEEPROM_BYTE_ADDR_MASK, SEEPROM_TOTAL_SIZE,
 177          "EEPROM - slow"},
 178         /* Expansion entry 0001 */
 179         {0x08000002, 0x4b808201, 0x00050081, 0x03840253, 0xaf020406,
 180          NONBUFFERED_FLAGS, SAIFUN_FLASH_PAGE_BITS, SAIFUN_FLASH_PAGE_SIZE,
 181          SAIFUN_FLASH_BYTE_ADDR_MASK, 0,
 182          "Entry 0001"},
 183         /* Saifun SA25F010 (non-buffered flash) */
 184         /* strap, cfg1, & write1 need updates */
 185         {0x04000001, 0x47808201, 0x00050081, 0x03840253, 0xaf020406,
 186          NONBUFFERED_FLAGS, SAIFUN_FLASH_PAGE_BITS, SAIFUN_FLASH_PAGE_SIZE,
 187          SAIFUN_FLASH_BYTE_ADDR_MASK, SAIFUN_FLASH_BASE_TOTAL_SIZE*2,
 188          "Non-buffered flash (128kB)"},
 189         /* Saifun SA25F020 (non-buffered flash) */
 190         /* strap, cfg1, & write1 need updates */
 191         {0x0c000003, 0x4f808201, 0x00050081, 0x03840253, 0xaf020406,
 192          NONBUFFERED_FLAGS, SAIFUN_FLASH_PAGE_BITS, SAIFUN_FLASH_PAGE_SIZE,
 193          SAIFUN_FLASH_BYTE_ADDR_MASK, SAIFUN_FLASH_BASE_TOTAL_SIZE*4,
 194          "Non-buffered flash (256kB)"},
 195         /* Expansion entry 0100 */
 196         {0x11000000, 0x53808201, 0x00050081, 0x03840253, 0xaf020406,
 197          NONBUFFERED_FLAGS, SAIFUN_FLASH_PAGE_BITS, SAIFUN_FLASH_PAGE_SIZE,
 198          SAIFUN_FLASH_BYTE_ADDR_MASK, 0,
 199          "Entry 0100"},
 200         /* Entry 0101: ST M45PE10 (non-buffered flash, TetonII B0) */
 201         {0x19000002, 0x5b808201, 0x000500db, 0x03840253, 0xaf020406,
 202          NONBUFFERED_FLAGS, ST_MICRO_FLASH_PAGE_BITS, ST_MICRO_FLASH_PAGE_SIZE,
 203          ST_MICRO_FLASH_BYTE_ADDR_MASK, ST_MICRO_FLASH_BASE_TOTAL_SIZE*2,
 204          "Entry 0101: ST M45PE10 (128kB non-bufferred)"},
 205         /* Entry 0110: ST M45PE20 (non-buffered flash)*/
 206         {0x15000001, 0x57808201, 0x000500db, 0x03840253, 0xaf020406,
 207          NONBUFFERED_FLAGS, ST_MICRO_FLASH_PAGE_BITS, ST_MICRO_FLASH_PAGE_SIZE,
 208          ST_MICRO_FLASH_BYTE_ADDR_MASK, ST_MICRO_FLASH_BASE_TOTAL_SIZE*4,
 209          "Entry 0110: ST M45PE20 (256kB non-bufferred)"},
 210         /* Saifun SA25F005 (non-buffered flash) */
 211         /* strap, cfg1, & write1 need updates */
 212         {0x1d000003, 0x5f808201, 0x00050081, 0x03840253, 0xaf020406,
 213          NONBUFFERED_FLAGS, SAIFUN_FLASH_PAGE_BITS, SAIFUN_FLASH_PAGE_SIZE,
 214          SAIFUN_FLASH_BYTE_ADDR_MASK, SAIFUN_FLASH_BASE_TOTAL_SIZE,
 215          "Non-buffered flash (64kB)"},
 216         /* Fast EEPROM */
 217         {0x22000000, 0x62808380, 0x009f0081, 0xa184a053, 0xaf000400,
 218          BUFFERED_FLAGS, SEEPROM_PAGE_BITS, SEEPROM_PAGE_SIZE,
 219          SEEPROM_BYTE_ADDR_MASK, SEEPROM_TOTAL_SIZE,
 220          "EEPROM - fast"},
 221         /* Expansion entry 1001 */
 222         {0x2a000002, 0x6b808201, 0x00050081, 0x03840253, 0xaf020406,
 223          NONBUFFERED_FLAGS, SAIFUN_FLASH_PAGE_BITS, SAIFUN_FLASH_PAGE_SIZE,
 224          SAIFUN_FLASH_BYTE_ADDR_MASK, 0,
 225          "Entry 1001"},
 226         /* Expansion entry 1010 */
 227         {0x26000001, 0x67808201, 0x00050081, 0x03840253, 0xaf020406,
 228          NONBUFFERED_FLAGS, SAIFUN_FLASH_PAGE_BITS, SAIFUN_FLASH_PAGE_SIZE,
 229          SAIFUN_FLASH_BYTE_ADDR_MASK, 0,
 230          "Entry 1010"},
 231         /* ATMEL AT45DB011B (buffered flash) */
 232         {0x2e000003, 0x6e808273, 0x00570081, 0x68848353, 0xaf000400,
 233          BUFFERED_FLAGS, BUFFERED_FLASH_PAGE_BITS, BUFFERED_FLASH_PAGE_SIZE,
 234          BUFFERED_FLASH_BYTE_ADDR_MASK, BUFFERED_FLASH_TOTAL_SIZE,
 235          "Buffered flash (128kB)"},
 236         /* Expansion entry 1100 */
 237         {0x33000000, 0x73808201, 0x00050081, 0x03840253, 0xaf020406,
 238          NONBUFFERED_FLAGS, SAIFUN_FLASH_PAGE_BITS, SAIFUN_FLASH_PAGE_SIZE,
 239          SAIFUN_FLASH_BYTE_ADDR_MASK, 0,
 240          "Entry 1100"},
 241         /* Expansion entry 1101 */
 242         {0x3b000002, 0x7b808201, 0x00050081, 0x03840253, 0xaf020406,
 243          NONBUFFERED_FLAGS, SAIFUN_FLASH_PAGE_BITS, SAIFUN_FLASH_PAGE_SIZE,
 244          SAIFUN_FLASH_BYTE_ADDR_MASK, 0,
 245          "Entry 1101"},
 246         /* Ateml Expansion entry 1110 */
 247         {0x37000001, 0x76808273, 0x00570081, 0x68848353, 0xaf000400,
 248          BUFFERED_FLAGS, BUFFERED_FLASH_PAGE_BITS, BUFFERED_FLASH_PAGE_SIZE,
 249          BUFFERED_FLASH_BYTE_ADDR_MASK, 0,
 250          "Entry 1110 (Atmel)"},
 251         /* ATMEL AT45DB021B (buffered flash) */
 252         {0x3f000003, 0x7e808273, 0x00570081, 0x68848353, 0xaf000400,
 253          BUFFERED_FLAGS, BUFFERED_FLASH_PAGE_BITS, BUFFERED_FLASH_PAGE_SIZE,
 254          BUFFERED_FLASH_BYTE_ADDR_MASK, BUFFERED_FLASH_TOTAL_SIZE*2,
 255          "Buffered flash (256kB)"},
 256 };
 257
 258 /*
 259  * The BCM5709 controllers transparently handle the
 260  * differences between Atmel 264 byte pages and all
 261  * flash devices which use 256 byte pages, so no
 262  * logical-to-physical mapping is required in the
 263  * driver.
 264  */
 265 static struct flash_spec flash_5709 = {
 266         .flags          = BCE_NV_BUFFERED,
 267         .page_bits      = BCM5709_FLASH_PAGE_BITS,
 268         .page_size      = BCM5709_FLASH_PAGE_SIZE,
 269         .addr_mask      = BCM5709_FLASH_BYTE_ADDR_MASK,
 270         .total_size     = BUFFERED_FLASH_TOTAL_SIZE * 2,
 271         .name           = "5709/5716 buffered flash (256kB)",
 272 };
 273
 274
 275 /****************************************************************************/
 276 /* FreeBSD device entry points.                                             */
 277 /****************************************************************************/
 278 static int  bce_probe                           (device_t);
 279 static int  bce_attach                          (device_t);
 280 static int  bce_detach                          (device_t);
 281 static int  bce_shutdown                        (device_t);
 282
 283
 284 /****************************************************************************/
 285 /* BCE Debug Data Structure Dump Routines                                   */
 286 /****************************************************************************/
 287 #ifdef BCE_DEBUG
 288 static u32      bce_reg_rd                              (struct bce_softc *, u32);
 289 static void     bce_reg_wr                              (struct bce_softc *, u32, u32);
 290 static void     bce_reg_wr16                    (struct bce_softc *, u32, u16);
 291 static u32  bce_ctx_rd                          (struct bce_softc *, u32, u32);
 292 static void bce_dump_enet           (struct bce_softc *, struct mbuf *);
 293 static void bce_dump_mbuf                       (struct bce_softc *, struct mbuf *);
 294 static void bce_dump_tx_mbuf_chain      (struct bce_softc *, u16, int);
 295 static void bce_dump_rx_mbuf_chain      (struct bce_softc *, u16, int);
 296 #ifdef ZERO_COPY_SOCKETS
 297 static void bce_dump_pg_mbuf_chain      (struct bce_softc *, u16, int);
 298 #endif
 299 static void bce_dump_txbd                       (struct bce_softc *, int, struct tx_bd *);
 300 static void bce_dump_rxbd                       (struct bce_softc *, int, struct rx_bd *);
 301 #ifdef ZERO_COPY_SOCKETS
 302 static void bce_dump_pgbd                       (struct bce_softc *, int, struct rx_bd *);
 303 #endif
 304 static void bce_dump_l2fhdr                     (struct bce_softc *, int, struct l2_fhdr *);
 305 static void bce_dump_ctx                        (struct bce_softc *, u16);
 306 static void bce_dump_ftqs                       (struct bce_softc *);
 307 static void bce_dump_tx_chain           (struct bce_softc *, u16, int);
 308 static void bce_dump_rx_chain           (struct bce_softc *, u16, int);
 309 #ifdef ZERO_COPY_SOCKETS
 310 static void bce_dump_pg_chain           (struct bce_softc *, u16, int);
 311 #endif
 312 static void bce_dump_status_block       (struct bce_softc *);
 313 static void bce_dump_stats_block        (struct bce_softc *);
 314 static void bce_dump_driver_state       (struct bce_softc *);
 315 static void bce_dump_hw_state           (struct bce_softc *);
 316 static void bce_dump_mq_regs        (struct bce_softc *);
 317 static void bce_dump_bc_state           (struct bce_softc *);
 318 static void bce_dump_txp_state          (struct bce_softc *, int);
 319 static void bce_dump_rxp_state          (struct bce_softc *, int);
 320 static void bce_dump_tpat_state         (struct bce_softc *, int);
 321 static void bce_dump_cp_state           (struct bce_softc *, int);
 322 static void bce_dump_com_state          (struct bce_softc *, int);
 323 static void bce_breakpoint                      (struct bce_softc *);
 324 #endif
 325
 326
 327 /****************************************************************************/
 328 /* BCE Register/Memory Access Routines                                      */
 329 /****************************************************************************/
 330 static u32  bce_reg_rd_ind                      (struct bce_softc *, u32);
 331 static void bce_reg_wr_ind                      (struct bce_softc *, u32, u32);
 332 static void bce_ctx_wr                          (struct bce_softc *, u32, u32, u32);
 333 static int  bce_miibus_read_reg         (device_t, int, int);
 334 static int  bce_miibus_write_reg        (device_t, int, int, int);
 335 static void bce_miibus_statchg          (device_t);
 336
 337
 338 /****************************************************************************/
 339 /* BCE NVRAM Access Routines                                                */
 340 /****************************************************************************/
 341 static int  bce_acquire_nvram_lock      (struct bce_softc *);
 342 static int  bce_release_nvram_lock      (struct bce_softc *);
 343 static void bce_enable_nvram_access     (struct bce_softc *);
 344 static void     bce_disable_nvram_access(struct bce_softc *);
 345 static int  bce_nvram_read_dword        (struct bce_softc *, u32, u8 *, u32);
 346 static int  bce_init_nvram                      (struct bce_softc *);
 347 static int  bce_nvram_read                      (struct bce_softc *, u32, u8 *, int);
 348 static int  bce_nvram_test                      (struct bce_softc *);
 349 #ifdef BCE_NVRAM_WRITE_SUPPORT
 350 static int  bce_enable_nvram_write      (struct bce_softc *);
 351 static void bce_disable_nvram_write     (struct bce_softc *);
 352 static int  bce_nvram_erase_page        (struct bce_softc *, u32);
 353 static int  bce_nvram_write_dword       (struct bce_softc *, u32, u8 *, u32);
 354 static int  bce_nvram_write                     (struct bce_softc *, u32, u8 *, int);
 355 #endif
 356
 357 /****************************************************************************/
 358 /*                                                                          */
 359 /****************************************************************************/
 360 static void bce_get_media                       (struct bce_softc *);
 361 static void bce_dma_map_addr            (void *, bus_dma_segment_t *, int, int);
 362 static int  bce_dma_alloc                       (device_t);
 363 static void bce_dma_free                        (struct bce_softc *);
 364 static void bce_release_resources       (struct bce_softc *);
 365
 366 /****************************************************************************/
 367 /* BCE Firmware Synchronization and Load                                    */
 368 /****************************************************************************/
 369 static int  bce_fw_sync                         (struct bce_softc *, u32);
 370 static void bce_load_rv2p_fw            (struct bce_softc *, u32 *, u32, u32);
 371 static void bce_load_cpu_fw                     (struct bce_softc *, struct cpu_reg *, struct fw_info *);
 372 static void bce_init_rxp_cpu            (struct bce_softc *);
 373 static void bce_init_txp_cpu            (struct bce_softc *);
 374 static void bce_init_tpat_cpu           (struct bce_softc *);
 375 static void bce_init_cp_cpu                     (struct bce_softc *);
 376 static void bce_init_com_cpu            (struct bce_softc *);
 377 static void bce_init_cpus                       (struct bce_softc *);
 378
 379 static void     bce_print_adapter_info  (struct bce_softc *);
 380 static void bce_probe_pci_caps          (device_t, struct bce_softc *);
 381 static void bce_stop                            (struct bce_softc *);
 382 static int  bce_reset                           (struct bce_softc *, u32);
 383 static int  bce_chipinit                        (struct bce_softc *);
 384 static int  bce_blockinit                       (struct bce_softc *);
 385
 386 static int  bce_init_tx_chain           (struct bce_softc *);
 387 static void bce_free_tx_chain           (struct bce_softc *);
 388
 389 static int  bce_get_rx_buf                      (struct bce_softc *, struct mbuf *, u16 *, u16 *, u32 *);
 390 static int  bce_init_rx_chain           (struct bce_softc *);
 391 static void bce_fill_rx_chain           (struct bce_softc *);
 392 static void bce_free_rx_chain           (struct bce_softc *);
 393
 394 #ifdef ZERO_COPY_SOCKETS
 395 static int  bce_get_pg_buf                      (struct bce_softc *, struct mbuf *, u16 *, u16 *);
 396 static int  bce_init_pg_chain           (struct bce_softc *);
 397 static void bce_fill_pg_chain           (struct bce_softc *);
 398 static void bce_free_pg_chain           (struct bce_softc *);
 399 #endif
 400
 401 static int  bce_tx_encap                        (struct bce_softc *, struct mbuf **);
 402 static void bce_start_locked            (struct ifnet *);
 403 static void bce_start                           (struct ifnet *);
 404 static int  bce_ioctl                           (struct ifnet *, u_long, caddr_t);
 405 static void bce_watchdog                        (struct bce_softc *);
 406 static int  bce_ifmedia_upd                     (struct ifnet *);
 407 static void bce_ifmedia_upd_locked      (struct ifnet *);
 408 static void bce_ifmedia_sts                     (struct ifnet *, struct ifmediareq *);
 409 static void bce_init_locked                     (struct bce_softc *);
 410 static void bce_init                            (void *);
 411 static void bce_mgmt_init_locked        (struct bce_softc *sc);
 412
 413 static void bce_init_ctx                        (struct bce_softc *);
 414 static void bce_get_mac_addr            (struct bce_softc *);
 415 static void bce_set_mac_addr            (struct bce_softc *);
 416 static void bce_phy_intr                        (struct bce_softc *);
 417 static inline u16 bce_get_hw_rx_cons(struct bce_softc *);
 418 static void bce_rx_intr                         (struct bce_softc *);
 419 static void bce_tx_intr                         (struct bce_softc *);
 420 static void bce_disable_intr            (struct bce_softc *);
 421 static void bce_enable_intr                     (struct bce_softc *, int);
 422
 423 static void bce_intr                            (void *);
 424 static void bce_set_rx_mode                     (struct bce_softc *);
 425 static void bce_stats_update            (struct bce_softc *);
 426 static void bce_tick                            (void *);
 427 static void bce_pulse                           (void *);
 428 static void bce_add_sysctls                     (struct bce_softc *);
 429
 430
 431 /****************************************************************************/
 432 /* FreeBSD device dispatch table.                                           */
 433 /****************************************************************************/
 434 static device_method_t bce_methods[] = {
 435         /* Device interface (device_if.h) */
 436         DEVMETHOD(device_probe,         bce_probe),
 437         DEVMETHOD(device_attach,        bce_attach),
 438         DEVMETHOD(device_detach,        bce_detach),
 439         DEVMETHOD(device_shutdown,      bce_shutdown),
 440 /* Supported by device interface but not used here. */
 441 /*      DEVMETHOD(device_identify,      bce_identify),      */
 442 /*      DEVMETHOD(device_suspend,       bce_suspend),       */
 443 /*      DEVMETHOD(device_resume,        bce_resume),        */
 444 /*      DEVMETHOD(device_quiesce,       bce_quiesce),       */
 445
 446         /* Bus interface (bus_if.h) */
 447         DEVMETHOD(bus_print_child,      bus_generic_print_child),
 448         DEVMETHOD(bus_driver_added,     bus_generic_driver_added),
 449
 450         /* MII interface (miibus_if.h) */
 451         DEVMETHOD(miibus_readreg,       bce_miibus_read_reg),
 452         DEVMETHOD(miibus_writereg,      bce_miibus_write_reg),
 453         DEVMETHOD(miibus_statchg,       bce_miibus_statchg),
 454 /* Supported by MII interface but not used here.       */
 455 /*      DEVMETHOD(miibus_linkchg,       bce_miibus_linkchg),   */
 456 /*      DEVMETHOD(miibus_mediainit,     bce_miibus_mediainit), */
 457
 458         { 0, 0 }
 459 };
 460
 461 static driver_t bce_driver = {
 462         "bce",
 463         bce_methods,
 464         sizeof(struct bce_softc)
 465 };
 466
 467 static devclass_t bce_devclass;
 468
 469 MODULE_DEPEND(bce, pci, 1, 1, 1);
 470 MODULE_DEPEND(bce, ether, 1, 1, 1);
 471 MODULE_DEPEND(bce, miibus, 1, 1, 1);
 472
 473 DRIVER_MODULE(bce, pci, bce_driver, bce_devclass, 0, 0);
 474 DRIVER_MODULE(miibus, bce, miibus_driver, miibus_devclass, 0, 0);
 475
 476
 477 /****************************************************************************/
 478 /* Tunable device values                                                    */
 479 /****************************************************************************/
 480 SYSCTL_NODE(_hw, OID_AUTO, bce, CTLFLAG_RD, 0, "bce driver parameters");
 481
 482 /* Allowable values are TRUE or FALSE */
 483 static int bce_tso_enable = TRUE;
 484 TUNABLE_INT("hw.bce.tso_enable", &bce_tso_enable);
 485 SYSCTL_UINT(_hw_bce, OID_AUTO, tso_enable, CTLFLAG_RDTUN, &bce_tso_enable, 0,
 486 "TSO Enable/Disable");
 487
 488 /* Allowable values are 0 (IRQ), 1 (MSI/IRQ), and 2 (MSI-X/MSI/IRQ) */
 489 /* ToDo: Add MSI-X support. */
 490 static int bce_msi_enable = 1;
 491 TUNABLE_INT("hw.bce.msi_enable", &bce_msi_enable);
 492 SYSCTL_UINT(_hw_bce, OID_AUTO, msi_enable, CTLFLAG_RDTUN, &bce_msi_enable, 0,
 493 "MSI-X|MSI|INTx selector");
 494
 495 /* ToDo: Add tunable to enable/disable strict MTU handling. */
 496 /* Currently allows "loose" RX MTU checking (i.e. sets the  */
 497 /* H/W RX MTU to the size of the largest receive buffer, or */
 498 /* 2048 bytes). This will cause a UNH failure but is more   */
 499 /* desireable from a functional perspective.                */
 500
 501
 502 /****************************************************************************/
 503 /* Device probe function.                                                   */
 504 /*                                                                          */
 505 /* Compares the device to the driver's list of supported devices and        */
 506 /* reports back to the OS whether this is the right driver for the device.  */
 507 /*                                                                          */
 508 /* Returns:                                                                 */
 509 /*   BUS_PROBE_DEFAULT on success, positive value on failure.               */
 510 /****************************************************************************/
 511 static int
 512 bce_probe(device_t dev)
 513 {
 514         struct bce_type *t;
 515         struct bce_softc *sc;
 516         char *descbuf;
 517         u16 vid = 0, did = 0, svid = 0, sdid = 0;
 518
 519         t = bce_devs;
 520
 521         sc = device_get_softc(dev);
 522         bzero(sc, sizeof(struct bce_softc));
 523         sc->bce_unit = device_get_unit(dev);
 524         sc->bce_dev = dev;
 525
 526         /* Get the data for the device to be probed. */
 527         vid  = pci_get_vendor(dev);
 528         did  = pci_get_device(dev);
 529         svid = pci_get_subvendor(dev);
 530         sdid = pci_get_subdevice(dev);
 531
 532         DBPRINT(sc, BCE_EXTREME_LOAD,
 533                 "%s(); VID = 0x%04X, DID = 0x%04X, SVID = 0x%04X, "
 534                 "SDID = 0x%04X\n", __FUNCTION__, vid, did, svid, sdid);
 535
 536         /* Look through the list of known devices for a match. */
 537         while(t->bce_name != NULL) {
 538
 539                 if ((vid == t->bce_vid) && (did == t->bce_did) &&
 540                         ((svid == t->bce_svid) || (t->bce_svid == PCI_ANY_ID)) &&
 541                         ((sdid == t->bce_sdid) || (t->bce_sdid == PCI_ANY_ID))) {
 542
 543                         descbuf = malloc(BCE_DEVDESC_MAX, M_TEMP, M_NOWAIT);
 544
 545                         if (descbuf == NULL)
 546                                 return(ENOMEM);
 547
 548                         /* Print out the device identity. */
 549                         snprintf(descbuf, BCE_DEVDESC_MAX, "%s (%c%d)",
 550                                 t->bce_name,
 551                             (((pci_read_config(dev, PCIR_REVID, 4) & 0xf0) >> 4) + 'A'),
 552                             (pci_read_config(dev, PCIR_REVID, 4) & 0xf));
 553
 554                         device_set_desc_copy(dev, descbuf);
 555                         free(descbuf, M_TEMP);
 556                         return(BUS_PROBE_DEFAULT);
 557                 }
 558                 t++;
 559         }
 560
 561         return(ENXIO);
 562 }
 563
 564
 565 /****************************************************************************/
 566 /* PCI Capabilities Probe Function.                                         */
 567 /*                                                                          */
 568 /* Walks the PCI capabiites list for the device to find what features are   */
 569 /* supported.                                                               */
 570 /*                                                                          */
 571 /* Returns:                                                                 */
 572 /*   None.                                                                  */
 573 /****************************************************************************/
 574 static void
 575 bce_print_adapter_info(struct bce_softc *sc)
 576 {
 577         DBENTER(BCE_VERBOSE_LOAD);
 578
 579         BCE_PRINTF("ASIC (0x%08X); ", sc->bce_chipid);
 580         printf("Rev (%c%d); ", ((BCE_CHIP_ID(sc) & 0xf000) >> 12) + 'A',
 581                 ((BCE_CHIP_ID(sc) & 0x0ff0) >> 4));
 582
 583         /* Bus info. */
 584         if (sc->bce_flags & BCE_PCIE_FLAG) {
 585                 printf("Bus (PCIe x%d, ", sc->link_width);
 586                 switch (sc->link_speed) {
 587                         case 1: printf("2.5Gbps); "); break;
 588                         case 2: printf("5Gbps); "); break;
 589                         default: printf("Unknown link speed); ");
 590                 }
 591         } else {
 592                 printf("Bus (PCI%s, %s, %dMHz); ",
 593                         ((sc->bce_flags & BCE_PCIX_FLAG) ? "-X" : ""),
 594                         ((sc->bce_flags & BCE_PCI_32BIT_FLAG) ? "32-bit" : "64-bit"),
 595                         sc->bus_speed_mhz);
 596         }
 597
 598         /* Firmware version and device features. */
 599         printf("B/C (0x%08X); Flags( ", sc->bce_bc_ver);
 600 #ifdef ZERO_COPY_SOCKETS
 601         printf("SPLT ");
 602 #endif
 603         if (sc->bce_flags & BCE_MFW_ENABLE_FLAG)
 604                 printf("MFW ");
 605         if (sc->bce_flags & BCE_USING_MSI_FLAG)
 606                 printf("MSI ");
 607         if (sc->bce_flags & BCE_USING_MSIX_FLAG)
 608                 printf("MSI-X ");
 609         if (sc->bce_phy_flags & BCE_PHY_2_5G_CAPABLE_FLAG)
 610                 printf("2.5G ");
 611         printf(")\n");
 612
 613         DBEXIT(BCE_VERBOSE_LOAD);
 614 }
 615
 616
 617 /****************************************************************************/
 618 /* PCI Capabilities Probe Function.                                         */
 619 /*                                                                          */
 620 /* Walks the PCI capabiites list for the device to find what features are   */
 621 /* supported.                                                               */
 622 /*                                                                          */
 623 /* Returns:                                                                 */
 624 /*   None.                                                                  */
 625 /****************************************************************************/
 626 static void
 627 bce_probe_pci_caps(device_t dev, struct bce_softc *sc)
 628 {
 629         u32 reg;
 630
 631         DBENTER(BCE_VERBOSE_LOAD);
 632
 633         /* Check if PCI-X capability is enabled. */
 634         if (pci_find_extcap(dev, PCIY_PCIX, &reg) == 0) {
 635                 if (reg != 0)
 636                         sc->bce_cap_flags |= BCE_PCIX_CAPABLE_FLAG;
 637         }
 638
 639         /* Check if PCIe capability is enabled. */
 640         if (pci_find_extcap(dev, PCIY_EXPRESS, &reg) == 0) {
 641                 if (reg != 0) {
 642                         u16 link_status = pci_read_config(dev, reg + 0x12, 2);
 643                         DBPRINT(sc, BCE_INFO_LOAD, "PCIe link_status = 0x%08X\n",
 644                                 link_status);
 645                         sc->link_speed = link_status & 0xf;
 646                         sc->link_width = (link_status >> 4) & 0x3f;
 647                         sc->bce_cap_flags |= BCE_PCIE_CAPABLE_FLAG;
 648                         sc->bce_flags |= BCE_PCIE_FLAG;
 649                 }
 650         }
 651
 652         /* Check if MSI capability is enabled. */
 653         if (pci_find_extcap(dev, PCIY_MSI, &reg) == 0) {
 654                 if (reg != 0)
 655                         sc->bce_cap_flags |= BCE_MSI_CAPABLE_FLAG;
 656         }
 657
 658         /* Check if MSI-X capability is enabled. */
 659         if (pci_find_extcap(dev, PCIY_MSIX, &reg) == 0) {
 660                 if (reg != 0)
 661                         sc->bce_cap_flags |= BCE_MSIX_CAPABLE_FLAG;
 662         }
 663
 664         DBEXIT(BCE_VERBOSE_LOAD);
 665 }
 666
 667
 668 /****************************************************************************/
 669 /* Device attach function.                                                  */
 670 /*                                                                          */
 671 /* Allocates device resources, performs secondary chip identification,      */
 672 /* resets and initializes the hardware, and initializes driver instance     */
 673 /* variables.                                                               */
 674 /*                                                                          */
 675 /* Returns:                                                                 */
 676 /*   0 on success, positive value on failure.                               */
 677 /****************************************************************************/
 678 static int
 679 bce_attach(device_t dev)
 680 {
 681         struct bce_softc *sc;
 682         struct ifnet *ifp;
 683         u32 val;
 684         int error, rid, rc = 0;
 685
 686         sc = device_get_softc(dev);
 687         sc->bce_dev = dev;
 688
 689         DBENTER(BCE_VERBOSE_LOAD | BCE_VERBOSE_RESET);
 690
 691         sc->bce_unit = device_get_unit(dev);
 692
 693         /* Set initial device and PHY flags */
 694         sc->bce_flags = 0;
 695         sc->bce_phy_flags = 0;
 696
 697         pci_enable_busmaster(dev);
 698
 699         /* Allocate PCI memory resources. */
 700         rid = PCIR_BAR(0);
 701         sc->bce_res_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 702                 &rid, RF_ACTIVE | PCI_RF_DENSE);
 703
 704         if (sc->bce_res_mem == NULL) {
 705                 BCE_PRINTF("%s(%d): PCI memory allocation failed\n",
 706                         __FILE__, __LINE__);
 707                 rc = ENXIO;
 708                 goto bce_attach_fail;
 709         }
 710
 711         /* Get various resource handles. */
 712         sc->bce_btag    = rman_get_bustag(sc->bce_res_mem);
 713         sc->bce_bhandle = rman_get_bushandle(sc->bce_res_mem);
 714         sc->bce_vhandle = (vm_offset_t) rman_get_virtual(sc->bce_res_mem);
 715
 716         bce_probe_pci_caps(dev, sc);
 717
 718         rid = 1;
 719 #if 0
 720         /* Try allocating MSI-X interrupts. */
 721         if ((sc->bce_cap_flags & BCE_MSIX_CAPABLE_FLAG) &&
 722                 (bce_msi_enable >= 2) &&
 723                 ((sc->bce_res_irq = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 724                 &rid, RF_ACTIVE)) != NULL)) {
 725
 726                 msi_needed = sc->bce_msi_count = 1;
 727
 728                 if (((error = pci_alloc_msix(dev, &sc->bce_msi_count)) != 0) ||
 729                         (sc->bce_msi_count != msi_needed)) {
 730                         BCE_PRINTF("%s(%d): MSI-X allocation failed! Requested = %d,"
 731                                 "Received = %d, error = %d\n", __FILE__, __LINE__,
 732                                 msi_needed, sc->bce_msi_count, error);
 733                         sc->bce_msi_count = 0;
 734                         pci_release_msi(dev);
 735                         bus_release_resource(dev, SYS_RES_MEMORY, rid,
 736                                 sc->bce_res_irq);
 737                         sc->bce_res_irq = NULL;
 738                 } else {
 739                         DBPRINT(sc, BCE_INFO_LOAD, "%s(): Using MSI-X interrupt.\n",
 740                                 __FUNCTION__);
 741                         sc->bce_flags |= BCE_USING_MSIX_FLAG;
 742                         sc->bce_intr = bce_intr;
 743                 }
 744         }
 745 #endif
 746
 747         /* Try allocating a MSI interrupt. */
 748         if ((sc->bce_cap_flags & BCE_MSI_CAPABLE_FLAG) &&
 749                 (bce_msi_enable >= 1) && (sc->bce_msi_count == 0)) {
 750                 sc->bce_msi_count = 1;
 751                 if ((error = pci_alloc_msi(dev, &sc->bce_msi_count)) != 0) {
 752                         BCE_PRINTF("%s(%d): MSI allocation failed! error = %d\n",
 753                                 __FILE__, __LINE__, error);
 754                         sc->bce_msi_count = 0;
 755                         pci_release_msi(dev);
 756                 } else {
 757                         DBPRINT(sc, BCE_INFO_LOAD, "%s(): Using MSI interrupt.\n",
 758                                 __FUNCTION__);
 759                         sc->bce_flags |= BCE_USING_MSI_FLAG;
 760                         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
 761                                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716))
 762                                 sc->bce_flags |= BCE_ONE_SHOT_MSI_FLAG;
 763                         sc->bce_irq_rid = 1;
 764                         sc->bce_intr = bce_intr;
 765                 }
 766         }
 767
 768         /* Try allocating a legacy interrupt. */
 769         if (sc->bce_msi_count == 0) {
 770                 DBPRINT(sc, BCE_INFO_LOAD, "%s(): Using INTx interrupt.\n",
 771                         __FUNCTION__);
 772                 rid = 0;
 773                 sc->bce_intr = bce_intr;
 774         }
 775
 776         sc->bce_res_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ,
 777                 &rid, RF_SHAREABLE | RF_ACTIVE);
 778
 779         sc->bce_irq_rid = rid;
 780
 781         /* Report any IRQ allocation errors. */
 782         if (sc->bce_res_irq == NULL) {
 783                 BCE_PRINTF("%s(%d): PCI map interrupt failed!\n",
 784                         __FILE__, __LINE__);
 785                 rc = ENXIO;
 786                 goto bce_attach_fail;
 787         }
 788
 789         /* Initialize mutex for the current device instance. */
 790         BCE_LOCK_INIT(sc, device_get_nameunit(dev));
 791
 792         /*
 793          * Configure byte swap and enable indirect register access.
 794          * Rely on CPU to do target byte swapping on big endian systems.
 795          * Access to registers outside of PCI configurtion space are not
 796          * valid until this is done.
 797          */
 798         pci_write_config(dev, BCE_PCICFG_MISC_CONFIG,
 799                                BCE_PCICFG_MISC_CONFIG_REG_WINDOW_ENA |
 800                                BCE_PCICFG_MISC_CONFIG_TARGET_MB_WORD_SWAP, 4);
 801
 802         /* Save ASIC revsion info. */
 803         sc->bce_chipid =  REG_RD(sc, BCE_MISC_ID);
 804
 805         /* Weed out any non-production controller revisions. */
 806         switch(BCE_CHIP_ID(sc)) {
 807                 case BCE_CHIP_ID_5706_A0:
 808                 case BCE_CHIP_ID_5706_A1:
 809                 case BCE_CHIP_ID_5708_A0:
 810                 case BCE_CHIP_ID_5708_B0:
 811                 case BCE_CHIP_ID_5709_A0:
 812                 case BCE_CHIP_ID_5709_B0:
 813                 case BCE_CHIP_ID_5709_B1:
 814                 case BCE_CHIP_ID_5709_B2:
 815                         BCE_PRINTF("%s(%d): Unsupported controller revision (%c%d)!\n",
 816                                 __FILE__, __LINE__,
 817                                 (((pci_read_config(dev, PCIR_REVID, 4) & 0xf0) >> 4) + 'A'),
 818                             (pci_read_config(dev, PCIR_REVID, 4) & 0xf));
 819                         rc = ENODEV;
 820                         goto bce_attach_fail;
 821         }
 822
 823         /*
 824          * The embedded PCIe to PCI-X bridge (EPB)
 825          * in the 5708 cannot address memory above
 826          * 40 bits (E7_5708CB1_23043 & E6_5708SB1_23043).
 827          */
 828         if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5708)
 829                 sc->max_bus_addr = BCE_BUS_SPACE_MAXADDR;
 830         else
 831                 sc->max_bus_addr = BUS_SPACE_MAXADDR;
 832
 833         /*
 834          * Find the base address for shared memory access.
 835          * Newer versions of bootcode use a signature and offset
 836          * while older versions use a fixed address.
 837          */
 838         val = REG_RD_IND(sc, BCE_SHM_HDR_SIGNATURE);
 839         if ((val & BCE_SHM_HDR_SIGNATURE_SIG_MASK) == BCE_SHM_HDR_SIGNATURE_SIG)
 840                 /* Multi-port devices use different offsets in shared memory. */
 841                 sc->bce_shmem_base = REG_RD_IND(sc, BCE_SHM_HDR_ADDR_0 +
 842                         (pci_get_function(sc->bce_dev) << 2));
 843         else
 844                 sc->bce_shmem_base = HOST_VIEW_SHMEM_BASE;
 845
 846         DBPRINT(sc, BCE_VERBOSE_FIRMWARE, "%s(): bce_shmem_base = 0x%08X\n",
 847                 __FUNCTION__, sc->bce_shmem_base);
 848
 849         /* Fetch the bootcode revision. */
 850         sc->bce_bc_ver = REG_RD_IND(sc, sc->bce_shmem_base +
 851                 BCE_DEV_INFO_BC_REV);
 852
 853         /* Check if any management firmware is running. */
 854         val = REG_RD_IND(sc, sc->bce_shmem_base + BCE_PORT_FEATURE);
 855         if (val & (BCE_PORT_FEATURE_ASF_ENABLED | BCE_PORT_FEATURE_IMD_ENABLED))
 856                 sc->bce_flags |= BCE_MFW_ENABLE_FLAG;
 857
 858         /* Get PCI bus information (speed and type). */
 859         val = REG_RD(sc, BCE_PCICFG_MISC_STATUS);
 860         if (val & BCE_PCICFG_MISC_STATUS_PCIX_DET) {
 861                 u32 clkreg;
 862
 863                 sc->bce_flags |= BCE_PCIX_FLAG;
 864
 865                 clkreg = REG_RD(sc, BCE_PCICFG_PCI_CLOCK_CONTROL_BITS);
 866
 867                 clkreg &= BCE_PCICFG_PCI_CLOCK_CONTROL_BITS_PCI_CLK_SPD_DET;
 868                 switch (clkreg) {
 869                 case BCE_PCICFG_PCI_CLOCK_CONTROL_BITS_PCI_CLK_SPD_DET_133MHZ:
 870                         sc->bus_speed_mhz = 133;
 871                         break;
 872
 873                 case BCE_PCICFG_PCI_CLOCK_CONTROL_BITS_PCI_CLK_SPD_DET_95MHZ:
 874                         sc->bus_speed_mhz = 100;
 875                         break;
 876
 877                 case BCE_PCICFG_PCI_CLOCK_CONTROL_BITS_PCI_CLK_SPD_DET_66MHZ:
 878                 case BCE_PCICFG_PCI_CLOCK_CONTROL_BITS_PCI_CLK_SPD_DET_80MHZ:
 879                         sc->bus_speed_mhz = 66;
 880                         break;
 881
 882                 case BCE_PCICFG_PCI_CLOCK_CONTROL_BITS_PCI_CLK_SPD_DET_48MHZ:
 883                 case BCE_PCICFG_PCI_CLOCK_CONTROL_BITS_PCI_CLK_SPD_DET_55MHZ:
 884                         sc->bus_speed_mhz = 50;
 885                         break;
 886
 887                 case BCE_PCICFG_PCI_CLOCK_CONTROL_BITS_PCI_CLK_SPD_DET_LOW:
 888                 case BCE_PCICFG_PCI_CLOCK_CONTROL_BITS_PCI_CLK_SPD_DET_32MHZ:
 889                 case BCE_PCICFG_PCI_CLOCK_CONTROL_BITS_PCI_CLK_SPD_DET_38MHZ:
 890                         sc->bus_speed_mhz = 33;
 891                         break;
 892                 }
 893         } else {
 894                 if (val & BCE_PCICFG_MISC_STATUS_M66EN)
 895                         sc->bus_speed_mhz = 66;
 896                 else
 897                         sc->bus_speed_mhz = 33;
 898         }
 899
 900         if (val & BCE_PCICFG_MISC_STATUS_32BIT_DET)
 901                 sc->bce_flags |= BCE_PCI_32BIT_FLAG;
 902
 903         /* Reset the controller and announce to bootcode that driver is present. */
 904         if (bce_reset(sc, BCE_DRV_MSG_CODE_RESET)) {
 905                 BCE_PRINTF("%s(%d): Controller reset failed!\n",
 906                         __FILE__, __LINE__);
 907                 rc = ENXIO;
 908                 goto bce_attach_fail;
 909         }
 910
 911         /* Initialize the controller. */
 912         if (bce_chipinit(sc)) {
 913                 BCE_PRINTF("%s(%d): Controller initialization failed!\n",
 914                         __FILE__, __LINE__);
 915                 rc = ENXIO;
 916                 goto bce_attach_fail;
 917         }
 918
 919         /* Perform NVRAM test. */
 920         if (bce_nvram_test(sc)) {
 921                 BCE_PRINTF("%s(%d): NVRAM test failed!\n",
 922                         __FILE__, __LINE__);
 923                 rc = ENXIO;
 924                 goto bce_attach_fail;
 925         }
 926
 927         /* Fetch the permanent Ethernet MAC address. */
 928         bce_get_mac_addr(sc);
 929
 930         /*
 931          * Trip points control how many BDs
 932          * should be ready before generating an
 933          * interrupt while ticks control how long
 934          * a BD can sit in the chain before
 935          * generating an interrupt.  Set the default
 936          * values for the RX and TX chains.
 937          */
 938
 939 #ifdef BCE_DEBUG
 940         /* Force more frequent interrupts. */
 941         sc->bce_tx_quick_cons_trip_int = 1;
 942         sc->bce_tx_quick_cons_trip     = 1;
 943         sc->bce_tx_ticks_int           = 0;
 944         sc->bce_tx_ticks               = 0;
 945
 946         sc->bce_rx_quick_cons_trip_int = 1;
 947         sc->bce_rx_quick_cons_trip     = 1;
 948         sc->bce_rx_ticks_int           = 0;
 949         sc->bce_rx_ticks               = 0;
 950 #else
 951         /* Improve throughput at the expense of increased latency. */
 952         sc->bce_tx_quick_cons_trip_int = 20;
 953         sc->bce_tx_quick_cons_trip     = 20;
 954         sc->bce_tx_ticks_int           = 80;
 955         sc->bce_tx_ticks               = 80;
 956
 957         sc->bce_rx_quick_cons_trip_int = 6;
 958         sc->bce_rx_quick_cons_trip     = 6;
 959         sc->bce_rx_ticks_int           = 18;
 960         sc->bce_rx_ticks               = 18;
 961 #endif
 962
 963         /* Update statistics once every second. */
 964         sc->bce_stats_ticks = 1000000 & 0xffff00;
 965
 966         /* Find the media type for the adapter. */
 967         bce_get_media(sc);
 968
 969         /* Store data needed by PHY driver for backplane applications */
 970         sc->bce_shared_hw_cfg = REG_RD_IND(sc, sc->bce_shmem_base +
 971                 BCE_SHARED_HW_CFG_CONFIG);
 972         sc->bce_port_hw_cfg   = REG_RD_IND(sc, sc->bce_shmem_base +
 973                 BCE_PORT_HW_CFG_CONFIG);
 974
 975         /* Allocate DMA memory resources. */
 976         if (bce_dma_alloc(dev)) {
 977                 BCE_PRINTF("%s(%d): DMA resource allocation failed!\n",
 978                     __FILE__, __LINE__);
 979                 rc = ENXIO;
 980                 goto bce_attach_fail;
 981         }
 982
 983         /* Allocate an ifnet structure. */
 984         ifp = sc->bce_ifp = if_alloc(IFT_ETHER);
 985         if (ifp == NULL) {
 986                 BCE_PRINTF("%s(%d): Interface allocation failed!\n",
 987                         __FILE__, __LINE__);
 988                 rc = ENXIO;
 989                 goto bce_attach_fail;
 990         }
 991
 992         /* Initialize the ifnet interface. */
 993         ifp->if_softc        = sc;
 994         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 995         ifp->if_flags        = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 996         ifp->if_ioctl        = bce_ioctl;
 997         ifp->if_start        = bce_start;
 998         ifp->if_init         = bce_init;
 999         ifp->if_mtu          = ETHERMTU;
1000
1001         if (bce_tso_enable) {
1002                 ifp->if_hwassist = BCE_IF_HWASSIST | CSUM_TSO;
1003                 ifp->if_capabilities = BCE_IF_CAPABILITIES | IFCAP_TSO4;
1004         } else {
1005                 ifp->if_hwassist = BCE_IF_HWASSIST;
1006                 ifp->if_capabilities = BCE_IF_CAPABILITIES;
1007         }
1008
1009         ifp->if_capenable    = ifp->if_capabilities;
1010
1011         /*
1012          * Assume standard mbuf sizes for buffer allocation.
1013          * This may change later if the MTU size is set to
1014          * something other than 1500.
1015          */
1016 #ifdef ZERO_COPY_SOCKETS
1017         sc->rx_bd_mbuf_alloc_size = MHLEN;
1018         /* Make sure offset is 16 byte aligned for hardware. */
1019         sc->rx_bd_mbuf_align_pad  = roundup2((MSIZE - MHLEN), 16) -
1020                 (MSIZE - MHLEN);
1021         sc->rx_bd_mbuf_data_len   = sc->rx_bd_mbuf_alloc_size -
1022                 sc->rx_bd_mbuf_align_pad;
1023         sc->pg_bd_mbuf_alloc_size = MCLBYTES;
1024 #else
1025         sc->rx_bd_mbuf_alloc_size = MCLBYTES;
1026         sc->rx_bd_mbuf_align_pad  = roundup2(MCLBYTES, 16) - MCLBYTES;
1027         sc->rx_bd_mbuf_data_len   = sc->rx_bd_mbuf_alloc_size -
1028                 sc->rx_bd_mbuf_align_pad;
1029 #endif
1030
1031         ifp->if_snd.ifq_drv_maxlen = USABLE_TX_BD;
1032         IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen);
1033         IFQ_SET_READY(&ifp->if_snd);
1034
1035         if (sc->bce_phy_flags & BCE_PHY_2_5G_CAPABLE_FLAG)
1036                 ifp->if_baudrate = IF_Mbps(2500ULL);
1037         else
1038                 ifp->if_baudrate = IF_Mbps(1000);
1039
1040         /* Check for an MII child bus by probing the PHY. */
1041         if (mii_phy_probe(dev, &sc->bce_miibus, bce_ifmedia_upd,
1042                 bce_ifmedia_sts)) {
1043                 BCE_PRINTF("%s(%d): No PHY found on child MII bus!\n",
1044                         __FILE__, __LINE__);
1045                 rc = ENXIO;
1046                 goto bce_attach_fail;
1047         }
1048
1049         /* Attach to the Ethernet interface list. */
1050         ether_ifattach(ifp, sc->eaddr);
1051
1052 #if __FreeBSD_version < 500000
1053         callout_init(&sc->bce_tick_callout);
1054         callout_init(&sc->bce_pulse_callout);
1055 #else
1056         callout_init_mtx(&sc->bce_tick_callout, &sc->bce_mtx, 0);
1057         callout_init_mtx(&sc->bce_pulse_callout, &sc->bce_mtx, 0);
1058 #endif
1059
1060         /* Hookup IRQ last. */
1061         rc = bus_setup_intr(dev, sc->bce_res_irq, INTR_TYPE_NET | INTR_MPSAFE,
1062                 NULL, bce_intr, sc, &sc->bce_intrhand);
1063
1064         if (rc) {
1065                 BCE_PRINTF("%s(%d): Failed to setup IRQ!\n",
1066                         __FILE__, __LINE__);
1067                 bce_detach(dev);
1068                 goto bce_attach_exit;
1069         }
1070
1071         /*
1072          * At this point we've acquired all the resources
1073          * we need to run so there's no turning back, we're
1074          * cleared for launch.
1075          */
1076
1077         /* Print some important debugging info. */
1078         DBRUNMSG(BCE_INFO, bce_dump_driver_state(sc));
1079
1080         /* Add the supported sysctls to the kernel. */
1081         bce_add_sysctls(sc);
1082
1083         BCE_LOCK(sc);
1084
1085         /*
1086          * The chip reset earlier notified the bootcode that
1087          * a driver is present.  We now need to start our pulse
1088          * routine so that the bootcode is reminded that we're
1089          * still running.
1090          */
1091         bce_pulse(sc);
1092
1093         bce_mgmt_init_locked(sc);
1094         BCE_UNLOCK(sc);
1095
1096         /* Finally, print some useful adapter info */
1097         bce_print_adapter_info(sc);
1098         DBPRINT(sc, BCE_FATAL, "%s(): sc = %p\n",
1099                 __FUNCTION__, sc);
1100
1101         goto bce_attach_exit;
1102
1103 bce_attach_fail:
1104         bce_release_resources(sc);
1105
1106 bce_attach_exit:
1107
1108         DBEXIT(BCE_VERBOSE_LOAD | BCE_VERBOSE_RESET);
1109
1110         return(rc);
1111 }
1112
1113
1114 /****************************************************************************/
1115 /* Device detach function.                                                  */
1116 /*                                                                          */
1117 /* Stops the controller, resets the controller, and releases resources.     */
1118 /*                                                                          */
1119 /* Returns:                                                                 */
1120 /*   0 on success, positive value on failure.                               */
1121 /****************************************************************************/
1122 static int
1123 bce_detach(device_t dev)
1124 {
1125         struct bce_softc *sc = device_get_softc(dev);
1126         struct ifnet *ifp;
1127         u32 msg;
1128
1129         DBENTER(BCE_VERBOSE_UNLOAD | BCE_VERBOSE_RESET);
1130
1131         ifp = sc->bce_ifp;
1132
1133         /* Stop and reset the controller. */
1134         BCE_LOCK(sc);
1135
1136         /* Stop the pulse so the bootcode can go to driver absent state. */
1137         callout_stop(&sc->bce_pulse_callout);
1138
1139         bce_stop(sc);
1140         if (sc->bce_flags & BCE_NO_WOL_FLAG)
1141                 msg = BCE_DRV_MSG_CODE_UNLOAD_LNK_DN;
1142         else
1143                 msg = BCE_DRV_MSG_CODE_UNLOAD;
1144         bce_reset(sc, msg);
1145
1146         BCE_UNLOCK(sc);
1147
1148         ether_ifdetach(ifp);
1149
1150         /* If we have a child device on the MII bus remove it too. */
1151         bus_generic_detach(dev);
1152         device_delete_child(dev, sc->bce_miibus);
1153
1154         /* Release all remaining resources. */
1155         bce_release_resources(sc);
1156
1157         DBEXIT(BCE_VERBOSE_UNLOAD | BCE_VERBOSE_RESET);
1158
1159         return(0);
1160 }
1161
1162
1163 /****************************************************************************/
1164 /* Device shutdown function.                                                */
1165 /*                                                                          */
1166 /* Stops and resets the controller.                                         */
1167 /*                                                                          */
1168 /* Returns:                                                                 */
1169 /*   0 on success, positive value on failure.                               */
1170 /****************************************************************************/
1171 static int
1172 bce_shutdown(device_t dev)
1173 {
1174         struct bce_softc *sc = device_get_softc(dev);
1175         u32 msg;
1176
1177         DBENTER(BCE_VERBOSE);
1178
1179         BCE_LOCK(sc);
1180         bce_stop(sc);
1181         if (sc->bce_flags & BCE_NO_WOL_FLAG)
1182                 msg = BCE_DRV_MSG_CODE_UNLOAD_LNK_DN;
1183         else
1184                 msg = BCE_DRV_MSG_CODE_UNLOAD;
1185         bce_reset(sc, msg);
1186         BCE_UNLOCK(sc);
1187
1188         DBEXIT(BCE_VERBOSE);
1189
1190         return (0);
1191 }
1192
1193
1194 #ifdef BCE_DEBUG
1195 /****************************************************************************/
1196 /* Register read.                                                           */
1197 /*                                                                          */
1198 /* Returns:                                                                 */
1199 /*   The value of the register.                                             */
1200 /****************************************************************************/
1201 static u32
1202 bce_reg_rd(struct bce_softc *sc, u32 offset)
1203 {
1204         u32 val = bus_space_read_4(sc->bce_btag, sc->bce_bhandle, offset);
1205         DBPRINT(sc, BCE_INSANE_REG, "%s(); offset = 0x%08X, val = 0x%08X\n",
1206                 __FUNCTION__, offset, val);
1207         return val;
1208 }
1209
1210
1211 /****************************************************************************/
1212 /* Register write (16 bit).                                                 */
1213 /*                                                                          */
1214 /* Returns:                                                                 */
1215 /*   Nothing.                                                               */
1216 /****************************************************************************/
1217 static void
1218 bce_reg_wr16(struct bce_softc *sc, u32 offset, u16 val)
1219 {
1220         DBPRINT(sc, BCE_INSANE_REG, "%s(); offset = 0x%08X, val = 0x%04X\n",
1221                 __FUNCTION__, offset, val);
1222         bus_space_write_2(sc->bce_btag, sc->bce_bhandle, offset, val);
1223 }
1224
1225
1226 /****************************************************************************/
1227 /* Register write.                                                          */
1228 /*                                                                          */
1229 /* Returns:                                                                 */
1230 /*   Nothing.                                                               */
1231 /****************************************************************************/
1232 static void
1233 bce_reg_wr(struct bce_softc *sc, u32 offset, u32 val)
1234 {
1235         DBPRINT(sc, BCE_INSANE_REG, "%s(); offset = 0x%08X, val = 0x%08X\n",
1236                 __FUNCTION__, offset, val);
1237         bus_space_write_4(sc->bce_btag, sc->bce_bhandle, offset, val);
1238 }
1239 #endif
1240
1241 /****************************************************************************/
1242 /* Indirect register read.                                                  */
1243 /*                                                                          */
1244 /* Reads NetXtreme II registers using an index/data register pair in PCI    */
1245 /* configuration space.  Using this mechanism avoids issues with posted     */
1246 /* reads but is much slower than memory-mapped I/O.                         */
1247 /*                                                                          */
1248 /* Returns:                                                                 */
1249 /*   The value of the register.                                             */
1250 /****************************************************************************/
1251 static u32
1252 bce_reg_rd_ind(struct bce_softc *sc, u32 offset)
1253 {
1254         device_t dev;
1255         dev = sc->bce_dev;
1256
1257         pci_write_config(dev, BCE_PCICFG_REG_WINDOW_ADDRESS, offset, 4);
1258 #ifdef BCE_DEBUG
1259         {
1260                 u32 val;
1261                 val = pci_read_config(dev, BCE_PCICFG_REG_WINDOW, 4);
1262                 DBPRINT(sc, BCE_INSANE_REG, "%s(); offset = 0x%08X, val = 0x%08X\n",
1263                         __FUNCTION__, offset, val);
1264                 return val;
1265         }
1266 #else
1267         return pci_read_config(dev, BCE_PCICFG_REG_WINDOW, 4);
1268 #endif
1269 }
1270
1271
1272 /****************************************************************************/
1273 /* Indirect register write.                                                 */
1274 /*                                                                          */
1275 /* Writes NetXtreme II registers using an index/data register pair in PCI   */
1276 /* configuration space.  Using this mechanism avoids issues with posted     */
1277 /* writes but is muchh slower than memory-mapped I/O.                       */
1278 /*                                                                          */
1279 /* Returns:                                                                 */
1280 /*   Nothing.                                                               */
1281 /****************************************************************************/
1282 static void
1283 bce_reg_wr_ind(struct bce_softc *sc, u32 offset, u32 val)
1284 {
1285         device_t dev;
1286         dev = sc->bce_dev;
1287
1288         DBPRINT(sc, BCE_INSANE_REG, "%s(); offset = 0x%08X, val = 0x%08X\n",
1289                 __FUNCTION__, offset, val);
1290
1291         pci_write_config(dev, BCE_PCICFG_REG_WINDOW_ADDRESS, offset, 4);
1292         pci_write_config(dev, BCE_PCICFG_REG_WINDOW, val, 4);
1293 }
1294
1295
1296 #ifdef BCE_DEBUG
1297 /****************************************************************************/
1298 /* Context memory read.                                                     */
1299 /*                                                                          */
1300 /* The NetXtreme II controller uses context memory to track connection      */
1301 /* information for L2 and higher network protocols.                         */
1302 /*                                                                          */
1303 /* Returns:                                                                 */
1304 /*   The requested 32 bit value of context memory.                          */
1305 /****************************************************************************/
1306 static u32
1307 bce_ctx_rd(struct bce_softc *sc, u32 cid_addr, u32 ctx_offset)
1308 {
1309         u32 idx, offset, retry_cnt = 5, val;
1310
1311         DBRUNIF((cid_addr > MAX_CID_ADDR || ctx_offset & 0x3 || cid_addr & CTX_MASK),
1312                 BCE_PRINTF("%s(): Invalid CID address: 0x%08X.\n",
1313                         __FUNCTION__, cid_addr));
1314
1315         offset = ctx_offset + cid_addr;
1316
1317         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
1318                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
1319
1320                 REG_WR(sc, BCE_CTX_CTX_CTRL, (offset | BCE_CTX_CTX_CTRL_READ_REQ));
1321
1322                 for (idx = 0; idx < retry_cnt; idx++) {
1323                         val = REG_RD(sc, BCE_CTX_CTX_CTRL);
1324                         if ((val & BCE_CTX_CTX_CTRL_READ_REQ) == 0)
1325                                 break;
1326                         DELAY(5);
1327                 }
1328
1329                 if (val & BCE_CTX_CTX_CTRL_READ_REQ)
1330                         BCE_PRINTF("%s(%d); Unable to read CTX memory: "
1331                                 "cid_addr = 0x%08X, offset = 0x%08X!\n",
1332                                 __FILE__, __LINE__, cid_addr, ctx_offset);
1333
1334                 val = REG_RD(sc, BCE_CTX_CTX_DATA);
1335         } else {
1336                 REG_WR(sc, BCE_CTX_DATA_ADR, offset);
1337                 val = REG_RD(sc, BCE_CTX_DATA);
1338         }
1339
1340         DBPRINT(sc, BCE_EXTREME_CTX, "%s(); cid_addr = 0x%08X, offset = 0x%08X, "
1341                 "val = 0x%08X\n", __FUNCTION__, cid_addr, ctx_offset, val);
1342
1343         return(val);
1344 }
1345 #endif
1346
1347
1348 /****************************************************************************/
1349 /* Context memory write.                                                    */
1350 /*                                                                          */
1351 /* The NetXtreme II controller uses context memory to track connection      */
1352 /* information for L2 and higher network protocols.                         */
1353 /*                                                                          */
1354 /* Returns:                                                                 */
1355 /*   Nothing.                                                               */
1356 /****************************************************************************/
1357 static void
1358 bce_ctx_wr(struct bce_softc *sc, u32 cid_addr, u32 ctx_offset, u32 ctx_val)
1359 {
1360         u32 idx, offset = ctx_offset + cid_addr;
1361         u32 val, retry_cnt = 5;
1362
1363         DBPRINT(sc, BCE_EXTREME_CTX, "%s(); cid_addr = 0x%08X, offset = 0x%08X, "
1364                 "val = 0x%08X\n", __FUNCTION__, cid_addr, ctx_offset, ctx_val);
1365
1366         DBRUNIF((cid_addr > MAX_CID_ADDR || ctx_offset & 0x3 || cid_addr & CTX_MASK),
1367                 BCE_PRINTF("%s(): Invalid CID address: 0x%08X.\n",
1368                         __FUNCTION__, cid_addr));
1369
1370         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
1371                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
1372
1373                 REG_WR(sc, BCE_CTX_CTX_DATA, ctx_val);
1374                 REG_WR(sc, BCE_CTX_CTX_CTRL, (offset | BCE_CTX_CTX_CTRL_WRITE_REQ));
1375
1376                 for (idx = 0; idx < retry_cnt; idx++) {
1377                         val = REG_RD(sc, BCE_CTX_CTX_CTRL);
1378                         if ((val & BCE_CTX_CTX_CTRL_WRITE_REQ) == 0)
1379                                 break;
1380                         DELAY(5);
1381                 }
1382
1383                 if (val & BCE_CTX_CTX_CTRL_WRITE_REQ)
1384                         BCE_PRINTF("%s(%d); Unable to write CTX memory: "
1385                                 "cid_addr = 0x%08X, offset = 0x%08X!\n",
1386                                 __FILE__, __LINE__, cid_addr, ctx_offset);
1387
1388         } else {
1389                 REG_WR(sc, BCE_CTX_DATA_ADR, offset);
1390                 REG_WR(sc, BCE_CTX_DATA, ctx_val);
1391         }
1392 }
1393
1394
1395 /****************************************************************************/
1396 /* PHY register read.                                                       */
1397 /*                                                                          */
1398 /* Implements register reads on the MII bus.                                */
1399 /*                                                                          */
1400 /* Returns:                                                                 */
1401 /*   The value of the register.                                             */
1402 /****************************************************************************/
1403 static int
1404 bce_miibus_read_reg(device_t dev, int phy, int reg)
1405 {
1406         struct bce_softc *sc;
1407         u32 val;
1408         int i;
1409
1410         sc = device_get_softc(dev);
1411
1412         /* Make sure we are accessing the correct PHY address. */
1413         if (phy != sc->bce_phy_addr) {
1414                 DBPRINT(sc, BCE_INSANE_PHY, "Invalid PHY address %d for PHY read!\n", phy);
1415                 return(0);
1416         }
1417
1418         if (sc->bce_phy_flags & BCE_PHY_INT_MODE_AUTO_POLLING_FLAG) {
1419                 val = REG_RD(sc, BCE_EMAC_MDIO_MODE);
1420                 val &= ~BCE_EMAC_MDIO_MODE_AUTO_POLL;
1421
1422                 REG_WR(sc, BCE_EMAC_MDIO_MODE, val);
1423                 REG_RD(sc, BCE_EMAC_MDIO_MODE);
1424
1425                 DELAY(40);
1426         }
1427
1428
1429         val = BCE_MIPHY(phy) | BCE_MIREG(reg) |
1430                 BCE_EMAC_MDIO_COMM_COMMAND_READ | BCE_EMAC_MDIO_COMM_DISEXT |
1431                 BCE_EMAC_MDIO_COMM_START_BUSY;
1432         REG_WR(sc, BCE_EMAC_MDIO_COMM, val);
1433
1434         for (i = 0; i < BCE_PHY_TIMEOUT; i++) {
1435                 DELAY(10);
1436
1437                 val = REG_RD(sc, BCE_EMAC_MDIO_COMM);
1438                 if (!(val & BCE_EMAC_MDIO_COMM_START_BUSY)) {
1439                         DELAY(5);
1440
1441                         val = REG_RD(sc, BCE_EMAC_MDIO_COMM);
1442                         val &= BCE_EMAC_MDIO_COMM_DATA;
1443
1444                         break;
1445                 }
1446         }
1447
1448         if (val & BCE_EMAC_MDIO_COMM_START_BUSY) {
1449                 BCE_PRINTF("%s(%d): Error: PHY read timeout! phy = %d, reg = 0x%04X\n",
1450                         __FILE__, __LINE__, phy, reg);
1451                 val = 0x0;
1452         } else {
1453                 val = REG_RD(sc, BCE_EMAC_MDIO_COMM);
1454         }
1455
1456
1457         if (sc->bce_phy_flags & BCE_PHY_INT_MODE_AUTO_POLLING_FLAG) {
1458                 val = REG_RD(sc, BCE_EMAC_MDIO_MODE);
1459                 val |= BCE_EMAC_MDIO_MODE_AUTO_POLL;
1460
1461                 REG_WR(sc, BCE_EMAC_MDIO_MODE, val);
1462                 REG_RD(sc, BCE_EMAC_MDIO_MODE);
1463
1464                 DELAY(40);
1465         }
1466
1467         DB_PRINT_PHY_REG(reg, val);
1468         return (val & 0xffff);
1469
1470 }
1471
1472
1473 /****************************************************************************/
1474 /* PHY register write.                                                      */
1475 /*                                                                          */
1476 /* Implements register writes on the MII bus.                               */
1477 /*                                                                          */
1478 /* Returns:                                                                 */
1479 /*   The value of the register.                                             */
1480 /****************************************************************************/
1481 static int
1482 bce_miibus_write_reg(device_t dev, int phy, int reg, int val)
1483 {
1484         struct bce_softc *sc;
1485         u32 val1;
1486         int i;
1487
1488         sc = device_get_softc(dev);
1489
1490         /* Make sure we are accessing the correct PHY address. */
1491         if (phy != sc->bce_phy_addr) {
1492                 DBPRINT(sc, BCE_INSANE_PHY, "Invalid PHY address %d for PHY write!\n", phy);
1493                 return(0);
1494         }
1495
1496         DB_PRINT_PHY_REG(reg, val);
1497
1498         if (sc->bce_phy_flags & BCE_PHY_INT_MODE_AUTO_POLLING_FLAG) {
1499                 val1 = REG_RD(sc, BCE_EMAC_MDIO_MODE);
1500                 val1 &= ~BCE_EMAC_MDIO_MODE_AUTO_POLL;
1501
1502                 REG_WR(sc, BCE_EMAC_MDIO_MODE, val1);
1503                 REG_RD(sc, BCE_EMAC_MDIO_MODE);
1504
1505                 DELAY(40);
1506         }
1507
1508         val1 = BCE_MIPHY(phy) | BCE_MIREG(reg) | val |
1509                 BCE_EMAC_MDIO_COMM_COMMAND_WRITE |
1510                 BCE_EMAC_MDIO_COMM_START_BUSY | BCE_EMAC_MDIO_COMM_DISEXT;
1511         REG_WR(sc, BCE_EMAC_MDIO_COMM, val1);
1512
1513         for (i = 0; i < BCE_PHY_TIMEOUT; i++) {
1514                 DELAY(10);
1515
1516                 val1 = REG_RD(sc, BCE_EMAC_MDIO_COMM);
1517                 if (!(val1 & BCE_EMAC_MDIO_COMM_START_BUSY)) {
1518                         DELAY(5);
1519                         break;
1520                 }
1521         }
1522
1523         if (val1 & BCE_EMAC_MDIO_COMM_START_BUSY)
1524                 BCE_PRINTF("%s(%d): PHY write timeout!\n",
1525                         __FILE__, __LINE__);
1526
1527         if (sc->bce_phy_flags & BCE_PHY_INT_MODE_AUTO_POLLING_FLAG) {
1528                 val1 = REG_RD(sc, BCE_EMAC_MDIO_MODE);
1529                 val1 |= BCE_EMAC_MDIO_MODE_AUTO_POLL;
1530
1531                 REG_WR(sc, BCE_EMAC_MDIO_MODE, val1);
1532                 REG_RD(sc, BCE_EMAC_MDIO_MODE);
1533
1534                 DELAY(40);
1535         }
1536
1537         return 0;
1538 }
1539
1540
1541 /****************************************************************************/
1542 /* MII bus status change.                                                   */
1543 /*                                                                          */
1544 /* Called by the MII bus driver when the PHY establishes link to set the    */
1545 /* MAC interface registers.                                                 */
1546 /*                                                                          */
1547 /* Returns:                                                                 */
1548 /*   Nothing.                                                               */
1549 /****************************************************************************/
1550 static void
1551 bce_miibus_statchg(device_t dev)
1552 {
1553         struct bce_softc *sc;
1554         struct mii_data *mii;
1555         int val;
1556
1557         sc = device_get_softc(dev);
1558
1559         DBENTER(BCE_VERBOSE_PHY);
1560
1561         mii = device_get_softc(sc->bce_miibus);
1562
1563         val = REG_RD(sc, BCE_EMAC_MODE);
1564         val &= ~(BCE_EMAC_MODE_PORT | BCE_EMAC_MODE_HALF_DUPLEX |
1565                 BCE_EMAC_MODE_MAC_LOOP | BCE_EMAC_MODE_FORCE_LINK |
1566                 BCE_EMAC_MODE_25G);
1567
1568         /* Set MII or GMII interface based on the speed negotiated by the PHY. */
1569         switch (IFM_SUBTYPE(mii->mii_media_active)) {
1570         case IFM_10_T:
1571                 if (BCE_CHIP_NUM(sc) != BCE_CHIP_NUM_5706) {
1572                         DBPRINT(sc, BCE_INFO, "Enabling 10Mb interface.\n");
1573                         val |= BCE_EMAC_MODE_PORT_MII_10;
1574                         break;
1575                 }
1576                 /* fall-through */
1577         case IFM_100_TX:
1578                 DBPRINT(sc, BCE_INFO, "Enabling MII interface.\n");
1579                 val |= BCE_EMAC_MODE_PORT_MII;
1580                 break;
1581         case IFM_2500_SX:
1582                 DBPRINT(sc, BCE_INFO, "Enabling 2.5G MAC mode.\n");
1583                 val |= BCE_EMAC_MODE_25G;
1584                 /* fall-through */
1585         case IFM_1000_T:
1586         case IFM_1000_SX:
1587                 DBPRINT(sc, BCE_INFO, "Enabling GMII interface.\n");
1588                 val |= BCE_EMAC_MODE_PORT_GMII;
1589                 break;
1590         default:
1591                 DBPRINT(sc, BCE_INFO, "Unknown speed, enabling default GMII "
1592                         "interface.\n");
1593                 val |= BCE_EMAC_MODE_PORT_GMII;
1594         }
1595
1596         /* Set half or full duplex based on the duplicity negotiated by the PHY. */
1597         if ((mii->mii_media_active & IFM_GMASK) == IFM_HDX) {
1598                 DBPRINT(sc, BCE_INFO, "Setting Half-Duplex interface.\n");
1599                 val |= BCE_EMAC_MODE_HALF_DUPLEX;
1600         } else
1601                 DBPRINT(sc, BCE_INFO, "Setting Full-Duplex interface.\n");
1602
1603         REG_WR(sc, BCE_EMAC_MODE, val);
1604
1605 #if 0
1606         /* ToDo: Enable flow control support in brgphy and bge. */
1607         /* FLAG0 is set if RX is enabled and FLAG1 if TX is enabled */
1608         if (mii->mii_media_active & IFM_FLAG0)
1609                 BCE_SETBIT(sc, BCE_EMAC_RX_MODE, BCE_EMAC_RX_MODE_FLOW_EN);
1610         if (mii->mii_media_active & IFM_FLAG1)
1611                 BCE_SETBIT(sc, BCE_EMAC_RX_MODE, BCE_EMAC_TX_MODE_FLOW_EN);
1612 #endif
1613
1614         DBEXIT(BCE_VERBOSE_PHY);
1615 }
1616
1617
1618 /****************************************************************************/
1619 /* Acquire NVRAM lock.                                                      */
1620 /*                                                                          */
1621 /* Before the NVRAM can be accessed the caller must acquire an NVRAM lock.  */
1622 /* Locks 0 and 2 are reserved, lock 1 is used by firmware and lock 2 is     */
1623 /* for use by the driver.                                                   */
1624 /*                                                                          */
1625 /* Returns:                                                                 */
1626 /*   0 on success, positive value on failure.                               */
1627 /****************************************************************************/
1628 static int
1629 bce_acquire_nvram_lock(struct bce_softc *sc)
1630 {
1631         u32 val;
1632         int j, rc = 0;
1633
1634         DBENTER(BCE_VERBOSE_NVRAM);
1635
1636         /* Request access to the flash interface. */
1637         REG_WR(sc, BCE_NVM_SW_ARB, BCE_NVM_SW_ARB_ARB_REQ_SET2);
1638         for (j = 0; j < NVRAM_TIMEOUT_COUNT; j++) {
1639                 val = REG_RD(sc, BCE_NVM_SW_ARB);
1640                 if (val & BCE_NVM_SW_ARB_ARB_ARB2)
1641                         break;
1642
1643                 DELAY(5);
1644         }
1645
1646         if (j >= NVRAM_TIMEOUT_COUNT) {
1647                 DBPRINT(sc, BCE_WARN, "Timeout acquiring NVRAM lock!\n");
1648                 rc = EBUSY;
1649         }
1650
1651         DBEXIT(BCE_VERBOSE_NVRAM);
1652         return (rc);
1653 }
1654
1655
1656 /****************************************************************************/
1657 /* Release NVRAM lock.                                                      */
1658 /*                                                                          */
1659 /* When the caller is finished accessing NVRAM the lock must be released.   */
1660 /* Locks 0 and 2 are reserved, lock 1 is used by firmware and lock 2 is     */
1661 /* for use by the driver.                                                   */
1662 /*                                                                          */
1663 /* Returns:                                                                 */
1664 /*   0 on success, positive value on failure.                               */
1665 /****************************************************************************/
1666 static int
1667 bce_release_nvram_lock(struct bce_softc *sc)
1668 {
1669         u32 val;
1670         int j, rc = 0;
1671
1672         DBENTER(BCE_VERBOSE_NVRAM);
1673
1674         /*
1675          * Relinquish nvram interface.
1676          */
1677         REG_WR(sc, BCE_NVM_SW_ARB, BCE_NVM_SW_ARB_ARB_REQ_CLR2);
1678
1679         for (j = 0; j < NVRAM_TIMEOUT_COUNT; j++) {
1680                 val = REG_RD(sc, BCE_NVM_SW_ARB);
1681                 if (!(val & BCE_NVM_SW_ARB_ARB_ARB2))
1682                         break;
1683
1684                 DELAY(5);
1685         }
1686
1687         if (j >= NVRAM_TIMEOUT_COUNT) {
1688                 DBPRINT(sc, BCE_WARN, "Timeout releasing NVRAM lock!\n");
1689                 rc = EBUSY;
1690         }
1691
1692         DBEXIT(BCE_VERBOSE_NVRAM);
1693         return (rc);
1694 }
1695
1696
1697 #ifdef BCE_NVRAM_WRITE_SUPPORT
1698 /****************************************************************************/
1699 /* Enable NVRAM write access.                                               */
1700 /*                                                                          */
1701 /* Before writing to NVRAM the caller must enable NVRAM writes.             */
1702 /*                                                                          */
1703 /* Returns:                                                                 */
1704 /*   0 on success, positive value on failure.                               */
1705 /****************************************************************************/
1706 static int
1707 bce_enable_nvram_write(struct bce_softc *sc)
1708 {
1709         u32 val;
1710         int rc = 0;
1711
1712         DBENTER(BCE_VERBOSE_NVRAM);
1713
1714         val = REG_RD(sc, BCE_MISC_CFG);
1715         REG_WR(sc, BCE_MISC_CFG, val | BCE_MISC_CFG_NVM_WR_EN_PCI);
1716
1717         if (!(sc->bce_flash_info->flags & BCE_NV_BUFFERED)) {
1718                 int j;
1719
1720                 REG_WR(sc, BCE_NVM_COMMAND, BCE_NVM_COMMAND_DONE);
1721                 REG_WR(sc, BCE_NVM_COMMAND,     BCE_NVM_COMMAND_WREN | BCE_NVM_COMMAND_DOIT);
1722
1723                 for (j = 0; j < NVRAM_TIMEOUT_COUNT; j++) {
1724                         DELAY(5);
1725
1726                         val = REG_RD(sc, BCE_NVM_COMMAND);
1727                         if (val & BCE_NVM_COMMAND_DONE)
1728                                 break;
1729                 }
1730
1731                 if (j >= NVRAM_TIMEOUT_COUNT) {
1732                         DBPRINT(sc, BCE_WARN, "Timeout writing NVRAM!\n");
1733                         rc = EBUSY;
1734                 }
1735         }
1736
1737         DBENTER(BCE_VERBOSE_NVRAM);
1738         return (rc);
1739 }
1740
1741
1742 /****************************************************************************/
1743 /* Disable NVRAM write access.                                              */
1744 /*                                                                          */
1745 /* When the caller is finished writing to NVRAM write access must be        */
1746 /* disabled.                                                                */
1747 /*                                                                          */
1748 /* Returns:                                                                 */
1749 /*   Nothing.                                                               */
1750 /****************************************************************************/
1751 static void
1752 bce_disable_nvram_write(struct bce_softc *sc)
1753 {
1754         u32 val;
1755
1756         DBENTER(BCE_VERBOSE_NVRAM);
1757
1758         val = REG_RD(sc, BCE_MISC_CFG);
1759         REG_WR(sc, BCE_MISC_CFG, val & ~BCE_MISC_CFG_NVM_WR_EN);
1760
1761         DBEXIT(BCE_VERBOSE_NVRAM);
1762
1763 }
1764 #endif
1765
1766
1767 /****************************************************************************/
1768 /* Enable NVRAM access.                                                     */
1769 /*                                                                          */
1770 /* Before accessing NVRAM for read or write operations the caller must      */
1771 /* enabled NVRAM access.                                                    */
1772 /*                                                                          */
1773 /* Returns:                                                                 */
1774 /*   Nothing.                                                               */
1775 /****************************************************************************/
1776 static void
1777 bce_enable_nvram_access(struct bce_softc *sc)
1778 {
1779         u32 val;
1780
1781         DBENTER(BCE_VERBOSE_NVRAM);
1782
1783         val = REG_RD(sc, BCE_NVM_ACCESS_ENABLE);
1784         /* Enable both bits, even on read. */
1785         REG_WR(sc, BCE_NVM_ACCESS_ENABLE,
1786                val | BCE_NVM_ACCESS_ENABLE_EN | BCE_NVM_ACCESS_ENABLE_WR_EN);
1787
1788         DBEXIT(BCE_VERBOSE_NVRAM);
1789 }
1790
1791
1792 /****************************************************************************/
1793 /* Disable NVRAM access.                                                    */
1794 /*                                                                          */
1795 /* When the caller is finished accessing NVRAM access must be disabled.     */
1796 /*                                                                          */
1797 /* Returns:                                                                 */
1798 /*   Nothing.                                                               */
1799 /****************************************************************************/
1800 static void
1801 bce_disable_nvram_access(struct bce_softc *sc)
1802 {
1803         u32 val;
1804
1805         DBENTER(BCE_VERBOSE_NVRAM);
1806
1807         val = REG_RD(sc, BCE_NVM_ACCESS_ENABLE);
1808
1809         /* Disable both bits, even after read. */
1810         REG_WR(sc, BCE_NVM_ACCESS_ENABLE,
1811                 val & ~(BCE_NVM_ACCESS_ENABLE_EN |
1812                         BCE_NVM_ACCESS_ENABLE_WR_EN));
1813
1814         DBEXIT(BCE_VERBOSE_NVRAM);
1815 }
1816
1817
1818 #ifdef BCE_NVRAM_WRITE_SUPPORT
1819 /****************************************************************************/
1820 /* Erase NVRAM page before writing.                                         */
1821 /*                                                                          */
1822 /* Non-buffered flash parts require that a page be erased before it is      */
1823 /* written.                                                                 */
1824 /*                                                                          */
1825 /* Returns:                                                                 */
1826 /*   0 on success, positive value on failure.                               */
1827 /****************************************************************************/
1828 static int
1829 bce_nvram_erase_page(struct bce_softc *sc, u32 offset)
1830 {
1831         u32 cmd;
1832         int j, rc = 0;
1833
1834         DBENTER(BCE_VERBOSE_NVRAM);
1835
1836         /* Buffered flash doesn't require an erase. */
1837         if (sc->bce_flash_info->flags & BCE_NV_BUFFERED)
1838                 goto bce_nvram_erase_page_exit;
1839
1840         /* Build an erase command. */
1841         cmd = BCE_NVM_COMMAND_ERASE | BCE_NVM_COMMAND_WR |
1842               BCE_NVM_COMMAND_DOIT;
1843
1844         /*
1845          * Clear the DONE bit separately, set the NVRAM adress to erase,
1846          * and issue the erase command.
1847          */
1848         REG_WR(sc, BCE_NVM_COMMAND, BCE_NVM_COMMAND_DONE);
1849         REG_WR(sc, BCE_NVM_ADDR, offset & BCE_NVM_ADDR_NVM_ADDR_VALUE);
1850         REG_WR(sc, BCE_NVM_COMMAND, cmd);
1851
1852         /* Wait for completion. */
1853         for (j = 0; j < NVRAM_TIMEOUT_COUNT; j++) {
1854                 u32 val;
1855
1856                 DELAY(5);
1857
1858                 val = REG_RD(sc, BCE_NVM_COMMAND);
1859                 if (val & BCE_NVM_COMMAND_DONE)
1860                         break;
1861         }
1862
1863         if (j >= NVRAM_TIMEOUT_COUNT) {
1864                 DBPRINT(sc, BCE_WARN, "Timeout erasing NVRAM.\n");
1865                 rc = EBUSY;
1866         }
1867
1868 bce_nvram_erase_page_exit:
1869         DBEXIT(BCE_VERBOSE_NVRAM);
1870         return (rc);
1871 }
1872 #endif /* BCE_NVRAM_WRITE_SUPPORT */
1873
1874
1875 /****************************************************************************/
1876 /* Read a dword (32 bits) from NVRAM.                                       */
1877 /*                                                                          */
1878 /* Read a 32 bit word from NVRAM.  The caller is assumed to have already    */
1879 /* obtained the NVRAM lock and enabled the controller for NVRAM access.     */
1880 /*                                                                          */
1881 /* Returns:                                                                 */
1882 /*   0 on success and the 32 bit value read, positive value on failure.     */
1883 /****************************************************************************/
1884 static int
1885 bce_nvram_read_dword(struct bce_softc *sc, u32 offset, u8 *ret_val,
1886                                                         u32 cmd_flags)
1887 {
1888         u32 cmd;
1889         int i, rc = 0;
1890
1891         DBENTER(BCE_EXTREME_NVRAM);
1892
1893         /* Build the command word. */
1894         cmd = BCE_NVM_COMMAND_DOIT | cmd_flags;
1895
1896         /* Calculate the offset for buffered flash if translation is used. */
1897         if (sc->bce_flash_info->flags & BCE_NV_TRANSLATE) {
1898                 offset = ((offset / sc->bce_flash_info->page_size) <<
1899                            sc->bce_flash_info->page_bits) +
1900                           (offset % sc->bce_flash_info->page_size);
1901         }
1902
1903         /*
1904          * Clear the DONE bit separately, set the address to read,
1905          * and issue the read.
1906          */
1907         REG_WR(sc, BCE_NVM_COMMAND, BCE_NVM_COMMAND_DONE);
1908         REG_WR(sc, BCE_NVM_ADDR, offset & BCE_NVM_ADDR_NVM_ADDR_VALUE);
1909         REG_WR(sc, BCE_NVM_COMMAND, cmd);
1910
1911         /* Wait for completion. */
1912         for (i = 0; i < NVRAM_TIMEOUT_COUNT; i++) {
1913                 u32 val;
1914
1915                 DELAY(5);
1916
1917                 val = REG_RD(sc, BCE_NVM_COMMAND);
1918                 if (val & BCE_NVM_COMMAND_DONE) {
1919                         val = REG_RD(sc, BCE_NVM_READ);
1920
1921                         val = bce_be32toh(val);
1922                         memcpy(ret_val, &val, 4);
1923                         break;
1924                 }
1925         }
1926
1927         /* Check for errors. */
1928         if (i >= NVRAM_TIMEOUT_COUNT) {
1929                 BCE_PRINTF("%s(%d): Timeout error reading NVRAM at offset 0x%08X!\n",
1930                         __FILE__, __LINE__, offset);
1931                 rc = EBUSY;
1932         }
1933
1934         DBEXIT(BCE_EXTREME_NVRAM);
1935         return(rc);
1936 }
1937
1938
1939 #ifdef BCE_NVRAM_WRITE_SUPPORT
1940 /****************************************************************************/
1941 /* Write a dword (32 bits) to NVRAM.                                        */
1942 /*                                                                          */
1943 /* Write a 32 bit word to NVRAM.  The caller is assumed to have already     */
1944 /* obtained the NVRAM lock, enabled the controller for NVRAM access, and    */
1945 /* enabled NVRAM write access.                                              */
1946 /*                                                                          */
1947 /* Returns:                                                                 */
1948 /*   0 on success, positive value on failure.                               */
1949 /****************************************************************************/
1950 static int
1951 bce_nvram_write_dword(struct bce_softc *sc, u32 offset, u8 *val,
1952         u32 cmd_flags)
1953 {
1954         u32 cmd, val32;
1955         int j, rc = 0;
1956
1957         DBENTER(BCE_VERBOSE_NVRAM);
1958
1959         /* Build the command word. */
1960         cmd = BCE_NVM_COMMAND_DOIT | BCE_NVM_COMMAND_WR | cmd_flags;
1961
1962         /* Calculate the offset for buffered flash if translation is used. */
1963         if (sc->bce_flash_info->flags & BCE_NV_TRANSLATE) {
1964                 offset = ((offset / sc->bce_flash_info->page_size) <<
1965                           sc->bce_flash_info->page_bits) +
1966                          (offset % sc->bce_flash_info->page_size);
1967         }
1968
1969         /*
1970          * Clear the DONE bit separately, convert NVRAM data to big-endian,
1971          * set the NVRAM address to write, and issue the write command
1972          */
1973         REG_WR(sc, BCE_NVM_COMMAND, BCE_NVM_COMMAND_DONE);
1974         memcpy(&val32, val, 4);
1975         val32 = htobe32(val32);
1976         REG_WR(sc, BCE_NVM_WRITE, val32);
1977         REG_WR(sc, BCE_NVM_ADDR, offset & BCE_NVM_ADDR_NVM_ADDR_VALUE);
1978         REG_WR(sc, BCE_NVM_COMMAND, cmd);
1979
1980         /* Wait for completion. */
1981         for (j = 0; j < NVRAM_TIMEOUT_COUNT; j++) {
1982                 DELAY(5);
1983
1984                 if (REG_RD(sc, BCE_NVM_COMMAND) & BCE_NVM_COMMAND_DONE)
1985                         break;
1986         }
1987         if (j >= NVRAM_TIMEOUT_COUNT) {
1988                 BCE_PRINTF("%s(%d): Timeout error writing NVRAM at offset 0x%08X\n",
1989                         __FILE__, __LINE__, offset);
1990                 rc = EBUSY;
1991         }
1992
1993         DBEXIT(BCE_VERBOSE_NVRAM);
1994         return (rc);
1995 }
1996 #endif /* BCE_NVRAM_WRITE_SUPPORT */
1997
1998
1999 /****************************************************************************/
2000 /* Initialize NVRAM access.                                                 */
2001 /*                                                                          */
2002 /* Identify the NVRAM device in use and prepare the NVRAM interface to      */
2003 /* access that device.                                                      */
2004 /*                                                                          */
2005 /* Returns:                                                                 */
2006 /*   0 on success, positive value on failure.                               */
2007 /****************************************************************************/
2008 static int
2009 bce_init_nvram(struct bce_softc *sc)
2010 {
2011         u32 val;
2012         int j, entry_count, rc = 0;
2013         struct flash_spec *flash;
2014
2015         DBENTER(BCE_VERBOSE_NVRAM);
2016
2017         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
2018                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
2019                 sc->bce_flash_info = &flash_5709;
2020                 goto bce_init_nvram_get_flash_size;
2021         }
2022
2023         /* Determine the selected interface. */
2024         val = REG_RD(sc, BCE_NVM_CFG1);
2025
2026         entry_count = sizeof(flash_table) / sizeof(struct flash_spec);
2027
2028         /*
2029          * Flash reconfiguration is required to support additional
2030          * NVRAM devices not directly supported in hardware.
2031          * Check if the flash interface was reconfigured
2032          * by the bootcode.
2033          */
2034
2035         if (val & 0x40000000) {
2036                 /* Flash interface reconfigured by bootcode. */
2037
2038                 DBPRINT(sc,BCE_INFO_LOAD,
2039                         "bce_init_nvram(): Flash WAS reconfigured.\n");
2040
2041                 for (j = 0, flash = &flash_table[0]; j < entry_count;
2042                      j++, flash++) {
2043                         if ((val & FLASH_BACKUP_STRAP_MASK) ==
2044                             (flash->config1 & FLASH_BACKUP_STRAP_MASK)) {
2045                                 sc->bce_flash_info = flash;
2046                                 break;
2047                         }
2048                 }
2049         } else {
2050                 /* Flash interface not yet reconfigured. */
2051                 u32 mask;
2052
2053                 DBPRINT(sc, BCE_INFO_LOAD, "%s(): Flash was NOT reconfigured.\n",
2054                         __FUNCTION__);
2055
2056                 if (val & (1 << 23))
2057                         mask = FLASH_BACKUP_STRAP_MASK;
2058                 else
2059                         mask = FLASH_STRAP_MASK;
2060
2061                 /* Look for the matching NVRAM device configuration data. */
2062                 for (j = 0, flash = &flash_table[0]; j < entry_count; j++, flash++) {
2063
2064                         /* Check if the device matches any of the known devices. */
2065                         if ((val & mask) == (flash->strapping & mask)) {
2066                                 /* Found a device match. */
2067                                 sc->bce_flash_info = flash;
2068
2069                                 /* Request access to the flash interface. */
2070                                 if ((rc = bce_acquire_nvram_lock(sc)) != 0)
2071                                         return rc;
2072
2073                                 /* Reconfigure the flash interface. */
2074                                 bce_enable_nvram_access(sc);
2075                                 REG_WR(sc, BCE_NVM_CFG1, flash->config1);
2076                                 REG_WR(sc, BCE_NVM_CFG2, flash->config2);
2077                                 REG_WR(sc, BCE_NVM_CFG3, flash->config3);
2078                                 REG_WR(sc, BCE_NVM_WRITE1, flash->write1);
2079                                 bce_disable_nvram_access(sc);
2080                                 bce_release_nvram_lock(sc);
2081
2082                                 break;
2083                         }
2084                 }
2085         }
2086
2087         /* Check if a matching device was found. */
2088         if (j == entry_count) {
2089                 sc->bce_flash_info = NULL;
2090                 BCE_PRINTF("%s(%d): Unknown Flash NVRAM found!\n",
2091                         __FILE__, __LINE__);
2092                 rc = ENODEV;
2093         }
2094
2095 bce_init_nvram_get_flash_size:
2096         /* Write the flash config data to the shared memory interface. */
2097         val = REG_RD_IND(sc, sc->bce_shmem_base + BCE_SHARED_HW_CFG_CONFIG2);
2098         val &= BCE_SHARED_HW_CFG2_NVM_SIZE_MASK;
2099         if (val)
2100                 sc->bce_flash_size = val;
2101         else
2102                 sc->bce_flash_size = sc->bce_flash_info->total_size;
2103
2104         DBPRINT(sc, BCE_INFO_LOAD, "%s(): Found %s, size = 0x%08X\n",
2105                 __FUNCTION__, sc->bce_flash_info->name,
2106                 sc->bce_flash_info->total_size);
2107
2108         DBEXIT(BCE_VERBOSE_NVRAM);
2109         return rc;
2110 }
2111
2112
2113 /****************************************************************************/
2114 /* Read an arbitrary range of data from NVRAM.                              */
2115 /*                                                                          */
2116 /* Prepares the NVRAM interface for access and reads the requested data     */
2117 /* into the supplied buffer.                                                */
2118 /*                                                                          */
2119 /* Returns:                                                                 */
2120 /*   0 on success and the data read, positive value on failure.             */
2121 /****************************************************************************/
2122 static int
2123 bce_nvram_read(struct bce_softc *sc, u32 offset, u8 *ret_buf,
2124         int buf_size)
2125 {
2126         int rc = 0;
2127         u32 cmd_flags, offset32, len32, extra;
2128
2129         DBENTER(BCE_VERBOSE_NVRAM);
2130
2131         if (buf_size == 0)
2132                 goto bce_nvram_read_exit;
2133
2134         /* Request access to the flash interface. */
2135         if ((rc = bce_acquire_nvram_lock(sc)) != 0)
2136                 goto bce_nvram_read_exit;
2137
2138         /* Enable access to flash interface */
2139         bce_enable_nvram_access(sc);
2140
2141         len32 = buf_size;
2142         offset32 = offset;
2143         extra = 0;
2144
2145         cmd_flags = 0;
2146
2147         if (offset32 & 3) {
2148                 u8 buf[4];
2149                 u32 pre_len;
2150
2151                 offset32 &= ~3;
2152                 pre_len = 4 - (offset & 3);
2153
2154                 if (pre_len >= len32) {
2155                         pre_len = len32;
2156                         cmd_flags = BCE_NVM_COMMAND_FIRST | BCE_NVM_COMMAND_LAST;
2157                 }
2158                 else {
2159                         cmd_flags = BCE_NVM_COMMAND_FIRST;
2160                 }
2161
2162                 rc = bce_nvram_read_dword(sc, offset32, buf, cmd_flags);
2163
2164                 if (rc)
2165                         return rc;
2166
2167                 memcpy(ret_buf, buf + (offset & 3), pre_len);
2168
2169                 offset32 += 4;
2170                 ret_buf += pre_len;
2171                 len32 -= pre_len;
2172         }
2173
2174         if (len32 & 3) {
2175                 extra = 4 - (len32 & 3);
2176                 len32 = (len32 + 4) & ~3;
2177         }
2178
2179         if (len32 == 4) {
2180                 u8 buf[4];
2181
2182                 if (cmd_flags)
2183                         cmd_flags = BCE_NVM_COMMAND_LAST;
2184                 else
2185                         cmd_flags = BCE_NVM_COMMAND_FIRST |
2186                                     BCE_NVM_COMMAND_LAST;
2187
2188                 rc = bce_nvram_read_dword(sc, offset32, buf, cmd_flags);
2189
2190                 memcpy(ret_buf, buf, 4 - extra);
2191         }
2192         else if (len32 > 0) {
2193                 u8 buf[4];
2194
2195                 /* Read the first word. */
2196                 if (cmd_flags)
2197                         cmd_flags = 0;
2198                 else
2199                         cmd_flags = BCE_NVM_COMMAND_FIRST;
2200
2201                 rc = bce_nvram_read_dword(sc, offset32, ret_buf, cmd_flags);
2202
2203                 /* Advance to the next dword. */
2204                 offset32 += 4;
2205                 ret_buf += 4;
2206                 len32 -= 4;
2207
2208                 while (len32 > 4 && rc == 0) {
2209                         rc = bce_nvram_read_dword(sc, offset32, ret_buf, 0);
2210
2211                         /* Advance to the next dword. */
2212                         offset32 += 4;
2213                         ret_buf += 4;
2214                         len32 -= 4;
2215                 }
2216
2217                 if (rc)
2218                         goto bce_nvram_read_locked_exit;
2219
2220                 cmd_flags = BCE_NVM_COMMAND_LAST;
2221                 rc = bce_nvram_read_dword(sc, offset32, buf, cmd_flags);
2222
2223                 memcpy(ret_buf, buf, 4 - extra);
2224         }
2225
2226 bce_nvram_read_locked_exit:
2227         /* Disable access to flash interface and release the lock. */
2228         bce_disable_nvram_access(sc);
2229         bce_release_nvram_lock(sc);
2230
2231 bce_nvram_read_exit:
2232         DBEXIT(BCE_VERBOSE_NVRAM);
2233         return rc;
2234 }
2235
2236
2237 #ifdef BCE_NVRAM_WRITE_SUPPORT
2238 /****************************************************************************/
2239 /* Write an arbitrary range of data from NVRAM.                             */
2240 /*                                                                          */
2241 /* Prepares the NVRAM interface for write access and writes the requested   */
2242 /* data from the supplied buffer.  The caller is responsible for            */
2243 /* calculating any appropriate CRCs.                                        */
2244 /*                                                                          */
2245 /* Returns:                                                                 */
2246 /*   0 on success, positive value on failure.                               */
2247 /****************************************************************************/
2248 static int
2249 bce_nvram_write(struct bce_softc *sc, u32 offset, u8 *data_buf,
2250         int buf_size)
2251 {
2252         u32 written, offset32, len32;
2253         u8 *buf, start[4], end[4];
2254         int rc = 0;
2255         int align_start, align_end;
2256
2257         DBENTER(BCE_VERBOSE_NVRAM);
2258
2259         buf = data_buf;
2260         offset32 = offset;
2261         len32 = buf_size;
2262         align_start = align_end = 0;
2263
2264         if ((align_start = (offset32 & 3))) {
2265                 offset32 &= ~3;
2266                 len32 += align_start;
2267                 if ((rc = bce_nvram_read(sc, offset32, start, 4)))
2268                         goto bce_nvram_write_exit;
2269         }
2270
2271         if (len32 & 3) {
2272                 if ((len32 > 4) || !align_start) {
2273                         align_end = 4 - (len32 & 3);
2274                         len32 += align_end;
2275                         if ((rc = bce_nvram_read(sc, offset32 + len32 - 4,
2276                                 end, 4))) {
2277                                 goto bce_nvram_write_exit;
2278                         }
2279                 }
2280         }
2281
2282         if (align_start || align_end) {
2283                 buf = malloc(len32, M_DEVBUF, M_NOWAIT);
2284                 if (buf == 0) {
2285                         rc = ENOMEM;
2286                         goto bce_nvram_write_exit;
2287                 }
2288
2289                 if (align_start) {
2290                         memcpy(buf, start, 4);
2291                 }
2292
2293                 if (align_end) {
2294                         memcpy(buf + len32 - 4, end, 4);
2295                 }
2296                 memcpy(buf + align_start, data_buf, buf_size);
2297         }
2298
2299         written = 0;
2300         while ((written < len32) && (rc == 0)) {
2301                 u32 page_start, page_end, data_start, data_end;
2302                 u32 addr, cmd_flags;
2303                 int i;
2304                 u8 flash_buffer[264];
2305
2306             /* Find the page_start addr */
2307                 page_start = offset32 + written;
2308                 page_start -= (page_start % sc->bce_flash_info->page_size);
2309                 /* Find the page_end addr */
2310                 page_end = page_start + sc->bce_flash_info->page_size;
2311                 /* Find the data_start addr */
2312                 data_start = (written == 0) ? offset32 : page_start;
2313                 /* Find the data_end addr */
2314                 data_end = (page_end > offset32 + len32) ?
2315                         (offset32 + len32) : page_end;
2316
2317                 /* Request access to the flash interface. */
2318                 if ((rc = bce_acquire_nvram_lock(sc)) != 0)
2319                         goto bce_nvram_write_exit;
2320
2321                 /* Enable access to flash interface */
2322                 bce_enable_nvram_access(sc);
2323
2324                 cmd_flags = BCE_NVM_COMMAND_FIRST;
2325                 if (!(sc->bce_flash_info->flags & BCE_NV_BUFFERED)) {
2326                         int j;
2327
2328                         /* Read the whole page into the buffer
2329                          * (non-buffer flash only) */
2330                         for (j = 0; j < sc->bce_flash_info->page_size; j += 4) {
2331                                 if (j == (sc->bce_flash_info->page_size - 4)) {
2332                                         cmd_flags |= BCE_NVM_COMMAND_LAST;
2333                                 }
2334                                 rc = bce_nvram_read_dword(sc,
2335                                         page_start + j,
2336                                         &flash_buffer[j],
2337                                         cmd_flags);
2338
2339                                 if (rc)
2340                                         goto bce_nvram_write_locked_exit;
2341
2342                                 cmd_flags = 0;
2343                         }
2344                 }
2345
2346                 /* Enable writes to flash interface (unlock write-protect) */
2347                 if ((rc = bce_enable_nvram_write(sc)) != 0)
2348                         goto bce_nvram_write_locked_exit;
2349
2350                 /* Erase the page */
2351                 if ((rc = bce_nvram_erase_page(sc, page_start)) != 0)
2352                         goto bce_nvram_write_locked_exit;
2353
2354                 /* Re-enable the write again for the actual write */
2355                 bce_enable_nvram_write(sc);
2356
2357                 /* Loop to write back the buffer data from page_start to
2358                  * data_start */
2359                 i = 0;
2360                 if (!(sc->bce_flash_info->flags & BCE_NV_BUFFERED)) {
2361                         for (addr = page_start; addr < data_start;
2362                                 addr += 4, i += 4) {
2363
2364                                 rc = bce_nvram_write_dword(sc, addr,
2365                                         &flash_buffer[i], cmd_flags);
2366
2367                                 if (rc != 0)
2368                                         goto bce_nvram_write_locked_exit;
2369
2370                                 cmd_flags = 0;
2371                         }
2372                 }
2373
2374                 /* Loop to write the new data from data_start to data_end */
2375                 for (addr = data_start; addr < data_end; addr += 4, i++) {
2376                         if ((addr == page_end - 4) ||
2377                                 ((sc->bce_flash_info->flags & BCE_NV_BUFFERED) &&
2378                                 (addr == data_end - 4))) {
2379
2380                                 cmd_flags |= BCE_NVM_COMMAND_LAST;
2381                         }
2382                         rc = bce_nvram_write_dword(sc, addr, buf,
2383                                 cmd_flags);
2384
2385                         if (rc != 0)
2386                                 goto bce_nvram_write_locked_exit;
2387
2388                         cmd_flags = 0;
2389                         buf += 4;
2390                 }
2391
2392                 /* Loop to write back the buffer data from data_end
2393                  * to page_end */
2394                 if (!(sc->bce_flash_info->flags & BCE_NV_BUFFERED)) {
2395                         for (addr = data_end; addr < page_end;
2396                                 addr += 4, i += 4) {
2397
2398                                 if (addr == page_end-4) {
2399                                         cmd_flags = BCE_NVM_COMMAND_LAST;
2400                                 }
2401                                 rc = bce_nvram_write_dword(sc, addr,
2402                                         &flash_buffer[i], cmd_flags);
2403
2404                                 if (rc != 0)
2405                                         goto bce_nvram_write_locked_exit;
2406
2407                                 cmd_flags = 0;
2408                         }
2409                 }
2410
2411                 /* Disable writes to flash interface (lock write-protect) */
2412                 bce_disable_nvram_write(sc);
2413
2414                 /* Disable access to flash interface */
2415                 bce_disable_nvram_access(sc);
2416                 bce_release_nvram_lock(sc);
2417
2418                 /* Increment written */
2419                 written += data_end - data_start;
2420         }
2421
2422         goto bce_nvram_write_exit;
2423
2424 bce_nvram_write_locked_exit:
2425                 bce_disable_nvram_write(sc);
2426                 bce_disable_nvram_access(sc);
2427                 bce_release_nvram_lock(sc);
2428
2429 bce_nvram_write_exit:
2430         if (align_start || align_end)
2431                 free(buf, M_DEVBUF);
2432
2433         DBEXIT(BCE_VERBOSE_NVRAM);
2434         return (rc);
2435 }
2436 #endif /* BCE_NVRAM_WRITE_SUPPORT */
2437
2438
2439 /****************************************************************************/
2440 /* Verifies that NVRAM is accessible and contains valid data.               */
2441 /*                                                                          */
2442 /* Reads the configuration data from NVRAM and verifies that the CRC is     */
2443 /* correct.                                                                 */
2444 /*                                                                          */
2445 /* Returns:                                                                 */
2446 /*   0 on success, positive value on failure.                               */
2447 /****************************************************************************/
2448 static int
2449 bce_nvram_test(struct bce_softc *sc)
2450 {
2451         u32 buf[BCE_NVRAM_SIZE / 4];
2452         u8 *data = (u8 *) buf;
2453         int rc = 0;
2454         u32 magic, csum;
2455
2456         DBENTER(BCE_VERBOSE_NVRAM | BCE_VERBOSE_LOAD | BCE_VERBOSE_RESET);
2457
2458         /*
2459          * Check that the device NVRAM is valid by reading
2460          * the magic value at offset 0.
2461          */
2462         if ((rc = bce_nvram_read(sc, 0, data, 4)) != 0) {
2463                 BCE_PRINTF("%s(%d): Unable to read NVRAM!\n", __FILE__, __LINE__);
2464                 goto bce_nvram_test_exit;
2465         }
2466
2467         /*
2468          * Verify that offset 0 of the NVRAM contains
2469          * a valid magic number.
2470          */
2471     magic = bce_be32toh(buf[0]);
2472         if (magic != BCE_NVRAM_MAGIC) {
2473                 rc = ENODEV;
2474                 BCE_PRINTF("%s(%d): Invalid NVRAM magic value! Expected: 0x%08X, "
2475                         "Found: 0x%08X\n",
2476                         __FILE__, __LINE__, BCE_NVRAM_MAGIC, magic);
2477                 goto bce_nvram_test_exit;
2478         }
2479
2480         /*
2481          * Verify that the device NVRAM includes valid
2482          * configuration data.
2483          */
2484         if ((rc = bce_nvram_read(sc, 0x100, data, BCE_NVRAM_SIZE)) != 0) {
2485                 BCE_PRINTF("%s(%d): Unable to read Manufacturing Information from "
2486                         "NVRAM!\n", __FILE__, __LINE__);
2487                 goto bce_nvram_test_exit;
2488         }
2489
2490         csum = ether_crc32_le(data, 0x100);
2491         if (csum != BCE_CRC32_RESIDUAL) {
2492                 rc = ENODEV;
2493                 BCE_PRINTF("%s(%d): Invalid Manufacturing Information NVRAM CRC! "
2494                         "Expected: 0x%08X, Found: 0x%08X\n",
2495                         __FILE__, __LINE__, BCE_CRC32_RESIDUAL, csum);
2496                 goto bce_nvram_test_exit;
2497         }
2498
2499         csum = ether_crc32_le(data + 0x100, 0x100);
2500         if (csum != BCE_CRC32_RESIDUAL) {
2501                 rc = ENODEV;
2502                 BCE_PRINTF("%s(%d): Invalid Feature Configuration Information "
2503                         "NVRAM CRC! Expected: 0x%08X, Found: 08%08X\n",
2504                         __FILE__, __LINE__, BCE_CRC32_RESIDUAL, csum);
2505         }
2506
2507 bce_nvram_test_exit:
2508         DBEXIT(BCE_VERBOSE_NVRAM | BCE_VERBOSE_LOAD | BCE_VERBOSE_RESET);
2509         return rc;
2510 }
2511
2512
2513 /****************************************************************************/
2514 /* Identifies the current media type of the controller and sets the PHY     */
2515 /* address.                                                                 */
2516 /*                                                                          */
2517 /* Returns:                                                                 */
2518 /*   Nothing.                                                               */
2519 /****************************************************************************/
2520 static void
2521 bce_get_media(struct bce_softc *sc)
2522 {
2523         u32 val;
2524
2525         DBENTER(BCE_VERBOSE);
2526
2527         /* Assume PHY address for copper controllers. */
2528         sc->bce_phy_addr = 1;
2529
2530         if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) {
2531                 u32 val = REG_RD(sc, BCE_MISC_DUAL_MEDIA_CTRL);
2532                 u32 bond_id = val & BCE_MISC_DUAL_MEDIA_CTRL_BOND_ID;
2533                 u32 strap;
2534
2535                 /*
2536                  * The BCM5709S is software configurable
2537                  * for Copper or SerDes operation.
2538                  */
2539                 if (bond_id == BCE_MISC_DUAL_MEDIA_CTRL_BOND_ID_C) {
2540                         DBPRINT(sc, BCE_INFO_LOAD, "5709 bonded for copper.\n");
2541                         goto bce_get_media_exit;
2542                 } else if (bond_id == BCE_MISC_DUAL_MEDIA_CTRL_BOND_ID_S) {
2543                         DBPRINT(sc, BCE_INFO_LOAD, "5709 bonded for dual media.\n");
2544                         sc->bce_phy_flags |= BCE_PHY_SERDES_FLAG;
2545                         goto bce_get_media_exit;
2546                 }
2547
2548                 if (val & BCE_MISC_DUAL_MEDIA_CTRL_STRAP_OVERRIDE)
2549                         strap = (val & BCE_MISC_DUAL_MEDIA_CTRL_PHY_CTRL) >> 21;
2550                 else
2551                         strap = (val & BCE_MISC_DUAL_MEDIA_CTRL_PHY_CTRL_STRAP) >> 8;
2552
2553                 if (pci_get_function(sc->bce_dev) == 0) {
2554                         switch (strap) {
2555                         case 0x4:
2556                         case 0x5:
2557                         case 0x6:
2558                                 DBPRINT(sc, BCE_INFO_LOAD,
2559                                         "BCM5709 s/w configured for SerDes.\n");
2560                                 sc->bce_phy_flags |= BCE_PHY_SERDES_FLAG;
2561                         default:
2562                                 DBPRINT(sc, BCE_INFO_LOAD,
2563                                         "BCM5709 s/w configured for Copper.\n");
2564                         }
2565                 } else {
2566                         switch (strap) {
2567                         case 0x1:
2568                         case 0x2:
2569                         case 0x4:
2570                                 DBPRINT(sc, BCE_INFO_LOAD,
2571                                         "BCM5709 s/w configured for SerDes.\n");
2572                                 sc->bce_phy_flags |= BCE_PHY_SERDES_FLAG;
2573                         default:
2574                                 DBPRINT(sc, BCE_INFO_LOAD,
2575                                         "BCM5709 s/w configured for Copper.\n");
2576                         }
2577                 }
2578
2579         } else if (BCE_CHIP_BOND_ID(sc) & BCE_CHIP_BOND_ID_SERDES_BIT)
2580                 sc->bce_phy_flags |= BCE_PHY_SERDES_FLAG;
2581
2582         if (sc->bce_phy_flags & BCE_PHY_SERDES_FLAG) {
2583                 sc->bce_flags |= BCE_NO_WOL_FLAG;
2584                 if (BCE_CHIP_NUM(sc) != BCE_CHIP_NUM_5706) {
2585                         sc->bce_phy_addr = 2;
2586                         val = REG_RD_IND(sc, sc->bce_shmem_base +
2587                                  BCE_SHARED_HW_CFG_CONFIG);
2588                         if (val & BCE_SHARED_HW_CFG_PHY_2_5G) {
2589                                 sc->bce_phy_flags |= BCE_PHY_2_5G_CAPABLE_FLAG;
2590                                 DBPRINT(sc, BCE_INFO_LOAD, "Found 2.5Gb capable adapter\n");
2591                         }
2592                 }
2593         } else if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5706) ||
2594                    (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5708))
2595                 sc->bce_phy_flags |= BCE_PHY_CRC_FIX_FLAG;
2596
2597 bce_get_media_exit:
2598         DBPRINT(sc, (BCE_INFO_LOAD | BCE_INFO_PHY),
2599                 "Using PHY address %d.\n", sc->bce_phy_addr);
2600
2601         DBEXIT(BCE_VERBOSE);
2602 }
2603
2604
2605 /****************************************************************************/
2606 /* Free any DMA memory owned by the driver.                                 */
2607 /*                                                                          */
2608 /* Scans through each data structre that requires DMA memory and frees      */
2609 /* the memory if allocated.                                                 */
2610 /*                                                                          */
2611 /* Returns:                                                                 */
2612 /*   Nothing.                                                               */
2613 /****************************************************************************/
2614 static void
2615 bce_dma_free(struct bce_softc *sc)
2616 {
2617         int i;
2618
2619         DBENTER(BCE_VERBOSE_RESET | BCE_VERBOSE_UNLOAD | BCE_VERBOSE_CTX);
2620
2621         /* Free, unmap, and destroy the status block. */
2622         if (sc->status_block != NULL) {
2623                 bus_dmamem_free(
2624                         sc->status_tag,
2625                     sc->status_block,
2626                     sc->status_map);
2627                 sc->status_block = NULL;
2628         }
2629
2630         if (sc->status_map != NULL) {
2631                 bus_dmamap_unload(
2632                         sc->status_tag,
2633                     sc->status_map);
2634                 bus_dmamap_destroy(sc->status_tag,
2635                     sc->status_map);
2636                 sc->status_map = NULL;
2637         }
2638
2639         if (sc->status_tag != NULL) {
2640                 bus_dma_tag_destroy(sc->status_tag);
2641                 sc->status_tag = NULL;
2642         }
2643
2644
2645         /* Free, unmap, and destroy the statistics block. */
2646         if (sc->stats_block != NULL) {
2647                 bus_dmamem_free(
2648                         sc->stats_tag,
2649                     sc->stats_block,
2650                     sc->stats_map);
2651                 sc->stats_block = NULL;
2652         }
2653
2654         if (sc->stats_map != NULL) {
2655                 bus_dmamap_unload(
2656                         sc->stats_tag,
2657                     sc->stats_map);
2658                 bus_dmamap_destroy(sc->stats_tag,
2659                     sc->stats_map);
2660                 sc->stats_map = NULL;
2661         }
2662
2663         if (sc->stats_tag != NULL) {
2664                 bus_dma_tag_destroy(sc->stats_tag);
2665                 sc->stats_tag = NULL;
2666         }
2667
2668
2669         /* Free, unmap and destroy all context memory pages. */
2670         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
2671                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
2672                 for (i = 0; i < sc->ctx_pages; i++ ) {
2673                         if (sc->ctx_block[i] != NULL) {
2674                                 bus_dmamem_free(
2675                                         sc->ctx_tag,
2676                                     sc->ctx_block[i],
2677                                     sc->ctx_map[i]);
2678                                 sc->ctx_block[i] = NULL;
2679                         }
2680
2681                         if (sc->ctx_map[i] != NULL) {
2682                                 bus_dmamap_unload(
2683                                         sc->ctx_tag,
2684                                 sc->ctx_map[i]);
2685                                 bus_dmamap_destroy(
2686                                         sc->ctx_tag,
2687                                     sc->ctx_map[i]);
2688                                 sc->ctx_map[i] = NULL;
2689                         }
2690                 }
2691
2692                 /* Destroy the context memory tag. */
2693                 if (sc->ctx_tag != NULL) {
2694                         bus_dma_tag_destroy(sc->ctx_tag);
2695                         sc->ctx_tag = NULL;
2696                 }
2697         }
2698
2699
2700         /* Free, unmap and destroy all TX buffer descriptor chain pages. */
2701         for (i = 0; i < TX_PAGES; i++ ) {
2702                 if (sc->tx_bd_chain[i] != NULL) {
2703                         bus_dmamem_free(
2704                                 sc->tx_bd_chain_tag,
2705                             sc->tx_bd_chain[i],
2706                             sc->tx_bd_chain_map[i]);
2707                         sc->tx_bd_chain[i] = NULL;
2708                 }
2709
2710                 if (sc->tx_bd_chain_map[i] != NULL) {
2711                         bus_dmamap_unload(
2712                                 sc->tx_bd_chain_tag,
2713                         sc->tx_bd_chain_map[i]);
2714                         bus_dmamap_destroy(
2715                                 sc->tx_bd_chain_tag,
2716                             sc->tx_bd_chain_map[i]);
2717                         sc->tx_bd_chain_map[i] = NULL;
2718                 }
2719         }
2720
2721         /* Destroy the TX buffer descriptor tag. */
2722         if (sc->tx_bd_chain_tag != NULL) {
2723                 bus_dma_tag_destroy(sc->tx_bd_chain_tag);
2724                 sc->tx_bd_chain_tag = NULL;
2725         }
2726
2727
2728         /* Free, unmap and destroy all RX buffer descriptor chain pages. */
2729         for (i = 0; i < RX_PAGES; i++ ) {
2730                 if (sc->rx_bd_chain[i] != NULL) {
2731                         bus_dmamem_free(
2732                                 sc->rx_bd_chain_tag,
2733                             sc->rx_bd_chain[i],
2734                             sc->rx_bd_chain_map[i]);
2735                         sc->rx_bd_chain[i] = NULL;
2736                 }
2737
2738                 if (sc->rx_bd_chain_map[i] != NULL) {
2739                         bus_dmamap_unload(
2740                                 sc->rx_bd_chain_tag,
2741                         sc->rx_bd_chain_map[i]);
2742                         bus_dmamap_destroy(
2743                                 sc->rx_bd_chain_tag,
2744                             sc->rx_bd_chain_map[i]);
2745                         sc->rx_bd_chain_map[i] = NULL;
2746                 }
2747         }
2748
2749         /* Destroy the RX buffer descriptor tag. */
2750         if (sc->rx_bd_chain_tag != NULL) {
2751                 bus_dma_tag_destroy(sc->rx_bd_chain_tag);
2752                 sc->rx_bd_chain_tag = NULL;
2753         }
2754
2755
2756 #ifdef ZERO_COPY_SOCKETS
2757         /* Free, unmap and destroy all page buffer descriptor chain pages. */
2758         for (i = 0; i < PG_PAGES; i++ ) {
2759                 if (sc->pg_bd_chain[i] != NULL) {
2760                         bus_dmamem_free(
2761                                 sc->pg_bd_chain_tag,
2762                             sc->pg_bd_chain[i],
2763                             sc->pg_bd_chain_map[i]);
2764                         sc->pg_bd_chain[i] = NULL;
2765                 }
2766
2767                 if (sc->pg_bd_chain_map[i] != NULL) {
2768                         bus_dmamap_unload(
2769                                 sc->pg_bd_chain_tag,
2770                         sc->pg_bd_chain_map[i]);
2771                         bus_dmamap_destroy(
2772                                 sc->pg_bd_chain_tag,
2773                             sc->pg_bd_chain_map[i]);
2774                         sc->pg_bd_chain_map[i] = NULL;
2775                 }
2776         }
2777
2778         /* Destroy the page buffer descriptor tag. */
2779         if (sc->pg_bd_chain_tag != NULL) {
2780                 bus_dma_tag_destroy(sc->pg_bd_chain_tag);
2781                 sc->pg_bd_chain_tag = NULL;
2782         }
2783 #endif
2784
2785
2786         /* Unload and destroy the TX mbuf maps. */
2787         for (i = 0; i < TOTAL_TX_BD; i++) {
2788                 if (sc->tx_mbuf_map[i] != NULL) {
2789                         bus_dmamap_unload(sc->tx_mbuf_tag,
2790                                 sc->tx_mbuf_map[i]);
2791                         bus_dmamap_destroy(sc->tx_mbuf_tag,
2792                                 sc->tx_mbuf_map[i]);
2793                         sc->tx_mbuf_map[i] = NULL;
2794                 }
2795         }
2796
2797         /* Destroy the TX mbuf tag. */
2798         if (sc->tx_mbuf_tag != NULL) {
2799                 bus_dma_tag_destroy(sc->tx_mbuf_tag);
2800                 sc->tx_mbuf_tag = NULL;
2801         }
2802
2803         /* Unload and destroy the RX mbuf maps. */
2804         for (i = 0; i < TOTAL_RX_BD; i++) {
2805                 if (sc->rx_mbuf_map[i] != NULL) {
2806                         bus_dmamap_unload(sc->rx_mbuf_tag,
2807                                 sc->rx_mbuf_map[i]);
2808                         bus_dmamap_destroy(sc->rx_mbuf_tag,
2809                                 sc->rx_mbuf_map[i]);
2810                         sc->rx_mbuf_map[i] = NULL;
2811                 }
2812         }
2813
2814         /* Destroy the RX mbuf tag. */
2815         if (sc->rx_mbuf_tag != NULL) {
2816                 bus_dma_tag_destroy(sc->rx_mbuf_tag);
2817                 sc->rx_mbuf_tag = NULL;
2818         }
2819
2820 #ifdef ZERO_COPY_SOCKETS
2821         /* Unload and destroy the page mbuf maps. */
2822         for (i = 0; i < TOTAL_PG_BD; i++) {
2823                 if (sc->pg_mbuf_map[i] != NULL) {
2824                         bus_dmamap_unload(sc->pg_mbuf_tag,
2825                                 sc->pg_mbuf_map[i]);
2826                         bus_dmamap_destroy(sc->pg_mbuf_tag,
2827                                 sc->pg_mbuf_map[i]);
2828                         sc->pg_mbuf_map[i] = NULL;
2829                 }
2830         }
2831
2832         /* Destroy the page mbuf tag. */
2833         if (sc->pg_mbuf_tag != NULL) {
2834                 bus_dma_tag_destroy(sc->pg_mbuf_tag);
2835                 sc->pg_mbuf_tag = NULL;
2836         }
2837 #endif
2838
2839         /* Destroy the parent tag */
2840         if (sc->parent_tag != NULL) {
2841                 bus_dma_tag_destroy(sc->parent_tag);
2842                 sc->parent_tag = NULL;
2843         }
2844
2845         DBEXIT(BCE_VERBOSE_RESET | BCE_VERBOSE_UNLOAD | BCE_VERBOSE_CTX);
2846 }
2847
2848
2849 /****************************************************************************/
2850 /* Get DMA memory from the OS.                                              */
2851 /*                                                                          */
2852 /* Validates that the OS has provided DMA buffers in response to a          */
2853 /* bus_dmamap_load() call and saves the physical address of those buffers.  */
2854 /* When the callback is used the OS will return 0 for the mapping function  */
2855 /* (bus_dmamap_load()) so we use the value of map_arg->maxsegs to pass any  */
2856 /* failures back to the caller.                                             */
2857 /*                                                                          */
2858 /* Returns:                                                                 */
2859 /*   Nothing.                                                               */
2860 /****************************************************************************/
2861 static void
2862 bce_dma_map_addr(void *arg, bus_dma_segment_t *segs, int nseg, int error)
2863 {
2864         bus_addr_t *busaddr = arg;
2865
2866         /* Simulate a mapping failure. */
2867         DBRUNIF(DB_RANDOMTRUE(dma_map_addr_failed_sim_control),
2868                 error = ENOMEM);
2869
2870         /* Check for an error and signal the caller that an error occurred. */
2871         if (error) {
2872                 *busaddr = 0;
2873         } else {
2874                 *busaddr = segs->ds_addr;
2875         }
2876
2877         return;
2878 }
2879
2880
2881 /****************************************************************************/
2882 /* Allocate any DMA memory needed by the driver.                            */
2883 /*                                                                          */
2884 /* Allocates DMA memory needed for the various global structures needed by  */
2885 /* hardware.                                                                */
2886 /*                                                                          */
2887 /* Memory alignment requirements:                                           */
2888 /* +-----------------+----------+----------+----------+----------+          */
2889 /* |                 |   5706   |   5708   |   5709   |   5716   |          */
2890 /* +-----------------+----------+----------+----------+----------+          */
2891 /* |Status Block     | 8 bytes  | 8 bytes  | 16 bytes | 16 bytes |          */
2892 /* |Statistics Block | 8 bytes  | 8 bytes  | 16 bytes | 16 bytes |          */
2893 /* |RX Buffers       | 16 bytes | 16 bytes | 16 bytes | 16 bytes |          */
2894 /* |PG Buffers       |   none   |   none   |   none   |   none   |          */
2895 /* |TX Buffers       |   none   |   none   |   none   |   none   |          */
2896 /* |Chain Pages(1)   |   4KiB   |   4KiB   |   4KiB   |   4KiB   |          */
2897 /* +-----------------+----------+----------+----------+----------+          */
2898 /*                                                                          */
2899 /* (1) Must align with CPU page size (BCM_PAGE_SZIE).                       */
2900 /*                                                                          */
2901 /* Returns:                                                                 */
2902 /*   0 for success, positive value for failure.                             */
2903 /****************************************************************************/
2904 static int
2905 bce_dma_alloc(device_t dev)
2906 {
2907         struct bce_softc *sc;
2908         int i, error, rc = 0;
2909         bus_size_t max_size, max_seg_size;
2910         int max_segments;
2911
2912         sc = device_get_softc(dev);
2913
2914         DBENTER(BCE_VERBOSE_RESET | BCE_VERBOSE_CTX);
2915
2916         /*
2917          * Allocate the parent bus DMA tag appropriate for PCI.
2918          */
2919         if (bus_dma_tag_create(NULL,
2920                         1,
2921                         BCE_DMA_BOUNDARY,
2922                         sc->max_bus_addr,
2923                         BUS_SPACE_MAXADDR,
2924                         NULL, NULL,
2925                         MAXBSIZE,
2926                         BUS_SPACE_UNRESTRICTED,
2927                         BUS_SPACE_MAXSIZE_32BIT,
2928                         0,
2929                         NULL, NULL,
2930                         &sc->parent_tag)) {
2931                 BCE_PRINTF("%s(%d): Could not allocate parent DMA tag!\n",
2932                         __FILE__, __LINE__);
2933                 rc = ENOMEM;
2934                 goto bce_dma_alloc_exit;
2935         }
2936
2937         /*
2938          * Create a DMA tag for the status block, allocate and clear the
2939          * memory, map the memory into DMA space, and fetch the physical
2940          * address of the block.
2941          */
2942         if (bus_dma_tag_create(sc->parent_tag,
2943                 BCE_DMA_ALIGN,
2944                 BCE_DMA_BOUNDARY,
2945                 sc->max_bus_addr,
2946                 BUS_SPACE_MAXADDR,
2947                 NULL, NULL,
2948                 BCE_STATUS_BLK_SZ,
2949                 1,
2950                 BCE_STATUS_BLK_SZ,
2951                 0,
2952                 NULL, NULL,
2953                 &sc->status_tag)) {
2954                 BCE_PRINTF("%s(%d): Could not allocate status block DMA tag!\n",
2955                         __FILE__, __LINE__);
2956                 rc = ENOMEM;
2957                 goto bce_dma_alloc_exit;
2958         }
2959
2960         if(bus_dmamem_alloc(sc->status_tag,
2961                 (void **)&sc->status_block,
2962                 BUS_DMA_NOWAIT,
2963                 &sc->status_map)) {
2964                 BCE_PRINTF("%s(%d): Could not allocate status block DMA memory!\n",
2965                         __FILE__, __LINE__);
2966                 rc = ENOMEM;
2967                 goto bce_dma_alloc_exit;
2968         }
2969
2970         bzero((char *)sc->status_block, BCE_STATUS_BLK_SZ);
2971
2972         error = bus_dmamap_load(sc->status_tag,
2973                 sc->status_map,
2974                 sc->status_block,
2975                 BCE_STATUS_BLK_SZ,
2976                 bce_dma_map_addr,
2977                 &sc->status_block_paddr,
2978                 BUS_DMA_NOWAIT);
2979
2980         if (error) {
2981                 BCE_PRINTF("%s(%d): Could not map status block DMA memory!\n",
2982                         __FILE__, __LINE__);
2983                 rc = ENOMEM;
2984                 goto bce_dma_alloc_exit;
2985         }
2986
2987         DBPRINT(sc, BCE_INFO, "%s(): status_block_paddr = 0x%jX\n",
2988                 __FUNCTION__, (uintmax_t) sc->status_block_paddr);
2989
2990         /*
2991          * Create a DMA tag for the statistics block, allocate and clear the
2992          * memory, map the memory into DMA space, and fetch the physical
2993          * address of the block.
2994          */
2995         if (bus_dma_tag_create(sc->parent_tag,
2996                 BCE_DMA_ALIGN,
2997                 BCE_DMA_BOUNDARY,
2998                 sc->max_bus_addr,
2999                 BUS_SPACE_MAXADDR,
3000                 NULL, NULL,
3001                 BCE_STATS_BLK_SZ,
3002                 1,
3003                 BCE_STATS_BLK_SZ,
3004                 0,
3005                 NULL, NULL,
3006                 &sc->stats_tag)) {
3007                 BCE_PRINTF("%s(%d): Could not allocate statistics block DMA tag!\n",
3008                         __FILE__, __LINE__);
3009                 rc = ENOMEM;
3010                 goto bce_dma_alloc_exit;
3011         }
3012
3013         if (bus_dmamem_alloc(sc->stats_tag,
3014                 (void **)&sc->stats_block,
3015                 BUS_DMA_NOWAIT,
3016                 &sc->stats_map)) {
3017                 BCE_PRINTF("%s(%d): Could not allocate statistics block DMA memory!\n",
3018                         __FILE__, __LINE__);
3019                 rc = ENOMEM;
3020                 goto bce_dma_alloc_exit;
3021         }
3022
3023         bzero((char *)sc->stats_block, BCE_STATS_BLK_SZ);
3024
3025         error = bus_dmamap_load(sc->stats_tag,
3026                 sc->stats_map,
3027                 sc->stats_block,
3028                 BCE_STATS_BLK_SZ,
3029                 bce_dma_map_addr,
3030                 &sc->stats_block_paddr,
3031                 BUS_DMA_NOWAIT);
3032
3033         if(error) {
3034                 BCE_PRINTF("%s(%d): Could not map statistics block DMA memory!\n",
3035                         __FILE__, __LINE__);
3036                 rc = ENOMEM;
3037                 goto bce_dma_alloc_exit;
3038         }
3039
3040         DBPRINT(sc, BCE_INFO, "%s(): stats_block_paddr = 0x%jX\n",
3041                 __FUNCTION__, (uintmax_t) sc->stats_block_paddr);
3042
3043         /* BCM5709 uses host memory as cache for context memory. */
3044         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
3045                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
3046                 sc->ctx_pages = 0x2000 / BCM_PAGE_SIZE;
3047                 if (sc->ctx_pages == 0)
3048                         sc->ctx_pages = 1;
3049
3050                 DBRUNIF((sc->ctx_pages > 512),
3051                         BCE_PRINTF("%s(%d): Too many CTX pages! %d > 512\n",
3052                                 __FILE__, __LINE__, sc->ctx_pages));
3053
3054                 /*
3055                  * Create a DMA tag for the context pages,
3056                  * allocate and clear the memory, map the
3057                  * memory into DMA space, and fetch the
3058                  * physical address of the block.
3059                  */
3060                 if(bus_dma_tag_create(sc->parent_tag,
3061                         BCM_PAGE_SIZE,
3062                     BCE_DMA_BOUNDARY,
3063                         sc->max_bus_addr,
3064                         BUS_SPACE_MAXADDR,
3065                         NULL, NULL,
3066                         BCM_PAGE_SIZE,
3067                         1,
3068                         BCM_PAGE_SIZE,
3069                         0,
3070                         NULL, NULL,
3071                         &sc->ctx_tag)) {
3072                         BCE_PRINTF("%s(%d): Could not allocate CTX DMA tag!\n",
3073                                 __FILE__, __LINE__);
3074                         rc = ENOMEM;
3075                         goto bce_dma_alloc_exit;
3076                 }
3077
3078                 for (i = 0; i < sc->ctx_pages; i++) {
3079
3080                         if(bus_dmamem_alloc(sc->ctx_tag,
3081                                 (void **)&sc->ctx_block[i],
3082                         BUS_DMA_NOWAIT,
3083                         &sc->ctx_map[i])) {
3084                                 BCE_PRINTF("%s(%d): Could not allocate CTX "
3085                                         "DMA memory!\n", __FILE__, __LINE__);
3086                                 rc = ENOMEM;
3087                                 goto bce_dma_alloc_exit;
3088                         }
3089
3090                         bzero((char *)sc->ctx_block[i], BCM_PAGE_SIZE);
3091
3092                         error = bus_dmamap_load(sc->ctx_tag,
3093                         sc->ctx_map[i],
3094                         sc->ctx_block[i],
3095                         BCM_PAGE_SIZE,
3096                         bce_dma_map_addr,
3097                         &sc->ctx_paddr[i],
3098                         BUS_DMA_NOWAIT);
3099
3100                         if (error) {
3101                                 BCE_PRINTF("%s(%d): Could not map CTX DMA memory!\n",
3102                                         __FILE__, __LINE__);
3103                                 rc = ENOMEM;
3104                                 goto bce_dma_alloc_exit;
3105                         }
3106
3107                         DBPRINT(sc, BCE_INFO, "%s(): ctx_paddr[%d] = 0x%jX\n",
3108                                 __FUNCTION__, i, (uintmax_t) sc->ctx_paddr[i]);
3109                 }
3110         }
3111
3112         /*
3113          * Create a DMA tag for the TX buffer descriptor chain,
3114          * allocate and clear the  memory, and fetch the
3115          * physical address of the block.
3116          */
3117         if(bus_dma_tag_create(sc->parent_tag,
3118                         BCM_PAGE_SIZE,
3119                     BCE_DMA_BOUNDARY,
3120                         sc->max_bus_addr,
3121                         BUS_SPACE_MAXADDR,
3122                         NULL, NULL,
3123                         BCE_TX_CHAIN_PAGE_SZ,
3124                         1,
3125                         BCE_TX_CHAIN_PAGE_SZ,
3126                         0,
3127                         NULL, NULL,
3128                         &sc->tx_bd_chain_tag)) {
3129                 BCE_PRINTF("%s(%d): Could not allocate TX descriptor chain DMA tag!\n",
3130                         __FILE__, __LINE__);
3131                 rc = ENOMEM;
3132                 goto bce_dma_alloc_exit;
3133         }
3134
3135         for (i = 0; i < TX_PAGES; i++) {
3136
3137                 if(bus_dmamem_alloc(sc->tx_bd_chain_tag,
3138                         (void **)&sc->tx_bd_chain[i],
3139                         BUS_DMA_NOWAIT,
3140                         &sc->tx_bd_chain_map[i])) {
3141                         BCE_PRINTF("%s(%d): Could not allocate TX descriptor "
3142                                 "chain DMA memory!\n", __FILE__, __LINE__);
3143                         rc = ENOMEM;
3144                         goto bce_dma_alloc_exit;
3145                 }
3146
3147                 error = bus_dmamap_load(sc->tx_bd_chain_tag,
3148                         sc->tx_bd_chain_map[i],
3149                         sc->tx_bd_chain[i],
3150                         BCE_TX_CHAIN_PAGE_SZ,
3151                         bce_dma_map_addr,
3152                         &sc->tx_bd_chain_paddr[i],
3153                         BUS_DMA_NOWAIT);
3154
3155                 if (error) {
3156                         BCE_PRINTF("%s(%d): Could not map TX descriptor chain DMA memory!\n",
3157                                 __FILE__, __LINE__);
3158                         rc = ENOMEM;
3159                         goto bce_dma_alloc_exit;
3160                 }
3161
3162                 DBPRINT(sc, BCE_INFO, "%s(): tx_bd_chain_paddr[%d] = 0x%jX\n",
3163                         __FUNCTION__, i, (uintmax_t) sc->tx_bd_chain_paddr[i]);
3164         }
3165
3166         /* Check the required size before mapping to conserve resources. */
3167         if (bce_tso_enable) {
3168                 max_size     = BCE_TSO_MAX_SIZE;
3169                 max_segments = BCE_MAX_SEGMENTS;
3170                 max_seg_size = BCE_TSO_MAX_SEG_SIZE;
3171         } else {
3172                 max_size     = MCLBYTES * BCE_MAX_SEGMENTS;
3173                 max_segments = BCE_MAX_SEGMENTS;
3174                 max_seg_size = MCLBYTES;
3175         }
3176
3177         /* Create a DMA tag for TX mbufs. */
3178         if (bus_dma_tag_create(sc->parent_tag,
3179                         1,
3180                         BCE_DMA_BOUNDARY,
3181                         sc->max_bus_addr,
3182                         BUS_SPACE_MAXADDR,
3183                         NULL, NULL,
3184                         max_size,
3185                         max_segments,
3186                         max_seg_size,
3187                         0,
3188                         NULL, NULL,
3189                         &sc->tx_mbuf_tag)) {
3190                 BCE_PRINTF("%s(%d): Could not allocate TX mbuf DMA tag!\n",
3191                         __FILE__, __LINE__);
3192                 rc = ENOMEM;
3193                 goto bce_dma_alloc_exit;
3194         }
3195
3196         /* Create DMA maps for the TX mbufs clusters. */
3197         for (i = 0; i < TOTAL_TX_BD; i++) {
3198                 if (bus_dmamap_create(sc->tx_mbuf_tag, BUS_DMA_NOWAIT,
3199                         &sc->tx_mbuf_map[i])) {
3200                         BCE_PRINTF("%s(%d): Unable to create TX mbuf DMA map!\n",
3201                                 __FILE__, __LINE__);
3202                         rc = ENOMEM;
3203                         goto bce_dma_alloc_exit;
3204                 }
3205         }
3206
3207         /*
3208          * Create a DMA tag for the RX buffer descriptor chain,
3209          * allocate and clear the memory, and fetch the physical
3210          * address of the blocks.
3211          */
3212         if (bus_dma_tag_create(sc->parent_tag,
3213                         BCM_PAGE_SIZE,
3214                         BCE_DMA_BOUNDARY,
3215                         BUS_SPACE_MAXADDR,
3216                         sc->max_bus_addr,
3217                         NULL, NULL,
3218                         BCE_RX_CHAIN_PAGE_SZ,
3219                         1,
3220                         BCE_RX_CHAIN_PAGE_SZ,
3221                         0,
3222                         NULL, NULL,
3223                         &sc->rx_bd_chain_tag)) {
3224                 BCE_PRINTF("%s(%d): Could not allocate RX descriptor chain DMA tag!\n",
3225                         __FILE__, __LINE__);
3226                 rc = ENOMEM;
3227                 goto bce_dma_alloc_exit;
3228         }
3229
3230         for (i = 0; i < RX_PAGES; i++) {
3231
3232                 if (bus_dmamem_alloc(sc->rx_bd_chain_tag,
3233                         (void **)&sc->rx_bd_chain[i],
3234                         BUS_DMA_NOWAIT,
3235                         &sc->rx_bd_chain_map[i])) {
3236                         BCE_PRINTF("%s(%d): Could not allocate RX descriptor chain "
3237                                 "DMA memory!\n", __FILE__, __LINE__);
3238                         rc = ENOMEM;
3239                         goto bce_dma_alloc_exit;
3240                 }
3241
3242                 bzero((char *)sc->rx_bd_chain[i], BCE_RX_CHAIN_PAGE_SZ);
3243
3244                 error = bus_dmamap_load(sc->rx_bd_chain_tag,
3245                         sc->rx_bd_chain_map[i],
3246                         sc->rx_bd_chain[i],
3247                         BCE_RX_CHAIN_PAGE_SZ,
3248                         bce_dma_map_addr,
3249                         &sc->rx_bd_chain_paddr[i],
3250                         BUS_DMA_NOWAIT);
3251
3252                 if (error) {
3253                         BCE_PRINTF("%s(%d): Could not map RX descriptor chain DMA memory!\n",
3254                                 __FILE__, __LINE__);
3255                         rc = ENOMEM;
3256                         goto bce_dma_alloc_exit;
3257                 }
3258
3259                 DBPRINT(sc, BCE_INFO, "%s(): rx_bd_chain_paddr[%d] = 0x%jX\n",
3260                         __FUNCTION__, i, (uintmax_t) sc->rx_bd_chain_paddr[i]);
3261         }
3262
3263         /*
3264          * Create a DMA tag for RX mbufs.
3265          */
3266 #ifdef ZERO_COPY_SOCKETS
3267         max_size = max_seg_size = ((sc->rx_bd_mbuf_alloc_size < MCLBYTES) ?
3268                 MCLBYTES : sc->rx_bd_mbuf_alloc_size);
3269 #else
3270         max_size = max_seg_size = MJUM9BYTES;
3271 #endif
3272         max_segments = 1;
3273
3274         DBPRINT(sc, BCE_INFO, "%s(): Creating rx_mbuf_tag (max size = 0x%jX "
3275                 "max segments = %d, max segment size = 0x%jX)\n", __FUNCTION__,
3276                 (uintmax_t) max_size, max_segments, (uintmax_t) max_seg_size);
3277
3278         if (bus_dma_tag_create(sc->parent_tag,
3279                         1,
3280                         BCE_DMA_BOUNDARY,
3281                         sc->max_bus_addr,
3282                         BUS_SPACE_MAXADDR,
3283                         NULL, NULL,
3284                         max_size,
3285                         max_segments,
3286                         max_seg_size,
3287                         0,
3288                         NULL, NULL,
3289                 &sc->rx_mbuf_tag)) {
3290                 BCE_PRINTF("%s(%d): Could not allocate RX mbuf DMA tag!\n",
3291                         __FILE__, __LINE__);
3292                 rc = ENOMEM;
3293                 goto bce_dma_alloc_exit;
3294         }
3295
3296         /* Create DMA maps for the RX mbuf clusters. */
3297         for (i = 0; i < TOTAL_RX_BD; i++) {
3298                 if (bus_dmamap_create(sc->rx_mbuf_tag, BUS_DMA_NOWAIT,
3299                                 &sc->rx_mbuf_map[i])) {
3300                         BCE_PRINTF("%s(%d): Unable to create RX mbuf DMA map!\n",
3301                                 __FILE__, __LINE__);
3302                         rc = ENOMEM;
3303                         goto bce_dma_alloc_exit;
3304                 }
3305         }
3306
3307 #ifdef ZERO_COPY_SOCKETS
3308         /*
3309          * Create a DMA tag for the page buffer descriptor chain,
3310          * allocate and clear the memory, and fetch the physical
3311          * address of the blocks.
3312          */
3313         if (bus_dma_tag_create(sc->parent_tag,
3314                         BCM_PAGE_SIZE,
3315                         BCE_DMA_BOUNDARY,
3316                         BUS_SPACE_MAXADDR,
3317                         sc->max_bus_addr,
3318                         NULL, NULL,
3319                         BCE_PG_CHAIN_PAGE_SZ,
3320                         1,
3321                         BCE_PG_CHAIN_PAGE_SZ,
3322                         0,
3323                         NULL, NULL,
3324                         &sc->pg_bd_chain_tag)) {
3325                 BCE_PRINTF("%s(%d): Could not allocate page descriptor chain DMA tag!\n",
3326                         __FILE__, __LINE__);
3327                 rc = ENOMEM;
3328                 goto bce_dma_alloc_exit;
3329         }
3330
3331         for (i = 0; i < PG_PAGES; i++) {
3332
3333                 if (bus_dmamem_alloc(sc->pg_bd_chain_tag,
3334                         (void **)&sc->pg_bd_chain[i],
3335                         BUS_DMA_NOWAIT,
3336                         &sc->pg_bd_chain_map[i])) {
3337                         BCE_PRINTF("%s(%d): Could not allocate page descriptor chain "
3338                                 "DMA memory!\n", __FILE__, __LINE__);
3339                         rc = ENOMEM;
3340                         goto bce_dma_alloc_exit;
3341                 }
3342
3343                 bzero((char *)sc->pg_bd_chain[i], BCE_PG_CHAIN_PAGE_SZ);
3344
3345                 error = bus_dmamap_load(sc->pg_bd_chain_tag,
3346                         sc->pg_bd_chain_map[i],
3347                         sc->pg_bd_chain[i],
3348                         BCE_PG_CHAIN_PAGE_SZ,
3349                         bce_dma_map_addr,
3350                         &sc->pg_bd_chain_paddr[i],
3351                         BUS_DMA_NOWAIT);
3352
3353                 if (error) {
3354                         BCE_PRINTF("%s(%d): Could not map page descriptor chain DMA memory!\n",
3355                                 __FILE__, __LINE__);
3356                         rc = ENOMEM;
3357                         goto bce_dma_alloc_exit;
3358                 }
3359
3360                 DBPRINT(sc, BCE_INFO, "%s(): pg_bd_chain_paddr[%d] = 0x%jX\n",
3361                         __FUNCTION__, i, (uintmax_t) sc->pg_bd_chain_paddr[i]);
3362         }
3363
3364         /*
3365          * Create a DMA tag for page mbufs.
3366          */
3367         max_size = max_seg_size = ((sc->pg_bd_mbuf_alloc_size < MCLBYTES) ?
3368                 MCLBYTES : sc->pg_bd_mbuf_alloc_size);
3369
3370         if (bus_dma_tag_create(sc->parent_tag,
3371                         1,
3372                         BCE_DMA_BOUNDARY,
3373                         sc->max_bus_addr,
3374                         BUS_SPACE_MAXADDR,
3375                         NULL, NULL,
3376                         max_size,
3377                         1,
3378                         max_seg_size,
3379                         0,
3380                         NULL, NULL,
3381                 &sc->pg_mbuf_tag)) {
3382                 BCE_PRINTF("%s(%d): Could not allocate page mbuf DMA tag!\n",
3383                         __FILE__, __LINE__);
3384                 rc = ENOMEM;
3385                 goto bce_dma_alloc_exit;
3386         }
3387
3388         /* Create DMA maps for the page mbuf clusters. */
3389         for (i = 0; i < TOTAL_PG_BD; i++) {
3390                 if (bus_dmamap_create(sc->pg_mbuf_tag, BUS_DMA_NOWAIT,
3391                                 &sc->pg_mbuf_map[i])) {
3392                         BCE_PRINTF("%s(%d): Unable to create page mbuf DMA map!\n",
3393                                 __FILE__, __LINE__);
3394                         rc = ENOMEM;
3395                         goto bce_dma_alloc_exit;
3396                 }
3397         }
3398 #endif
3399
3400 bce_dma_alloc_exit:
3401         DBEXIT(BCE_VERBOSE_RESET | BCE_VERBOSE_CTX);
3402         return(rc);
3403 }
3404
3405
3406 /****************************************************************************/
3407 /* Release all resources used by the driver.                                */
3408 /*                                                                          */
3409 /* Releases all resources acquired by the driver including interrupts,      */
3410 /* interrupt handler, interfaces, mutexes, and DMA memory.                  */
3411 /*                                                                          */
3412 /* Returns:                                                                 */
3413 /*   Nothing.                                                               */
3414 /****************************************************************************/
3415 static void
3416 bce_release_resources(struct bce_softc *sc)
3417 {
3418         device_t dev;
3419
3420         DBENTER(BCE_VERBOSE_RESET);
3421
3422         dev = sc->bce_dev;
3423
3424         bce_dma_free(sc);
3425
3426         if (sc->bce_intrhand != NULL) {
3427                 DBPRINT(sc, BCE_INFO_RESET, "Removing interrupt handler.\n");
3428                 bus_teardown_intr(dev, sc->bce_res_irq, sc->bce_intrhand);
3429         }
3430
3431         if (sc->bce_res_irq != NULL) {
3432                 DBPRINT(sc, BCE_INFO_RESET, "Releasing IRQ.\n");
3433                 bus_release_resource(dev, SYS_RES_IRQ, sc->bce_irq_rid,
3434                         sc->bce_res_irq);
3435         }
3436
3437         if (sc->bce_flags & (BCE_USING_MSI_FLAG | BCE_USING_MSIX_FLAG)) {
3438                 DBPRINT(sc, BCE_INFO_RESET, "Releasing MSI/MSI-X vector.\n");
3439                 pci_release_msi(dev);
3440         }
3441
3442         if (sc->bce_res_mem != NULL) {
3443                 DBPRINT(sc, BCE_INFO_RESET, "Releasing PCI memory.\n");
3444                 bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(0), sc->bce_res_mem);
3445         }
3446
3447         if (sc->bce_ifp != NULL) {
3448                 DBPRINT(sc, BCE_INFO_RESET, "Releasing IF.\n");
3449                 if_free(sc->bce_ifp);
3450         }
3451
3452         if (mtx_initialized(&sc->bce_mtx))
3453                 BCE_LOCK_DESTROY(sc);
3454
3455         DBEXIT(BCE_VERBOSE_RESET);
3456 }
3457
3458
3459 /****************************************************************************/
3460 /* Firmware synchronization.                                                */
3461 /*                                                                          */
3462 /* Before performing certain events such as a chip reset, synchronize with  */
3463 /* the firmware first.                                                      */
3464 /*                                                                          */
3465 /* Returns:                                                                 */
3466 /*   0 for success, positive value for failure.                             */
3467 /****************************************************************************/
3468 static int
3469 bce_fw_sync(struct bce_softc *sc, u32 msg_data)
3470 {
3471         int i, rc = 0;
3472         u32 val;
3473
3474         DBENTER(BCE_VERBOSE_RESET);
3475
3476         /* Don't waste any time if we've timed out before. */
3477         if (sc->bce_fw_timed_out) {
3478                 rc = EBUSY;
3479                 goto bce_fw_sync_exit;
3480         }
3481
3482         /* Increment the message sequence number. */
3483         sc->bce_fw_wr_seq++;
3484         msg_data |= sc->bce_fw_wr_seq;
3485
3486         DBPRINT(sc, BCE_VERBOSE_FIRMWARE, "bce_fw_sync(): msg_data = 0x%08X\n",
3487                 msg_data);
3488
3489         /* Send the message to the bootcode driver mailbox. */
3490         REG_WR_IND(sc, sc->bce_shmem_base + BCE_DRV_MB, msg_data);
3491
3492         /* Wait for the bootcode to acknowledge the message. */
3493         for (i = 0; i < FW_ACK_TIME_OUT_MS; i++) {
3494                 /* Check for a response in the bootcode firmware mailbox. */
3495                 val = REG_RD_IND(sc, sc->bce_shmem_base + BCE_FW_MB);
3496                 if ((val & BCE_FW_MSG_ACK) == (msg_data & BCE_DRV_MSG_SEQ))
3497                         break;
3498                 DELAY(1000);
3499         }
3500
3501         /* If we've timed out, tell the bootcode that we've stopped waiting. */
3502         if (((val & BCE_FW_MSG_ACK) != (msg_data & BCE_DRV_MSG_SEQ)) &&
3503                 ((msg_data & BCE_DRV_MSG_DATA) != BCE_DRV_MSG_DATA_WAIT0)) {
3504
3505                 BCE_PRINTF("%s(%d): Firmware synchronization timeout! "
3506                         "msg_data = 0x%08X\n",
3507                         __FILE__, __LINE__, msg_data);
3508
3509                 msg_data &= ~BCE_DRV_MSG_CODE;
3510                 msg_data |= BCE_DRV_MSG_CODE_FW_TIMEOUT;
3511
3512                 REG_WR_IND(sc, sc->bce_shmem_base + BCE_DRV_MB, msg_data);
3513
3514                 sc->bce_fw_timed_out = 1;
3515                 rc = EBUSY;
3516         }
3517
3518 bce_fw_sync_exit:
3519         DBEXIT(BCE_VERBOSE_RESET);
3520         return (rc);
3521 }
3522
3523
3524 /****************************************************************************/
3525 /* Load Receive Virtual 2 Physical (RV2P) processor firmware.               */
3526 /*                                                                          */
3527 /* Returns:                                                                 */
3528 /*   Nothing.                                                               */
3529 /****************************************************************************/
3530 static void
3531 bce_load_rv2p_fw(struct bce_softc *sc, u32 *rv2p_code,
3532         u32 rv2p_code_len, u32 rv2p_proc)
3533 {
3534         int i;
3535         u32 val;
3536
3537         DBENTER(BCE_VERBOSE_RESET);
3538
3539         /* Set the page size used by RV2P. */
3540         if (rv2p_proc == RV2P_PROC2) {
3541                 BCE_RV2P_PROC2_CHG_MAX_BD_PAGE(USABLE_RX_BD_PER_PAGE);
3542         }
3543
3544         for (i = 0; i < rv2p_code_len; i += 8) {
3545                 REG_WR(sc, BCE_RV2P_INSTR_HIGH, *rv2p_code);
3546                 rv2p_code++;
3547                 REG_WR(sc, BCE_RV2P_INSTR_LOW, *rv2p_code);
3548                 rv2p_code++;
3549
3550                 if (rv2p_proc == RV2P_PROC1) {
3551                         val = (i / 8) | BCE_RV2P_PROC1_ADDR_CMD_RDWR;
3552                         REG_WR(sc, BCE_RV2P_PROC1_ADDR_CMD, val);
3553                 }
3554                 else {
3555                         val = (i / 8) | BCE_RV2P_PROC2_ADDR_CMD_RDWR;
3556                         REG_WR(sc, BCE_RV2P_PROC2_ADDR_CMD, val);
3557                 }
3558         }
3559
3560         /* Reset the processor, un-stall is done later. */
3561         if (rv2p_proc == RV2P_PROC1) {
3562                 REG_WR(sc, BCE_RV2P_COMMAND, BCE_RV2P_COMMAND_PROC1_RESET);
3563         }
3564         else {
3565                 REG_WR(sc, BCE_RV2P_COMMAND, BCE_RV2P_COMMAND_PROC2_RESET);
3566         }
3567
3568         DBEXIT(BCE_VERBOSE_RESET);
3569 }
3570
3571
3572 /****************************************************************************/
3573 /* Load RISC processor firmware.                                            */
3574 /*                                                                          */
3575 /* Loads firmware from the file if_bcefw.h into the scratchpad memory       */
3576 /* associated with a particular processor.                                  */
3577 /*                                                                          */
3578 /* Returns:                                                                 */
3579 /*   Nothing.                                                               */
3580 /****************************************************************************/
3581 static void
3582 bce_load_cpu_fw(struct bce_softc *sc, struct cpu_reg *cpu_reg,
3583         struct fw_info *fw)
3584 {
3585         u32 offset;
3586         u32 val;
3587
3588         DBENTER(BCE_VERBOSE_RESET);
3589
3590         /* Halt the CPU. */
3591         val = REG_RD_IND(sc, cpu_reg->mode);
3592         val |= cpu_reg->mode_value_halt;
3593         REG_WR_IND(sc, cpu_reg->mode, val);
3594         REG_WR_IND(sc, cpu_reg->state, cpu_reg->state_value_clear);
3595
3596         /* Load the Text area. */
3597         offset = cpu_reg->spad_base + (fw->text_addr - cpu_reg->mips_view_base);
3598         if (fw->text) {
3599                 int j;
3600
3601                 for (j = 0; j < (fw->text_len / 4); j++, offset += 4) {
3602                         REG_WR_IND(sc, offset, fw->text[j]);
3603                 }
3604         }
3605
3606         /* Load the Data area. */
3607         offset = cpu_reg->spad_base + (fw->data_addr - cpu_reg->mips_view_base);
3608         if (fw->data) {
3609                 int j;
3610
3611                 for (j = 0; j < (fw->data_len / 4); j++, offset += 4) {
3612                         REG_WR_IND(sc, offset, fw->data[j]);
3613                 }
3614         }
3615
3616         /* Load the SBSS area. */
3617         offset = cpu_reg->spad_base + (fw->sbss_addr - cpu_reg->mips_view_base);
3618         if (fw->sbss) {
3619                 int j;
3620
3621                 for (j = 0; j < (fw->sbss_len / 4); j++, offset += 4) {
3622                         REG_WR_IND(sc, offset, fw->sbss[j]);
3623                 }
3624         }
3625
3626         /* Load the BSS area. */
3627         offset = cpu_reg->spad_base + (fw->bss_addr - cpu_reg->mips_view_base);
3628         if (fw->bss) {
3629                 int j;
3630
3631                 for (j = 0; j < (fw->bss_len/4); j++, offset += 4) {
3632                         REG_WR_IND(sc, offset, fw->bss[j]);
3633                 }
3634         }
3635
3636         /* Load the Read-Only area. */
3637         offset = cpu_reg->spad_base +
3638                 (fw->rodata_addr - cpu_reg->mips_view_base);
3639         if (fw->rodata) {
3640                 int j;
3641
3642                 for (j = 0; j < (fw->rodata_len / 4); j++, offset += 4) {
3643                         REG_WR_IND(sc, offset, fw->rodata[j]);
3644                 }
3645         }
3646
3647         /* Clear the pre-fetch instruction. */
3648         REG_WR_IND(sc, cpu_reg->inst, 0);
3649         REG_WR_IND(sc, cpu_reg->pc, fw->start_addr);
3650
3651         /* Start the CPU. */
3652         val = REG_RD_IND(sc, cpu_reg->mode);
3653         val &= ~cpu_reg->mode_value_halt;
3654         REG_WR_IND(sc, cpu_reg->state, cpu_reg->state_value_clear);
3655         REG_WR_IND(sc, cpu_reg->mode, val);
3656
3657         DBEXIT(BCE_VERBOSE_RESET);
3658 }
3659
3660
3661 /****************************************************************************/
3662 /* Initialize the RX CPU.                                                   */
3663 /*                                                                          */
3664 /* Returns:                                                                 */
3665 /*   Nothing.                                                               */
3666 /****************************************************************************/
3667 static void
3668 bce_init_rxp_cpu(struct bce_softc *sc)
3669 {
3670         struct cpu_reg cpu_reg;
3671         struct fw_info fw;
3672
3673         DBENTER(BCE_VERBOSE_RESET);
3674
3675         cpu_reg.mode = BCE_RXP_CPU_MODE;
3676         cpu_reg.mode_value_halt = BCE_RXP_CPU_MODE_SOFT_HALT;
3677         cpu_reg.mode_value_sstep = BCE_RXP_CPU_MODE_STEP_ENA;
3678         cpu_reg.state = BCE_RXP_CPU_STATE;
3679         cpu_reg.state_value_clear = 0xffffff;
3680         cpu_reg.gpr0 = BCE_RXP_CPU_REG_FILE;
3681         cpu_reg.evmask = BCE_RXP_CPU_EVENT_MASK;
3682         cpu_reg.pc = BCE_RXP_CPU_PROGRAM_COUNTER;
3683         cpu_reg.inst = BCE_RXP_CPU_INSTRUCTION;
3684         cpu_reg.bp = BCE_RXP_CPU_HW_BREAKPOINT;
3685         cpu_reg.spad_base = BCE_RXP_SCRATCH;
3686         cpu_reg.mips_view_base = 0x8000000;
3687
3688         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
3689                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
3690                 fw.ver_major = bce_RXP_b09FwReleaseMajor;
3691                 fw.ver_minor = bce_RXP_b09FwReleaseMinor;
3692                 fw.ver_fix = bce_RXP_b09FwReleaseFix;
3693                 fw.start_addr = bce_RXP_b09FwStartAddr;
3694
3695                 fw.text_addr = bce_RXP_b09FwTextAddr;
3696                 fw.text_len = bce_RXP_b09FwTextLen;
3697                 fw.text_index = 0;
3698                 fw.text = bce_RXP_b09FwText;
3699
3700                 fw.data_addr = bce_RXP_b09FwDataAddr;
3701                 fw.data_len = bce_RXP_b09FwDataLen;
3702                 fw.data_index = 0;
3703                 fw.data = bce_RXP_b09FwData;
3704
3705                 fw.sbss_addr = bce_RXP_b09FwSbssAddr;
3706                 fw.sbss_len = bce_RXP_b09FwSbssLen;
3707                 fw.sbss_index = 0;
3708                 fw.sbss = bce_RXP_b09FwSbss;
3709
3710                 fw.bss_addr = bce_RXP_b09FwBssAddr;
3711                 fw.bss_len = bce_RXP_b09FwBssLen;
3712                 fw.bss_index = 0;
3713                 fw.bss = bce_RXP_b09FwBss;
3714
3715                 fw.rodata_addr = bce_RXP_b09FwRodataAddr;
3716                 fw.rodata_len = bce_RXP_b09FwRodataLen;
3717                 fw.rodata_index = 0;
3718                 fw.rodata = bce_RXP_b09FwRodata;
3719         } else {
3720                 fw.ver_major = bce_RXP_b06FwReleaseMajor;
3721                 fw.ver_minor = bce_RXP_b06FwReleaseMinor;
3722                 fw.ver_fix = bce_RXP_b06FwReleaseFix;
3723                 fw.start_addr = bce_RXP_b06FwStartAddr;
3724
3725                 fw.text_addr = bce_RXP_b06FwTextAddr;
3726                 fw.text_len = bce_RXP_b06FwTextLen;
3727                 fw.text_index = 0;
3728                 fw.text = bce_RXP_b06FwText;
3729
3730                 fw.data_addr = bce_RXP_b06FwDataAddr;
3731                 fw.data_len = bce_RXP_b06FwDataLen;
3732                 fw.data_index = 0;
3733                 fw.data = bce_RXP_b06FwData;
3734
3735                 fw.sbss_addr = bce_RXP_b06FwSbssAddr;
3736                 fw.sbss_len = bce_RXP_b06FwSbssLen;
3737                 fw.sbss_index = 0;
3738                 fw.sbss = bce_RXP_b06FwSbss;
3739
3740                 fw.bss_addr = bce_RXP_b06FwBssAddr;
3741                 fw.bss_len = bce_RXP_b06FwBssLen;
3742                 fw.bss_index = 0;
3743                 fw.bss = bce_RXP_b06FwBss;
3744
3745                 fw.rodata_addr = bce_RXP_b06FwRodataAddr;
3746                 fw.rodata_len = bce_RXP_b06FwRodataLen;
3747                 fw.rodata_index = 0;
3748                 fw.rodata = bce_RXP_b06FwRodata;
3749         }
3750
3751         DBPRINT(sc, BCE_INFO_RESET, "Loading RX firmware.\n");
3752         bce_load_cpu_fw(sc, &cpu_reg, &fw);
3753
3754         DBEXIT(BCE_VERBOSE_RESET);
3755 }
3756
3757
3758 /****************************************************************************/
3759 /* Initialize the TX CPU.                                                   */
3760 /*                                                                          */
3761 /* Returns:                                                                 */
3762 /*   Nothing.                                                               */
3763 /****************************************************************************/
3764 static void
3765 bce_init_txp_cpu(struct bce_softc *sc)
3766 {
3767         struct cpu_reg cpu_reg;
3768         struct fw_info fw;
3769
3770         DBENTER(BCE_VERBOSE_RESET);
3771
3772         cpu_reg.mode = BCE_TXP_CPU_MODE;
3773         cpu_reg.mode_value_halt = BCE_TXP_CPU_MODE_SOFT_HALT;
3774         cpu_reg.mode_value_sstep = BCE_TXP_CPU_MODE_STEP_ENA;
3775         cpu_reg.state = BCE_TXP_CPU_STATE;
3776         cpu_reg.state_value_clear = 0xffffff;
3777         cpu_reg.gpr0 = BCE_TXP_CPU_REG_FILE;
3778         cpu_reg.evmask = BCE_TXP_CPU_EVENT_MASK;
3779         cpu_reg.pc = BCE_TXP_CPU_PROGRAM_COUNTER;
3780         cpu_reg.inst = BCE_TXP_CPU_INSTRUCTION;
3781         cpu_reg.bp = BCE_TXP_CPU_HW_BREAKPOINT;
3782         cpu_reg.spad_base = BCE_TXP_SCRATCH;
3783         cpu_reg.mips_view_base = 0x8000000;
3784
3785         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
3786                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
3787                 fw.ver_major = bce_TXP_b09FwReleaseMajor;
3788                 fw.ver_minor = bce_TXP_b09FwReleaseMinor;
3789                 fw.ver_fix = bce_TXP_b09FwReleaseFix;
3790                 fw.start_addr = bce_TXP_b09FwStartAddr;
3791
3792                 fw.text_addr = bce_TXP_b09FwTextAddr;
3793                 fw.text_len = bce_TXP_b09FwTextLen;
3794                 fw.text_index = 0;
3795                 fw.text = bce_TXP_b09FwText;
3796
3797                 fw.data_addr = bce_TXP_b09FwDataAddr;
3798                 fw.data_len = bce_TXP_b09FwDataLen;
3799                 fw.data_index = 0;
3800                 fw.data = bce_TXP_b09FwData;
3801
3802                 fw.sbss_addr = bce_TXP_b09FwSbssAddr;
3803                 fw.sbss_len = bce_TXP_b09FwSbssLen;
3804                 fw.sbss_index = 0;
3805                 fw.sbss = bce_TXP_b09FwSbss;
3806
3807                 fw.bss_addr = bce_TXP_b09FwBssAddr;
3808                 fw.bss_len = bce_TXP_b09FwBssLen;
3809                 fw.bss_index = 0;
3810                 fw.bss = bce_TXP_b09FwBss;
3811
3812                 fw.rodata_addr = bce_TXP_b09FwRodataAddr;
3813                 fw.rodata_len = bce_TXP_b09FwRodataLen;
3814                 fw.rodata_index = 0;
3815                 fw.rodata = bce_TXP_b09FwRodata;
3816         } else {
3817                 fw.ver_major = bce_TXP_b06FwReleaseMajor;
3818                 fw.ver_minor = bce_TXP_b06FwReleaseMinor;
3819                 fw.ver_fix = bce_TXP_b06FwReleaseFix;
3820                 fw.start_addr = bce_TXP_b06FwStartAddr;
3821
3822                 fw.text_addr = bce_TXP_b06FwTextAddr;
3823                 fw.text_len = bce_TXP_b06FwTextLen;
3824                 fw.text_index = 0;
3825                 fw.text = bce_TXP_b06FwText;
3826
3827                 fw.data_addr = bce_TXP_b06FwDataAddr;
3828                 fw.data_len = bce_TXP_b06FwDataLen;
3829                 fw.data_index = 0;
3830                 fw.data = bce_TXP_b06FwData;
3831
3832                 fw.sbss_addr = bce_TXP_b06FwSbssAddr;
3833                 fw.sbss_len = bce_TXP_b06FwSbssLen;
3834                 fw.sbss_index = 0;
3835                 fw.sbss = bce_TXP_b06FwSbss;
3836
3837                 fw.bss_addr = bce_TXP_b06FwBssAddr;
3838                 fw.bss_len = bce_TXP_b06FwBssLen;
3839                 fw.bss_index = 0;
3840                 fw.bss = bce_TXP_b06FwBss;
3841
3842                 fw.rodata_addr = bce_TXP_b06FwRodataAddr;
3843                 fw.rodata_len = bce_TXP_b06FwRodataLen;
3844                 fw.rodata_index = 0;
3845                 fw.rodata = bce_TXP_b06FwRodata;
3846         }
3847
3848         DBPRINT(sc, BCE_INFO_RESET, "Loading TX firmware.\n");
3849         bce_load_cpu_fw(sc, &cpu_reg, &fw);
3850
3851         DBEXIT(BCE_VERBOSE_RESET);
3852 }
3853
3854
3855 /****************************************************************************/
3856 /* Initialize the TPAT CPU.                                                 */
3857 /*                                                                          */
3858 /* Returns:                                                                 */
3859 /*   Nothing.                                                               */
3860 /****************************************************************************/
3861 static void
3862 bce_init_tpat_cpu(struct bce_softc *sc)
3863 {
3864         struct cpu_reg cpu_reg;
3865         struct fw_info fw;
3866
3867         DBENTER(BCE_VERBOSE_RESET);
3868
3869         cpu_reg.mode = BCE_TPAT_CPU_MODE;
3870         cpu_reg.mode_value_halt = BCE_TPAT_CPU_MODE_SOFT_HALT;
3871         cpu_reg.mode_value_sstep = BCE_TPAT_CPU_MODE_STEP_ENA;
3872         cpu_reg.state = BCE_TPAT_CPU_STATE;
3873         cpu_reg.state_value_clear = 0xffffff;
3874         cpu_reg.gpr0 = BCE_TPAT_CPU_REG_FILE;
3875         cpu_reg.evmask = BCE_TPAT_CPU_EVENT_MASK;
3876         cpu_reg.pc = BCE_TPAT_CPU_PROGRAM_COUNTER;
3877         cpu_reg.inst = BCE_TPAT_CPU_INSTRUCTION;
3878         cpu_reg.bp = BCE_TPAT_CPU_HW_BREAKPOINT;
3879         cpu_reg.spad_base = BCE_TPAT_SCRATCH;
3880         cpu_reg.mips_view_base = 0x8000000;
3881
3882         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
3883                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
3884                 fw.ver_major = bce_TPAT_b09FwReleaseMajor;
3885                 fw.ver_minor = bce_TPAT_b09FwReleaseMinor;
3886                 fw.ver_fix = bce_TPAT_b09FwReleaseFix;
3887                 fw.start_addr = bce_TPAT_b09FwStartAddr;
3888
3889                 fw.text_addr = bce_TPAT_b09FwTextAddr;
3890                 fw.text_len = bce_TPAT_b09FwTextLen;
3891                 fw.text_index = 0;
3892                 fw.text = bce_TPAT_b09FwText;
3893
3894                 fw.data_addr = bce_TPAT_b09FwDataAddr;
3895                 fw.data_len = bce_TPAT_b09FwDataLen;
3896                 fw.data_index = 0;
3897                 fw.data = bce_TPAT_b09FwData;
3898
3899                 fw.sbss_addr = bce_TPAT_b09FwSbssAddr;
3900                 fw.sbss_len = bce_TPAT_b09FwSbssLen;
3901                 fw.sbss_index = 0;
3902                 fw.sbss = bce_TPAT_b09FwSbss;
3903
3904                 fw.bss_addr = bce_TPAT_b09FwBssAddr;
3905                 fw.bss_len = bce_TPAT_b09FwBssLen;
3906                 fw.bss_index = 0;
3907                 fw.bss = bce_TPAT_b09FwBss;
3908
3909                 fw.rodata_addr = bce_TPAT_b09FwRodataAddr;
3910                 fw.rodata_len = bce_TPAT_b09FwRodataLen;
3911                 fw.rodata_index = 0;
3912                 fw.rodata = bce_TPAT_b09FwRodata;
3913         } else {
3914                 fw.ver_major = bce_TPAT_b06FwReleaseMajor;
3915                 fw.ver_minor = bce_TPAT_b06FwReleaseMinor;
3916                 fw.ver_fix = bce_TPAT_b06FwReleaseFix;
3917                 fw.start_addr = bce_TPAT_b06FwStartAddr;
3918
3919                 fw.text_addr = bce_TPAT_b06FwTextAddr;
3920                 fw.text_len = bce_TPAT_b06FwTextLen;
3921                 fw.text_index = 0;
3922                 fw.text = bce_TPAT_b06FwText;
3923
3924                 fw.data_addr = bce_TPAT_b06FwDataAddr;
3925                 fw.data_len = bce_TPAT_b06FwDataLen;
3926                 fw.data_index = 0;
3927                 fw.data = bce_TPAT_b06FwData;
3928
3929                 fw.sbss_addr = bce_TPAT_b06FwSbssAddr;
3930                 fw.sbss_len = bce_TPAT_b06FwSbssLen;
3931                 fw.sbss_index = 0;
3932                 fw.sbss = bce_TPAT_b06FwSbss;
3933
3934                 fw.bss_addr = bce_TPAT_b06FwBssAddr;
3935                 fw.bss_len = bce_TPAT_b06FwBssLen;
3936                 fw.bss_index = 0;
3937                 fw.bss = bce_TPAT_b06FwBss;
3938
3939                 fw.rodata_addr = bce_TPAT_b06FwRodataAddr;
3940                 fw.rodata_len = bce_TPAT_b06FwRodataLen;
3941                 fw.rodata_index = 0;
3942                 fw.rodata = bce_TPAT_b06FwRodata;
3943         }
3944
3945         DBPRINT(sc, BCE_INFO_RESET, "Loading TPAT firmware.\n");
3946         bce_load_cpu_fw(sc, &cpu_reg, &fw);
3947
3948         DBEXIT(BCE_VERBOSE_RESET);
3949 }
3950
3951
3952 /****************************************************************************/
3953 /* Initialize the CP CPU.                                                   */
3954 /*                                                                          */
3955 /* Returns:                                                                 */
3956 /*   Nothing.                                                               */
3957 /****************************************************************************/
3958 static void
3959 bce_init_cp_cpu(struct bce_softc *sc)
3960 {
3961         struct cpu_reg cpu_reg;
3962         struct fw_info fw;
3963
3964         DBENTER(BCE_VERBOSE_RESET);
3965
3966         cpu_reg.mode = BCE_CP_CPU_MODE;
3967         cpu_reg.mode_value_halt = BCE_CP_CPU_MODE_SOFT_HALT;
3968         cpu_reg.mode_value_sstep = BCE_CP_CPU_MODE_STEP_ENA;
3969         cpu_reg.state = BCE_CP_CPU_STATE;
3970         cpu_reg.state_value_clear = 0xffffff;
3971         cpu_reg.gpr0 = BCE_CP_CPU_REG_FILE;
3972         cpu_reg.evmask = BCE_CP_CPU_EVENT_MASK;
3973         cpu_reg.pc = BCE_CP_CPU_PROGRAM_COUNTER;
3974         cpu_reg.inst = BCE_CP_CPU_INSTRUCTION;
3975         cpu_reg.bp = BCE_CP_CPU_HW_BREAKPOINT;
3976         cpu_reg.spad_base = BCE_CP_SCRATCH;
3977         cpu_reg.mips_view_base = 0x8000000;
3978
3979         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
3980                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
3981                 fw.ver_major = bce_CP_b09FwReleaseMajor;
3982                 fw.ver_minor = bce_CP_b09FwReleaseMinor;
3983                 fw.ver_fix = bce_CP_b09FwReleaseFix;
3984                 fw.start_addr = bce_CP_b09FwStartAddr;
3985
3986                 fw.text_addr = bce_CP_b09FwTextAddr;
3987                 fw.text_len = bce_CP_b09FwTextLen;
3988                 fw.text_index = 0;
3989                 fw.text = bce_CP_b09FwText;
3990
3991                 fw.data_addr = bce_CP_b09FwDataAddr;
3992                 fw.data_len = bce_CP_b09FwDataLen;
3993                 fw.data_index = 0;
3994                 fw.data = bce_CP_b09FwData;
3995
3996                 fw.sbss_addr = bce_CP_b09FwSbssAddr;
3997                 fw.sbss_len = bce_CP_b09FwSbssLen;
3998                 fw.sbss_index = 0;
3999                 fw.sbss = bce_CP_b09FwSbss;
4000
4001                 fw.bss_addr = bce_CP_b09FwBssAddr;
4002                 fw.bss_len = bce_CP_b09FwBssLen;
4003                 fw.bss_index = 0;
4004                 fw.bss = bce_CP_b09FwBss;
4005
4006                 fw.rodata_addr = bce_CP_b09FwRodataAddr;
4007                 fw.rodata_len = bce_CP_b09FwRodataLen;
4008                 fw.rodata_index = 0;
4009                 fw.rodata = bce_CP_b09FwRodata;
4010         } else {
4011                 fw.ver_major = bce_CP_b06FwReleaseMajor;
4012                 fw.ver_minor = bce_CP_b06FwReleaseMinor;
4013                 fw.ver_fix = bce_CP_b06FwReleaseFix;
4014                 fw.start_addr = bce_CP_b06FwStartAddr;
4015
4016                 fw.text_addr = bce_CP_b06FwTextAddr;
4017                 fw.text_len = bce_CP_b06FwTextLen;
4018                 fw.text_index = 0;
4019                 fw.text = bce_CP_b06FwText;
4020
4021                 fw.data_addr = bce_CP_b06FwDataAddr;
4022                 fw.data_len = bce_CP_b06FwDataLen;
4023                 fw.data_index = 0;
4024                 fw.data = bce_CP_b06FwData;
4025
4026                 fw.sbss_addr = bce_CP_b06FwSbssAddr;
4027                 fw.sbss_len = bce_CP_b06FwSbssLen;
4028                 fw.sbss_index = 0;
4029                 fw.sbss = bce_CP_b06FwSbss;
4030
4031                 fw.bss_addr = bce_CP_b06FwBssAddr;
4032                 fw.bss_len = bce_CP_b06FwBssLen;
4033                 fw.bss_index = 0;
4034                 fw.bss = bce_CP_b06FwBss;
4035
4036                 fw.rodata_addr = bce_CP_b06FwRodataAddr;
4037                 fw.rodata_len = bce_CP_b06FwRodataLen;
4038                 fw.rodata_index = 0;
4039                 fw.rodata = bce_CP_b06FwRodata;
4040         }
4041
4042         DBPRINT(sc, BCE_INFO_RESET, "Loading CP firmware.\n");
4043         bce_load_cpu_fw(sc, &cpu_reg, &fw);
4044
4045         DBEXIT(BCE_VERBOSE_RESET);
4046 }
4047
4048
4049 /****************************************************************************/
4050 /* Initialize the COM CPU.                                                 */
4051 /*                                                                          */
4052 /* Returns:                                                                 */
4053 /*   Nothing.                                                               */
4054 /****************************************************************************/
4055 static void
4056 bce_init_com_cpu(struct bce_softc *sc)
4057 {
4058         struct cpu_reg cpu_reg;
4059         struct fw_info fw;
4060
4061         DBENTER(BCE_VERBOSE_RESET);
4062
4063         cpu_reg.mode = BCE_COM_CPU_MODE;
4064         cpu_reg.mode_value_halt = BCE_COM_CPU_MODE_SOFT_HALT;
4065         cpu_reg.mode_value_sstep = BCE_COM_CPU_MODE_STEP_ENA;
4066         cpu_reg.state = BCE_COM_CPU_STATE;
4067         cpu_reg.state_value_clear = 0xffffff;
4068         cpu_reg.gpr0 = BCE_COM_CPU_REG_FILE;
4069         cpu_reg.evmask = BCE_COM_CPU_EVENT_MASK;
4070         cpu_reg.pc = BCE_COM_CPU_PROGRAM_COUNTER;
4071         cpu_reg.inst = BCE_COM_CPU_INSTRUCTION;
4072         cpu_reg.bp = BCE_COM_CPU_HW_BREAKPOINT;
4073         cpu_reg.spad_base = BCE_COM_SCRATCH;
4074         cpu_reg.mips_view_base = 0x8000000;
4075
4076         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
4077                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
4078                 fw.ver_major = bce_COM_b09FwReleaseMajor;
4079                 fw.ver_minor = bce_COM_b09FwReleaseMinor;
4080                 fw.ver_fix = bce_COM_b09FwReleaseFix;
4081                 fw.start_addr = bce_COM_b09FwStartAddr;
4082
4083                 fw.text_addr = bce_COM_b09FwTextAddr;
4084                 fw.text_len = bce_COM_b09FwTextLen;
4085                 fw.text_index = 0;
4086                 fw.text = bce_COM_b09FwText;
4087
4088                 fw.data_addr = bce_COM_b09FwDataAddr;
4089                 fw.data_len = bce_COM_b09FwDataLen;
4090                 fw.data_index = 0;
4091                 fw.data = bce_COM_b09FwData;
4092
4093                 fw.sbss_addr = bce_COM_b09FwSbssAddr;
4094                 fw.sbss_len = bce_COM_b09FwSbssLen;
4095                 fw.sbss_index = 0;
4096                 fw.sbss = bce_COM_b09FwSbss;
4097
4098                 fw.bss_addr = bce_COM_b09FwBssAddr;
4099                 fw.bss_len = bce_COM_b09FwBssLen;
4100                 fw.bss_index = 0;
4101                 fw.bss = bce_COM_b09FwBss;
4102
4103                 fw.rodata_addr = bce_COM_b09FwRodataAddr;
4104                 fw.rodata_len = bce_COM_b09FwRodataLen;
4105                 fw.rodata_index = 0;
4106                 fw.rodata = bce_COM_b09FwRodata;
4107         } else {
4108                 fw.ver_major = bce_COM_b06FwReleaseMajor;
4109                 fw.ver_minor = bce_COM_b06FwReleaseMinor;
4110                 fw.ver_fix = bce_COM_b06FwReleaseFix;
4111                 fw.start_addr = bce_COM_b06FwStartAddr;
4112
4113                 fw.text_addr = bce_COM_b06FwTextAddr;
4114                 fw.text_len = bce_COM_b06FwTextLen;
4115                 fw.text_index = 0;
4116                 fw.text = bce_COM_b06FwText;
4117
4118                 fw.data_addr = bce_COM_b06FwDataAddr;
4119                 fw.data_len = bce_COM_b06FwDataLen;
4120                 fw.data_index = 0;
4121                 fw.data = bce_COM_b06FwData;
4122
4123                 fw.sbss_addr = bce_COM_b06FwSbssAddr;
4124                 fw.sbss_len = bce_COM_b06FwSbssLen;
4125                 fw.sbss_index = 0;
4126                 fw.sbss = bce_COM_b06FwSbss;
4127
4128                 fw.bss_addr = bce_COM_b06FwBssAddr;
4129                 fw.bss_len = bce_COM_b06FwBssLen;
4130                 fw.bss_index = 0;
4131                 fw.bss = bce_COM_b06FwBss;
4132
4133                 fw.rodata_addr = bce_COM_b06FwRodataAddr;
4134                 fw.rodata_len = bce_COM_b06FwRodataLen;
4135                 fw.rodata_index = 0;
4136                 fw.rodata = bce_COM_b06FwRodata;
4137         }
4138
4139         DBPRINT(sc, BCE_INFO_RESET, "Loading COM firmware.\n");
4140         bce_load_cpu_fw(sc, &cpu_reg, &fw);
4141
4142         DBEXIT(BCE_VERBOSE_RESET);
4143 }
4144
4145
4146 /****************************************************************************/
4147 /* Initialize the RV2P, RX, TX, TPAT, COM, and CP CPUs.                     */
4148 /*                                                                          */
4149 /* Loads the firmware for each CPU and starts the CPU.                      */
4150 /*                                                                          */
4151 /* Returns:                                                                 */
4152 /*   Nothing.                                                               */
4153 /****************************************************************************/
4154 static void
4155 bce_init_cpus(struct bce_softc *sc)
4156 {
4157         DBENTER(BCE_VERBOSE_RESET);
4158
4159         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
4160                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
4161
4162                 if ((BCE_CHIP_REV(sc) == BCE_CHIP_REV_Ax)) {
4163                         bce_load_rv2p_fw(sc, bce_xi90_rv2p_proc1,
4164                                 sizeof(bce_xi90_rv2p_proc1), RV2P_PROC1);
4165                         bce_load_rv2p_fw(sc, bce_xi90_rv2p_proc2,
4166                                 sizeof(bce_xi90_rv2p_proc2), RV2P_PROC2);
4167                 } else {
4168                         bce_load_rv2p_fw(sc, bce_xi_rv2p_proc1,
4169                                 sizeof(bce_xi_rv2p_proc1), RV2P_PROC1);
4170                         bce_load_rv2p_fw(sc, bce_xi_rv2p_proc2,
4171                                 sizeof(bce_xi_rv2p_proc2), RV2P_PROC2);
4172                 }
4173
4174         } else {
4175                 bce_load_rv2p_fw(sc, bce_rv2p_proc1,
4176                         sizeof(bce_rv2p_proc1), RV2P_PROC1);
4177                 bce_load_rv2p_fw(sc, bce_rv2p_proc2,
4178                         sizeof(bce_rv2p_proc2), RV2P_PROC2);
4179         }
4180
4181         bce_init_rxp_cpu(sc);
4182         bce_init_txp_cpu(sc);
4183         bce_init_tpat_cpu(sc);
4184         bce_init_com_cpu(sc);
4185         bce_init_cp_cpu(sc);
4186
4187         DBEXIT(BCE_VERBOSE_RESET);
4188 }
4189
4190
4191 /****************************************************************************/
4192 /* Initialize context memory.                                               */
4193 /*                                                                          */
4194 /* Clears the memory associated with each Context ID (CID).                 */
4195 /*                                                                          */
4196 /* Returns:                                                                 */
4197 /*   Nothing.                                                               */
4198 /****************************************************************************/
4199 static void
4200 bce_init_ctx(struct bce_softc *sc)
4201 {
4202
4203         DBENTER(BCE_VERBOSE_RESET | BCE_VERBOSE_CTX);
4204
4205         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
4206                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
4207                 /* DRC: Replace this constant value with a #define. */
4208                 int i, retry_cnt = 10;
4209                 u32 val;
4210
4211                 DBPRINT(sc, BCE_INFO_CTX, "Initializing 5709 context.\n");
4212
4213                 /*
4214                  * BCM5709 context memory may be cached
4215                  * in host memory so prepare the host memory
4216                  * for access.
4217                  */
4218                 val = BCE_CTX_COMMAND_ENABLED | BCE_CTX_COMMAND_MEM_INIT | (1 << 12);
4219                 val |= (BCM_PAGE_BITS - 8) << 16;
4220                 REG_WR(sc, BCE_CTX_COMMAND, val);
4221
4222                 /* Wait for mem init command to complete. */
4223                 for (i = 0; i < retry_cnt; i++) {
4224                         val = REG_RD(sc, BCE_CTX_COMMAND);
4225                         if (!(val & BCE_CTX_COMMAND_MEM_INIT))
4226                                 break;
4227                         DELAY(2);
4228                 }
4229
4230                 /* ToDo: Consider returning an error here. */
4231                 DBRUNIF((val & BCE_CTX_COMMAND_MEM_INIT),
4232                         BCE_PRINTF("%s(): Context memory initialization failed!\n",
4233                         __FUNCTION__));
4234
4235                 for (i = 0; i < sc->ctx_pages; i++) {
4236                         int j;
4237
4238                         /* Set the physical address of the context memory cache. */
4239                         REG_WR(sc, BCE_CTX_HOST_PAGE_TBL_DATA0,
4240                                 BCE_ADDR_LO(sc->ctx_paddr[i] & 0xfffffff0) |
4241                                 BCE_CTX_HOST_PAGE_TBL_DATA0_VALID);
4242                         REG_WR(sc, BCE_CTX_HOST_PAGE_TBL_DATA1,
4243                                 BCE_ADDR_HI(sc->ctx_paddr[i]));
4244                         REG_WR(sc, BCE_CTX_HOST_PAGE_TBL_CTRL, i |
4245                                 BCE_CTX_HOST_PAGE_TBL_CTRL_WRITE_REQ);
4246
4247                         /* Verify that the context memory write was successful. */
4248                         for (j = 0; j < retry_cnt; j++) {
4249                                 val = REG_RD(sc, BCE_CTX_HOST_PAGE_TBL_CTRL);
4250                                 if ((val & BCE_CTX_HOST_PAGE_TBL_CTRL_WRITE_REQ) == 0)
4251                                         break;
4252                                 DELAY(5);
4253                         }
4254
4255                         /* ToDo: Consider returning an error here. */
4256                         DBRUNIF((val & BCE_CTX_HOST_PAGE_TBL_CTRL_WRITE_REQ),
4257                                 BCE_PRINTF("%s(): Failed to initialize context page %d!\n",
4258                                 __FUNCTION__, i));
4259                 }
4260         } else {
4261                 u32 vcid_addr, offset;
4262
4263                 DBPRINT(sc, BCE_INFO, "Initializing 5706/5708 context.\n");
4264
4265                 /*
4266                  * For the 5706/5708, context memory is local to
4267                  * the controller, so initialize the controller
4268                  * context memory.
4269                  */
4270
4271                 vcid_addr = GET_CID_ADDR(96);
4272                 while (vcid_addr) {
4273
4274                         vcid_addr -= PHY_CTX_SIZE;
4275
4276                         REG_WR(sc, BCE_CTX_VIRT_ADDR, 0);
4277                         REG_WR(sc, BCE_CTX_PAGE_TBL, vcid_addr);
4278
4279             for(offset = 0; offset < PHY_CTX_SIZE; offset += 4) {
4280                 CTX_WR(sc, 0x00, offset, 0);
4281             }
4282
4283                         REG_WR(sc, BCE_CTX_VIRT_ADDR, vcid_addr);
4284                         REG_WR(sc, BCE_CTX_PAGE_TBL, vcid_addr);
4285                 }
4286
4287         }
4288         DBEXIT(BCE_VERBOSE_RESET | BCE_VERBOSE_CTX);
4289 }
4290
4291
4292 /****************************************************************************/
4293 /* Fetch the permanent MAC address of the controller.                       */
4294 /*                                                                          */
4295 /* Returns:                                                                 */
4296 /*   Nothing.                                                               */
4297 /****************************************************************************/
4298 static void
4299 bce_get_mac_addr(struct bce_softc *sc)
4300 {
4301         u32 mac_lo = 0, mac_hi = 0;
4302
4303         DBENTER(BCE_VERBOSE_RESET);
4304         /*
4305          * The NetXtreme II bootcode populates various NIC
4306          * power-on and runtime configuration items in a
4307          * shared memory area.  The factory configured MAC
4308          * address is available from both NVRAM and the
4309          * shared memory area so we'll read the value from
4310          * shared memory for speed.
4311          */
4312
4313         mac_hi = REG_RD_IND(sc, sc->bce_shmem_base +
4314                 BCE_PORT_HW_CFG_MAC_UPPER);
4315         mac_lo = REG_RD_IND(sc, sc->bce_shmem_base +
4316                 BCE_PORT_HW_CFG_MAC_LOWER);
4317
4318         if ((mac_lo == 0) && (mac_hi == 0)) {
4319                 BCE_PRINTF("%s(%d): Invalid Ethernet address!\n",
4320                         __FILE__, __LINE__);
4321         } else {
4322                 sc->eaddr[0] = (u_char)(mac_hi >> 8);
4323                 sc->eaddr[1] = (u_char)(mac_hi >> 0);
4324                 sc->eaddr[2] = (u_char)(mac_lo >> 24);
4325                 sc->eaddr[3] = (u_char)(mac_lo >> 16);
4326                 sc->eaddr[4] = (u_char)(mac_lo >> 8);
4327                 sc->eaddr[5] = (u_char)(mac_lo >> 0);
4328         }
4329
4330         DBPRINT(sc, BCE_INFO_MISC, "Permanent Ethernet address = %6D\n", sc->eaddr, ":");
4331         DBEXIT(BCE_VERBOSE_RESET);
4332 }
4333
4334
4335 /****************************************************************************/
4336 /* Program the MAC address.                                                 */
4337 /*                                                                          */
4338 /* Returns:                                                                 */
4339 /*   Nothing.                                                               */
4340 /****************************************************************************/
4341 static void
4342 bce_set_mac_addr(struct bce_softc *sc)
4343 {
4344         u32 val;
4345         u8 *mac_addr = sc->eaddr;
4346
4347         /* ToDo: Add support for setting multiple MAC addresses. */
4348
4349         DBENTER(BCE_VERBOSE_RESET);
4350         DBPRINT(sc, BCE_INFO_MISC, "Setting Ethernet address = %6D\n", sc->eaddr, ":");
4351
4352         val = (mac_addr[0] << 8) | mac_addr[1];
4353
4354         REG_WR(sc, BCE_EMAC_MAC_MATCH0, val);
4355
4356         val = (mac_addr[2] << 24) | (mac_addr[3] << 16) |
4357                 (mac_addr[4] << 8) | mac_addr[5];
4358
4359         REG_WR(sc, BCE_EMAC_MAC_MATCH1, val);
4360
4361         DBEXIT(BCE_VERBOSE_RESET);
4362 }
4363
4364
4365 /****************************************************************************/
4366 /* Stop the controller.                                                     */
4367 /*                                                                          */
4368 /* Returns:                                                                 */
4369 /*   Nothing.                                                               */
4370 /****************************************************************************/
4371 static void
4372 bce_stop(struct bce_softc *sc)
4373 {
4374         struct ifnet *ifp;
4375         struct ifmedia_entry *ifm;
4376         struct mii_data *mii = NULL;
4377         int mtmp, itmp;
4378
4379         DBENTER(BCE_VERBOSE_RESET);
4380
4381         BCE_LOCK_ASSERT(sc);
4382
4383         ifp = sc->bce_ifp;
4384
4385         mii = device_get_softc(sc->bce_miibus);
4386
4387         callout_stop(&sc->bce_tick_callout);
4388
4389         /* Disable the transmit/receive blocks. */
4390         REG_WR(sc, BCE_MISC_ENABLE_CLR_BITS, BCE_MISC_ENABLE_CLR_DEFAULT);
4391         REG_RD(sc, BCE_MISC_ENABLE_CLR_BITS);
4392         DELAY(20);
4393
4394         bce_disable_intr(sc);
4395
4396         /* Free RX buffers. */
4397 #ifdef ZERO_COPY_SOCKETS
4398         bce_free_pg_chain(sc);
4399 #endif
4400         bce_free_rx_chain(sc);
4401
4402         /* Free TX buffers. */
4403         bce_free_tx_chain(sc);
4404
4405         /*
4406          * Isolate/power down the PHY, but leave the media selection
4407          * unchanged so that things will be put back to normal when
4408          * we bring the interface back up.
4409          */
4410
4411         itmp = ifp->if_flags;
4412         ifp->if_flags |= IFF_UP;
4413
4414         /* If we are called from bce_detach(), mii is already NULL. */
4415         if (mii != NULL) {
4416                 ifm = mii->mii_media.ifm_cur;
4417                 mtmp = ifm->ifm_media;
4418                 ifm->ifm_media = IFM_ETHER | IFM_NONE;
4419                 mii_mediachg(mii);
4420                 ifm->ifm_media = mtmp;
4421         }
4422
4423         ifp->if_flags = itmp;
4424         sc->watchdog_timer = 0;
4425
4426         sc->bce_link = 0;
4427
4428         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
4429
4430         DBEXIT(BCE_VERBOSE_RESET);
4431 }
4432
4433
4434 static int
4435 bce_reset(struct bce_softc *sc, u32 reset_code)
4436 {
4437         u32 val;
4438         int i, rc = 0;
4439
4440         DBENTER(BCE_VERBOSE_RESET);
4441
4442         DBPRINT(sc, BCE_VERBOSE_RESET, "%s(): reset_code = 0x%08X\n",
4443                 __FUNCTION__, reset_code);
4444
4445         /* Wait for pending PCI transactions to complete. */
4446         REG_WR(sc, BCE_MISC_ENABLE_CLR_BITS,
4447                BCE_MISC_ENABLE_CLR_BITS_TX_DMA_ENABLE |
4448                BCE_MISC_ENABLE_CLR_BITS_DMA_ENGINE_ENABLE |
4449                BCE_MISC_ENABLE_CLR_BITS_RX_DMA_ENABLE |
4450                BCE_MISC_ENABLE_CLR_BITS_HOST_COALESCE_ENABLE);
4451         val = REG_RD(sc, BCE_MISC_ENABLE_CLR_BITS);
4452         DELAY(5);
4453
4454         /* Disable DMA */
4455         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
4456                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
4457                 val = REG_RD(sc, BCE_MISC_NEW_CORE_CTL);
4458                 val &= ~BCE_MISC_NEW_CORE_CTL_DMA_ENABLE;
4459                 REG_WR(sc, BCE_MISC_NEW_CORE_CTL, val);
4460         }
4461
4462         /* Assume bootcode is running. */
4463         sc->bce_fw_timed_out = 0;
4464
4465         /* Give the firmware a chance to prepare for the reset. */
4466         rc = bce_fw_sync(sc, BCE_DRV_MSG_DATA_WAIT0 | reset_code);
4467         if (rc)
4468                 goto bce_reset_exit;
4469
4470         /* Set a firmware reminder that this is a soft reset. */
4471         REG_WR_IND(sc, sc->bce_shmem_base + BCE_DRV_RESET_SIGNATURE,
4472                    BCE_DRV_RESET_SIGNATURE_MAGIC);
4473
4474         /* Dummy read to force the chip to complete all current transactions. */
4475         val = REG_RD(sc, BCE_MISC_ID);
4476
4477         /* Chip reset. */
4478         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
4479                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
4480                 REG_WR(sc, BCE_MISC_COMMAND, BCE_MISC_COMMAND_SW_RESET);
4481                 REG_RD(sc, BCE_MISC_COMMAND);
4482                 DELAY(5);
4483
4484                 val = BCE_PCICFG_MISC_CONFIG_REG_WINDOW_ENA |
4485                       BCE_PCICFG_MISC_CONFIG_TARGET_MB_WORD_SWAP;
4486
4487                 pci_write_config(sc->bce_dev, BCE_PCICFG_MISC_CONFIG, val, 4);
4488         } else {
4489                 val = BCE_PCICFG_MISC_CONFIG_CORE_RST_REQ |
4490                         BCE_PCICFG_MISC_CONFIG_REG_WINDOW_ENA |
4491                         BCE_PCICFG_MISC_CONFIG_TARGET_MB_WORD_SWAP;
4492                 REG_WR(sc, BCE_PCICFG_MISC_CONFIG, val);
4493
4494                 /* Allow up to 30us for reset to complete. */
4495                 for (i = 0; i < 10; i++) {
4496                         val = REG_RD(sc, BCE_PCICFG_MISC_CONFIG);
4497                         if ((val & (BCE_PCICFG_MISC_CONFIG_CORE_RST_REQ |
4498                                 BCE_PCICFG_MISC_CONFIG_CORE_RST_BSY)) == 0) {
4499                                 break;
4500                         }
4501                         DELAY(10);
4502                 }
4503
4504                 /* Check that reset completed successfully. */
4505                 if (val & (BCE_PCICFG_MISC_CONFIG_CORE_RST_REQ |
4506                         BCE_PCICFG_MISC_CONFIG_CORE_RST_BSY)) {
4507                         BCE_PRINTF("%s(%d): Reset failed!\n",
4508                                 __FILE__, __LINE__);
4509                         rc = EBUSY;
4510                         goto bce_reset_exit;
4511                 }
4512         }
4513
4514         /* Make sure byte swapping is properly configured. */
4515         val = REG_RD(sc, BCE_PCI_SWAP_DIAG0);
4516         if (val != 0x01020304) {
4517                 BCE_PRINTF("%s(%d): Byte swap is incorrect!\n",
4518                         __FILE__, __LINE__);
4519                 rc = ENODEV;
4520                 goto bce_reset_exit;
4521         }
4522
4523         /* Just completed a reset, assume that firmware is running again. */
4524         sc->bce_fw_timed_out = 0;
4525
4526         /* Wait for the firmware to finish its initialization. */
4527         rc = bce_fw_sync(sc, BCE_DRV_MSG_DATA_WAIT1 | reset_code);
4528         if (rc)
4529                 BCE_PRINTF("%s(%d): Firmware did not complete initialization!\n",
4530                         __FILE__, __LINE__);
4531
4532 bce_reset_exit:
4533         DBEXIT(BCE_VERBOSE_RESET);
4534         return (rc);
4535 }
4536
4537
4538 static int
4539 bce_chipinit(struct bce_softc *sc)
4540 {
4541         u32 val;
4542         int rc = 0;
4543
4544         DBENTER(BCE_VERBOSE_RESET);
4545
4546         bce_disable_intr(sc);
4547
4548         /*
4549          * Initialize DMA byte/word swapping, configure the number of DMA
4550          * channels and PCI clock compensation delay.
4551          */
4552         val = BCE_DMA_CONFIG_DATA_BYTE_SWAP |
4553               BCE_DMA_CONFIG_DATA_WORD_SWAP |
4554 #if BYTE_ORDER == BIG_ENDIAN
4555               BCE_DMA_CONFIG_CNTL_BYTE_SWAP |
4556 #endif
4557               BCE_DMA_CONFIG_CNTL_WORD_SWAP |
4558               DMA_READ_CHANS << 12 |
4559               DMA_WRITE_CHANS << 16;
4560
4561         val |= (0x2 << 20) | BCE_DMA_CONFIG_CNTL_PCI_COMP_DLY;
4562
4563         if ((sc->bce_flags & BCE_PCIX_FLAG) && (sc->bus_speed_mhz == 133))
4564                 val |= BCE_DMA_CONFIG_PCI_FAST_CLK_CMP;
4565
4566         /*
4567          * This setting resolves a problem observed on certain Intel PCI
4568          * chipsets that cannot handle multiple outstanding DMA operations.
4569          * See errata E9_5706A1_65.
4570          */
4571         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5706) &&
4572             (BCE_CHIP_ID(sc) != BCE_CHIP_ID_5706_A0) &&
4573             !(sc->bce_flags & BCE_PCIX_FLAG))
4574                 val |= BCE_DMA_CONFIG_CNTL_PING_PONG_DMA;
4575
4576         REG_WR(sc, BCE_DMA_CONFIG, val);
4577
4578         /* Enable the RX_V2P and Context state machines before access. */
4579         REG_WR(sc, BCE_MISC_ENABLE_SET_BITS,
4580                BCE_MISC_ENABLE_SET_BITS_HOST_COALESCE_ENABLE |
4581                BCE_MISC_ENABLE_STATUS_BITS_RX_V2P_ENABLE |
4582                BCE_MISC_ENABLE_STATUS_BITS_CONTEXT_ENABLE);
4583
4584         /* Initialize context mapping and zero out the quick contexts. */
4585         bce_init_ctx(sc);
4586
4587         /* Initialize the on-boards CPUs */
4588         bce_init_cpus(sc);
4589
4590         /* Prepare NVRAM for access. */
4591         if (bce_init_nvram(sc)) {
4592                 rc = ENODEV;
4593                 goto bce_chipinit_exit;
4594         }
4595
4596         /* Set the kernel bypass block size */
4597         val = REG_RD(sc, BCE_MQ_CONFIG);
4598         val &= ~BCE_MQ_CONFIG_KNL_BYP_BLK_SIZE;
4599         val |= BCE_MQ_CONFIG_KNL_BYP_BLK_SIZE_256;
4600
4601         /* Enable bins used on the 5709. */
4602         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
4603                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
4604                 val |= BCE_MQ_CONFIG_BIN_MQ_MODE;
4605                 if (BCE_CHIP_ID(sc) == BCE_CHIP_ID_5709_A1)
4606                         val |= BCE_MQ_CONFIG_HALT_DIS;
4607         }
4608
4609         REG_WR(sc, BCE_MQ_CONFIG, val);
4610
4611         val = 0x10000 + (MAX_CID_CNT * MB_KERNEL_CTX_SIZE);
4612         REG_WR(sc, BCE_MQ_KNL_BYP_WIND_START, val);
4613         REG_WR(sc, BCE_MQ_KNL_WIND_END, val);
4614
4615         /* Set the page size and clear the RV2P processor stall bits. */
4616         val = (BCM_PAGE_BITS - 8) << 24;
4617         REG_WR(sc, BCE_RV2P_CONFIG, val);
4618
4619         /* Configure page size. */
4620         val = REG_RD(sc, BCE_TBDR_CONFIG);
4621         val &= ~BCE_TBDR_CONFIG_PAGE_SIZE;
4622         val |= (BCM_PAGE_BITS - 8) << 24 | 0x40;
4623         REG_WR(sc, BCE_TBDR_CONFIG, val);
4624
4625         /* Set the perfect match control register to default. */
4626         REG_WR_IND(sc, BCE_RXP_PM_CTRL, 0);
4627
4628 bce_chipinit_exit:
4629         DBEXIT(BCE_VERBOSE_RESET);
4630
4631         return(rc);
4632 }
4633
4634
4635 /****************************************************************************/
4636 /* Initialize the controller in preparation to send/receive traffic.        */
4637 /*                                                                          */
4638 /* Returns:                                                                 */
4639 /*   0 for success, positive value for failure.                             */
4640 /****************************************************************************/
4641 static int
4642 bce_blockinit(struct bce_softc *sc)
4643 {
4644         u32 reg, val;
4645         int rc = 0;
4646
4647         DBENTER(BCE_VERBOSE_RESET);
4648
4649         /* Load the hardware default MAC address. */
4650         bce_set_mac_addr(sc);
4651
4652         /* Set the Ethernet backoff seed value */
4653         val = sc->eaddr[0]         + (sc->eaddr[1] << 8) +
4654               (sc->eaddr[2] << 16) + (sc->eaddr[3]     ) +
4655               (sc->eaddr[4] << 8)  + (sc->eaddr[5] << 16);
4656         REG_WR(sc, BCE_EMAC_BACKOFF_SEED, val);
4657
4658         sc->last_status_idx = 0;
4659         sc->rx_mode = BCE_EMAC_RX_MODE_SORT_MODE;
4660
4661         /* Set up link change interrupt generation. */
4662         REG_WR(sc, BCE_EMAC_ATTENTION_ENA, BCE_EMAC_ATTENTION_ENA_LINK);
4663
4664         /* Program the physical address of the status block. */
4665         REG_WR(sc, BCE_HC_STATUS_ADDR_L,
4666                 BCE_ADDR_LO(sc->status_block_paddr));
4667         REG_WR(sc, BCE_HC_STATUS_ADDR_H,
4668                 BCE_ADDR_HI(sc->status_block_paddr));
4669
4670         /* Program the physical address of the statistics block. */
4671         REG_WR(sc, BCE_HC_STATISTICS_ADDR_L,
4672                 BCE_ADDR_LO(sc->stats_block_paddr));
4673         REG_WR(sc, BCE_HC_STATISTICS_ADDR_H,
4674                 BCE_ADDR_HI(sc->stats_block_paddr));
4675
4676         /* Program various host coalescing parameters. */
4677         REG_WR(sc, BCE_HC_TX_QUICK_CONS_TRIP,
4678                 (sc->bce_tx_quick_cons_trip_int << 16) | sc->bce_tx_quick_cons_trip);
4679         REG_WR(sc, BCE_HC_RX_QUICK_CONS_TRIP,
4680                 (sc->bce_rx_quick_cons_trip_int << 16) | sc->bce_rx_quick_cons_trip);
4681         REG_WR(sc, BCE_HC_COMP_PROD_TRIP,
4682                 (sc->bce_comp_prod_trip_int << 16) | sc->bce_comp_prod_trip);
4683         REG_WR(sc, BCE_HC_TX_TICKS,
4684                 (sc->bce_tx_ticks_int << 16) | sc->bce_tx_ticks);
4685         REG_WR(sc, BCE_HC_RX_TICKS,
4686                 (sc->bce_rx_ticks_int << 16) | sc->bce_rx_ticks);
4687         REG_WR(sc, BCE_HC_COM_TICKS,
4688                 (sc->bce_com_ticks_int << 16) | sc->bce_com_ticks);
4689         REG_WR(sc, BCE_HC_CMD_TICKS,
4690                 (sc->bce_cmd_ticks_int << 16) | sc->bce_cmd_ticks);
4691         REG_WR(sc, BCE_HC_STATS_TICKS,
4692                 (sc->bce_stats_ticks & 0xffff00));
4693         REG_WR(sc, BCE_HC_STAT_COLLECT_TICKS, 0xbb8);  /* 3ms */
4694
4695         /* Configure the Host Coalescing block. */
4696         val = BCE_HC_CONFIG_RX_TMR_MODE | BCE_HC_CONFIG_TX_TMR_MODE |
4697                       BCE_HC_CONFIG_COLLECT_STATS;
4698
4699 #if 0
4700         /* ToDo: Add MSI-X support. */
4701         if (sc->bce_flags & BCE_USING_MSIX_FLAG) {
4702                 u32 base = ((BCE_TX_VEC - 1) * BCE_HC_SB_CONFIG_SIZE) +
4703                            BCE_HC_SB_CONFIG_1;
4704
4705                 REG_WR(sc, BCE_HC_MSIX_BIT_VECTOR, BCE_HC_MSIX_BIT_VECTOR_VAL);
4706
4707                 REG_WR(sc, base, BCE_HC_SB_CONFIG_1_TX_TMR_MODE |
4708                         BCE_HC_SB_CONFIG_1_ONE_SHOT);
4709
4710                 REG_WR(sc, base + BCE_HC_TX_QUICK_CONS_TRIP_OFF,
4711                         (sc->tx_quick_cons_trip_int << 16) |
4712                          sc->tx_quick_cons_trip);
4713
4714                 REG_WR(sc, base + BCE_HC_TX_TICKS_OFF,
4715                         (sc->tx_ticks_int << 16) | sc->tx_ticks);
4716
4717                 val |= BCE_HC_CONFIG_SB_ADDR_INC_128B;
4718         }
4719
4720         /*
4721          * Tell the HC block to automatically set the
4722          * INT_MASK bit after an MSI/MSI-X interrupt
4723          * is generated so the driver doesn't have to.
4724          */
4725         if (sc->bce_flags & BCE_ONE_SHOT_MSI_FLAG)
4726                 val |= BCE_HC_CONFIG_ONE_SHOT;
4727
4728         /* Set the MSI-X status blocks to 128 byte boundaries. */
4729         if (sc->bce_flags & BCE_USING_MSIX_FLAG)
4730                 val |= BCE_HC_CONFIG_SB_ADDR_INC_128B;
4731 #endif
4732
4733         REG_WR(sc, BCE_HC_CONFIG, val);
4734
4735         /* Clear the internal statistics counters. */
4736         REG_WR(sc, BCE_HC_COMMAND, BCE_HC_COMMAND_CLR_STAT_NOW);
4737
4738         /* Verify that bootcode is running. */
4739         reg = REG_RD_IND(sc, sc->bce_shmem_base + BCE_DEV_INFO_SIGNATURE);
4740
4741         DBRUNIF(DB_RANDOMTRUE(bootcode_running_failure_sim_control),
4742                 BCE_PRINTF("%s(%d): Simulating bootcode failure.\n",
4743                         __FILE__, __LINE__);
4744                 reg = 0);
4745
4746         if ((reg & BCE_DEV_INFO_SIGNATURE_MAGIC_MASK) !=
4747             BCE_DEV_INFO_SIGNATURE_MAGIC) {
4748                 BCE_PRINTF("%s(%d): Bootcode not running! Found: 0x%08X, "
4749                         "Expected: 08%08X\n", __FILE__, __LINE__,
4750                         (reg & BCE_DEV_INFO_SIGNATURE_MAGIC_MASK),
4751                         BCE_DEV_INFO_SIGNATURE_MAGIC);
4752                 rc = ENODEV;
4753                 goto bce_blockinit_exit;
4754         }
4755
4756         /* Enable DMA */
4757         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
4758                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
4759                 val = REG_RD(sc, BCE_MISC_NEW_CORE_CTL);
4760                 val |= BCE_MISC_NEW_CORE_CTL_DMA_ENABLE;
4761                 REG_WR(sc, BCE_MISC_NEW_CORE_CTL, val);
4762         }
4763
4764         /* Allow bootcode to apply any additional fixes before enabling MAC. */
4765         rc = bce_fw_sync(sc, BCE_DRV_MSG_DATA_WAIT2 | BCE_DRV_MSG_CODE_RESET);
4766
4767         /* Enable link state change interrupt generation. */
4768         REG_WR(sc, BCE_HC_ATTN_BITS_ENABLE, STATUS_ATTN_BITS_LINK_STATE);
4769
4770         /* Enable all remaining blocks in the MAC. */
4771         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709)     ||
4772                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716))
4773                 REG_WR(sc, BCE_MISC_ENABLE_SET_BITS, BCE_MISC_ENABLE_DEFAULT_XI);
4774         else
4775                 REG_WR(sc, BCE_MISC_ENABLE_SET_BITS, BCE_MISC_ENABLE_DEFAULT);
4776
4777         REG_RD(sc, BCE_MISC_ENABLE_SET_BITS);
4778         DELAY(20);
4779
4780         /* Save the current host coalescing block settings. */
4781         sc->hc_command = REG_RD(sc, BCE_HC_COMMAND);
4782
4783 bce_blockinit_exit:
4784         DBEXIT(BCE_VERBOSE_RESET);
4785
4786         return (rc);
4787 }
4788
4789
4790 /****************************************************************************/
4791 /* Encapsulate an mbuf into the rx_bd chain.                                */
4792 /*                                                                          */
4793 /* Returns:                                                                 */
4794 /*   0 for success, positive value for failure.                             */
4795 /****************************************************************************/
4796 static int
4797 bce_get_rx_buf(struct bce_softc *sc, struct mbuf *m, u16 *prod,
4798         u16 *chain_prod, u32 *prod_bseq)
4799 {
4800         bus_dmamap_t map;
4801         bus_dma_segment_t segs[BCE_MAX_SEGMENTS];
4802         struct mbuf *m_new = NULL;
4803         struct rx_bd *rxbd;
4804         int nsegs, error, rc = 0;
4805 #ifdef BCE_DEBUG
4806         u16 debug_chain_prod = *chain_prod;
4807 #endif
4808
4809         DBENTER(BCE_EXTREME_RESET | BCE_EXTREME_RECV | BCE_EXTREME_LOAD);
4810
4811         /* Make sure the inputs are valid. */
4812         DBRUNIF((*chain_prod > MAX_RX_BD),
4813                 BCE_PRINTF("%s(%d): RX producer out of range: 0x%04X > 0x%04X\n",
4814                 __FILE__, __LINE__, *chain_prod, (u16) MAX_RX_BD));
4815
4816         DBPRINT(sc, BCE_EXTREME_RECV, "%s(enter): prod = 0x%04X, chain_prod = 0x%04X, "
4817                 "prod_bseq = 0x%08X\n", __FUNCTION__, *prod, *chain_prod, *prod_bseq);
4818
4819         /* Update some debug statistic counters */
4820         DBRUNIF((sc->free_rx_bd < sc->rx_low_watermark),
4821                 sc->rx_low_watermark = sc->free_rx_bd);
4822         DBRUNIF((sc->free_rx_bd == sc->max_rx_bd), sc->rx_empty_count++);
4823
4824         /* Check whether this is a new mbuf allocation. */
4825         if (m == NULL) {
4826
4827                 /* Simulate an mbuf allocation failure. */
4828                 DBRUNIF(DB_RANDOMTRUE(mbuf_alloc_failed_sim_control),
4829                         sc->mbuf_alloc_failed_count++;
4830                         sc->mbuf_alloc_failed_sim_count++;
4831                         rc = ENOBUFS;
4832                         goto bce_get_rx_buf_exit);
4833
4834                 /* This is a new mbuf allocation. */
4835 #ifdef ZERO_COPY_SOCKETS
4836                 MGETHDR(m_new, M_DONTWAIT, MT_DATA);
4837 #else
4838                 if (sc->rx_bd_mbuf_alloc_size <= MCLBYTES)
4839                         m_new = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR);
4840                 else
4841                         m_new = m_getjcl(M_DONTWAIT, MT_DATA, M_PKTHDR, sc->rx_bd_mbuf_alloc_size);
4842 #endif
4843
4844                 if (m_new == NULL) {
4845                         sc->mbuf_alloc_failed_count++;
4846                         rc = ENOBUFS;
4847                         goto bce_get_rx_buf_exit;
4848                 }
4849
4850                 DBRUN(sc->debug_rx_mbuf_alloc++);
4851         } else {
4852                 /* Reuse an existing mbuf. */
4853                 m_new = m;
4854         }
4855
4856         /* Make sure we have a valid packet header. */
4857         M_ASSERTPKTHDR(m_new);
4858
4859         /* Initialize the mbuf size and pad if necessary for alignment. */
4860         m_new->m_pkthdr.len = m_new->m_len = sc->rx_bd_mbuf_alloc_size;
4861         m_adj(m_new, sc->rx_bd_mbuf_align_pad);
4862
4863         /* ToDo: Consider calling m_fragment() to test error handling. */
4864
4865         /* Map the mbuf cluster into device memory. */
4866         map = sc->rx_mbuf_map[*chain_prod];
4867         error = bus_dmamap_load_mbuf_sg(sc->rx_mbuf_tag, map, m_new,
4868             segs, &nsegs, BUS_DMA_NOWAIT);
4869
4870         /* Handle any mapping errors. */
4871         if (error) {
4872                 BCE_PRINTF("%s(%d): Error mapping mbuf into RX chain (%d)!\n",
4873                         __FILE__, __LINE__, error);
4874
4875                 sc->dma_map_addr_rx_failed_count++;
4876                 m_freem(m_new);
4877
4878                 DBRUN(sc->debug_rx_mbuf_alloc--);
4879
4880                 rc = ENOBUFS;
4881                 goto bce_get_rx_buf_exit;
4882         }
4883
4884         /* All mbufs must map to a single segment. */
4885         KASSERT(nsegs == 1, ("%s(): Too many segments returned (%d)!",
4886                  __FUNCTION__, nsegs));
4887
4888         /* ToDo: Do we need bus_dmamap_sync(,,BUS_DMASYNC_PREWRITE) here? */
4889
4890         /* Setup the rx_bd for the segment. */
4891         rxbd = &sc->rx_bd_chain[RX_PAGE(*chain_prod)][RX_IDX(*chain_prod)];
4892
4893         rxbd->rx_bd_haddr_lo  = htole32(BCE_ADDR_LO(segs[0].ds_addr));
4894         rxbd->rx_bd_haddr_hi  = htole32(BCE_ADDR_HI(segs[0].ds_addr));
4895         rxbd->rx_bd_len       = htole32(segs[0].ds_len);
4896         rxbd->rx_bd_flags     = htole32(RX_BD_FLAGS_START | RX_BD_FLAGS_END);
4897         *prod_bseq += segs[0].ds_len;
4898
4899         /* Save the mbuf and update our counter. */
4900         sc->rx_mbuf_ptr[*chain_prod] = m_new;
4901         sc->free_rx_bd -= nsegs;
4902
4903         DBRUNMSG(BCE_INSANE_RECV, bce_dump_rx_mbuf_chain(sc, debug_chain_prod,
4904                 nsegs));
4905
4906         DBPRINT(sc, BCE_EXTREME_RECV, "%s(exit): prod = 0x%04X, chain_prod = 0x%04X, "
4907                 "prod_bseq = 0x%08X\n", __FUNCTION__, *prod, *chain_prod, *prod_bseq);
4908
4909 bce_get_rx_buf_exit:
4910         DBEXIT(BCE_EXTREME_RESET | BCE_EXTREME_RECV | BCE_EXTREME_LOAD);
4911
4912         return(rc);
4913 }
4914
4915
4916 #ifdef ZERO_COPY_SOCKETS
4917 /****************************************************************************/
4918 /* Encapsulate an mbuf cluster into the page chain.                        */
4919 /*                                                                          */
4920 /* Returns:                                                                 */
4921 /*   0 for success, positive value for failure.                             */
4922 /****************************************************************************/
4923 static int
4924 bce_get_pg_buf(struct bce_softc *sc, struct mbuf *m, u16 *prod,
4925         u16 *prod_idx)
4926 {
4927         bus_dmamap_t map;
4928         bus_addr_t busaddr;
4929         struct mbuf *m_new = NULL;
4930         struct rx_bd *pgbd;
4931         int error, rc = 0;
4932 #ifdef BCE_DEBUG
4933         u16 debug_prod_idx = *prod_idx;
4934 #endif
4935
4936         DBENTER(BCE_EXTREME_RESET | BCE_EXTREME_RECV | BCE_EXTREME_LOAD);
4937
4938         /* Make sure the inputs are valid. */
4939         DBRUNIF((*prod_idx > MAX_PG_BD),
4940                 BCE_PRINTF("%s(%d): page producer out of range: 0x%04X > 0x%04X\n",
4941                 __FILE__, __LINE__, *prod_idx, (u16) MAX_PG_BD));
4942
4943         DBPRINT(sc, BCE_EXTREME_RECV, "%s(enter): prod = 0x%04X, "
4944                 "chain_prod = 0x%04X\n", __FUNCTION__, *prod, *prod_idx);
4945
4946         /* Update counters if we've hit a new low or run out of pages. */
4947         DBRUNIF((sc->free_pg_bd < sc->pg_low_watermark),
4948                 sc->pg_low_watermark = sc->free_pg_bd);
4949         DBRUNIF((sc->free_pg_bd == sc->max_pg_bd), sc->pg_empty_count++);
4950
4951         /* Check whether this is a new mbuf allocation. */
4952         if (m == NULL) {
4953
4954                 /* Simulate an mbuf allocation failure. */
4955                 DBRUNIF(DB_RANDOMTRUE(mbuf_alloc_failed_sim_control),
4956                         sc->mbuf_alloc_failed_count++;
4957                         sc->mbuf_alloc_failed_sim_count++;
4958                         rc = ENOBUFS;
4959                         goto bce_get_pg_buf_exit);
4960
4961                 /* This is a new mbuf allocation. */
4962                 m_new = m_getcl(M_DONTWAIT, MT_DATA, 0);
4963                 if (m_new == NULL) {
4964                         sc->mbuf_alloc_failed_count++;
4965                         rc = ENOBUFS;
4966                         goto bce_get_pg_buf_exit;
4967                 }
4968
4969                 DBRUN(sc->debug_pg_mbuf_alloc++);
4970         } else {
4971                 /* Reuse an existing mbuf. */
4972                 m_new = m;
4973                 m_new->m_data = m_new->m_ext.ext_buf;
4974         }
4975
4976         m_new->m_len = sc->pg_bd_mbuf_alloc_size;
4977
4978         /* ToDo: Consider calling m_fragment() to test error handling. */
4979
4980         /* Map the mbuf cluster into device memory. */
4981         map = sc->pg_mbuf_map[*prod_idx];
4982         error = bus_dmamap_load(sc->pg_mbuf_tag, map, mtod(m_new, void *),
4983             sc->pg_bd_mbuf_alloc_size, bce_dma_map_addr, &busaddr, BUS_DMA_NOWAIT);
4984
4985         /* Handle any mapping errors. */
4986         if (error) {
4987                 BCE_PRINTF("%s(%d): Error mapping mbuf into page chain!\n",
4988                         __FILE__, __LINE__);
4989
4990                 m_freem(m_new);
4991                 DBRUN(sc->debug_pg_mbuf_alloc--);
4992
4993                 rc = ENOBUFS;
4994                 goto bce_get_pg_buf_exit;
4995         }
4996
4997         /* ToDo: Do we need bus_dmamap_sync(,,BUS_DMASYNC_PREWRITE) here? */
4998
4999         /*
5000          * The page chain uses the same rx_bd data structure
5001          * as the receive chain but doesn't require a byte sequence (bseq).
5002          */
5003         pgbd = &sc->pg_bd_chain[PG_PAGE(*prod_idx)][PG_IDX(*prod_idx)];
5004
5005         pgbd->rx_bd_haddr_lo  = htole32(BCE_ADDR_LO(busaddr));
5006         pgbd->rx_bd_haddr_hi  = htole32(BCE_ADDR_HI(busaddr));
5007         pgbd->rx_bd_len       = htole32(sc->pg_bd_mbuf_alloc_size);
5008         pgbd->rx_bd_flags     = htole32(RX_BD_FLAGS_START | RX_BD_FLAGS_END);
5009
5010         /* Save the mbuf and update our counter. */
5011         sc->pg_mbuf_ptr[*prod_idx] = m_new;
5012         sc->free_pg_bd--;
5013
5014         DBRUNMSG(BCE_INSANE_RECV, bce_dump_pg_mbuf_chain(sc, debug_prod_idx,
5015                 1));
5016
5017         DBPRINT(sc, BCE_EXTREME_RECV, "%s(exit): prod = 0x%04X, "
5018                 "prod_idx = 0x%04X\n", __FUNCTION__, *prod, *prod_idx);
5019
5020 bce_get_pg_buf_exit:
5021         DBEXIT(BCE_EXTREME_RESET | BCE_EXTREME_RECV | BCE_EXTREME_LOAD);
5022
5023         return(rc);
5024 }
5025 #endif /* ZERO_COPY_SOCKETS */
5026
5027 /****************************************************************************/
5028 /* Initialize the TX context memory.                                        */
5029 /*                                                                          */
5030 /* Returns:                                                                 */
5031 /*   Nothing                                                                */
5032 /****************************************************************************/
5033 static void
5034 bce_init_tx_context(struct bce_softc *sc)
5035 {
5036         u32 val;
5037
5038         DBENTER(BCE_VERBOSE_RESET | BCE_VERBOSE_SEND | BCE_VERBOSE_CTX);
5039
5040         /* Initialize the context ID for an L2 TX chain. */
5041         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
5042                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
5043                 /* Set the CID type to support an L2 connection. */
5044                 val = BCE_L2CTX_TX_TYPE_TYPE_L2_XI | BCE_L2CTX_TX_TYPE_SIZE_L2_XI;
5045                 CTX_WR(sc, GET_CID_ADDR(TX_CID), BCE_L2CTX_TX_TYPE_XI, val);
5046                 val = BCE_L2CTX_TX_CMD_TYPE_TYPE_L2_XI | (8 << 16);
5047                 CTX_WR(sc, GET_CID_ADDR(TX_CID), BCE_L2CTX_TX_CMD_TYPE_XI, val);
5048
5049                 /* Point the hardware to the first page in the chain. */
5050                 val = BCE_ADDR_HI(sc->tx_bd_chain_paddr[0]);
5051                 CTX_WR(sc, GET_CID_ADDR(TX_CID), BCE_L2CTX_TX_TBDR_BHADDR_HI_XI, val);
5052                 val = BCE_ADDR_LO(sc->tx_bd_chain_paddr[0]);
5053                 CTX_WR(sc, GET_CID_ADDR(TX_CID), BCE_L2CTX_TX_TBDR_BHADDR_LO_XI, val);
5054         } else {
5055                 /* Set the CID type to support an L2 connection. */
5056                 val = BCE_L2CTX_TX_TYPE_TYPE_L2 | BCE_L2CTX_TX_TYPE_SIZE_L2;
5057                 CTX_WR(sc, GET_CID_ADDR(TX_CID), BCE_L2CTX_TX_TYPE, val);
5058                 val = BCE_L2CTX_TX_CMD_TYPE_TYPE_L2 | (8 << 16);
5059                 CTX_WR(sc, GET_CID_ADDR(TX_CID), BCE_L2CTX_TX_CMD_TYPE, val);
5060
5061                 /* Point the hardware to the first page in the chain. */
5062                 val = BCE_ADDR_HI(sc->tx_bd_chain_paddr[0]);
5063                 CTX_WR(sc, GET_CID_ADDR(TX_CID), BCE_L2CTX_TX_TBDR_BHADDR_HI, val);
5064                 val = BCE_ADDR_LO(sc->tx_bd_chain_paddr[0]);
5065                 CTX_WR(sc, GET_CID_ADDR(TX_CID), BCE_L2CTX_TX_TBDR_BHADDR_LO, val);
5066         }
5067
5068         DBEXIT(BCE_VERBOSE_RESET | BCE_VERBOSE_SEND | BCE_VERBOSE_CTX);
5069 }
5070
5071
5072 /****************************************************************************/
5073 /* Allocate memory and initialize the TX data structures.                   */
5074 /*                                                                          */
5075 /* Returns:                                                                 */
5076 /*   0 for success, positive value for failure.                             */
5077 /****************************************************************************/
5078 static int
5079 bce_init_tx_chain(struct bce_softc *sc)
5080 {
5081         struct tx_bd *txbd;
5082         int i, rc = 0;
5083
5084         DBENTER(BCE_VERBOSE_RESET | BCE_VERBOSE_SEND | BCE_VERBOSE_LOAD);
5085
5086         /* Set the initial TX producer/consumer indices. */
5087         sc->tx_prod        = 0;
5088         sc->tx_cons        = 0;
5089         sc->tx_prod_bseq   = 0;
5090         sc->used_tx_bd     = 0;
5091         sc->max_tx_bd      = USABLE_TX_BD;
5092         DBRUN(sc->tx_hi_watermark = USABLE_TX_BD);
5093         DBRUN(sc->tx_full_count = 0);
5094
5095         /*
5096          * The NetXtreme II supports a linked-list structre called
5097          * a Buffer Descriptor Chain (or BD chain).  A BD chain
5098          * consists of a series of 1 or more chain pages, each of which
5099          * consists of a fixed number of BD entries.
5100          * The last BD entry on each page is a pointer to the next page
5101          * in the chain, and the last pointer in the BD chain
5102          * points back to the beginning of the chain.
5103          */
5104
5105         /* Set the TX next pointer chain entries. */
5106         for (i = 0; i < TX_PAGES; i++) {
5107                 int j;
5108
5109                 txbd = &sc->tx_bd_chain[i][USABLE_TX_BD_PER_PAGE];
5110
5111                 /* Check if we've reached the last page. */
5112                 if (i == (TX_PAGES - 1))
5113                         j = 0;
5114                 else
5115                         j = i + 1;
5116
5117                 txbd->tx_bd_haddr_hi = htole32(BCE_ADDR_HI(sc->tx_bd_chain_paddr[j]));
5118                 txbd->tx_bd_haddr_lo = htole32(BCE_ADDR_LO(sc->tx_bd_chain_paddr[j]));
5119         }
5120
5121         bce_init_tx_context(sc);
5122
5123         DBRUNMSG(BCE_INSANE_SEND, bce_dump_tx_chain(sc, 0, TOTAL_TX_BD));
5124         DBEXIT(BCE_VERBOSE_RESET | BCE_VERBOSE_SEND | BCE_VERBOSE_LOAD);
5125
5126         return(rc);
5127 }
5128
5129
5130 /****************************************************************************/
5131 /* Free memory and clear the TX data structures.                            */
5132 /*                                                                          */
5133 /* Returns:                                                                 */
5134 /*   Nothing.                                                               */
5135 /****************************************************************************/
5136 static void
5137 bce_free_tx_chain(struct bce_softc *sc)
5138 {
5139         int i;
5140
5141         DBENTER(BCE_VERBOSE_RESET | BCE_VERBOSE_SEND | BCE_VERBOSE_UNLOAD);
5142
5143         /* Unmap, unload, and free any mbufs still in the TX mbuf chain. */
5144         for (i = 0; i < TOTAL_TX_BD; i++) {
5145                 if (sc->tx_mbuf_ptr[i] != NULL) {
5146                         if (sc->tx_mbuf_map[i] != NULL)
5147                                 bus_dmamap_sync(sc->tx_mbuf_tag, sc->tx_mbuf_map[i],
5148                                         BUS_DMASYNC_POSTWRITE);
5149                         m_freem(sc->tx_mbuf_ptr[i]);
5150                         sc->tx_mbuf_ptr[i] = NULL;
5151                         DBRUN(sc->debug_tx_mbuf_alloc--);
5152                 }
5153         }
5154
5155         /* Clear each TX chain page. */
5156         for (i = 0; i < TX_PAGES; i++)
5157                 bzero((char *)sc->tx_bd_chain[i], BCE_TX_CHAIN_PAGE_SZ);
5158
5159         sc->used_tx_bd     = 0;
5160
5161         /* Check if we lost any mbufs in the process. */
5162         DBRUNIF((sc->debug_tx_mbuf_alloc),
5163                 BCE_PRINTF("%s(%d): Memory leak! Lost %d mbufs "
5164                         "from tx chain!\n",
5165                         __FILE__, __LINE__, sc->debug_tx_mbuf_alloc));
5166
5167         DBEXIT(BCE_VERBOSE_RESET | BCE_VERBOSE_SEND | BCE_VERBOSE_UNLOAD);
5168 }
5169
5170
5171 /****************************************************************************/
5172 /* Initialize the RX context memory.                                        */
5173 /*                                                                          */
5174 /* Returns:                                                                 */
5175 /*   Nothing                                                                */
5176 /****************************************************************************/
5177 static void
5178 bce_init_rx_context(struct bce_softc *sc)
5179 {
5180         u32 val;
5181
5182         DBENTER(BCE_VERBOSE_RESET | BCE_VERBOSE_RECV | BCE_VERBOSE_CTX);
5183
5184         /* Initialize the type, size, and BD cache levels for the RX context. */
5185         val = BCE_L2CTX_RX_CTX_TYPE_CTX_BD_CHN_TYPE_VALUE |
5186                 BCE_L2CTX_RX_CTX_TYPE_SIZE_L2 |
5187                 (0x02 << BCE_L2CTX_RX_BD_PRE_READ_SHIFT);
5188
5189         /*
5190          * Set the level for generating pause frames
5191          * when the number of available rx_bd's gets
5192          * too low (the low watermark) and the level
5193          * when pause frames can be stopped (the high
5194          * watermark).
5195          */
5196         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
5197                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
5198                 u32 lo_water, hi_water;
5199
5200                 lo_water = BCE_L2CTX_RX_LO_WATER_MARK_DEFAULT;
5201                 hi_water = USABLE_RX_BD / 4;
5202
5203                 lo_water /= BCE_L2CTX_RX_LO_WATER_MARK_SCALE;
5204                 hi_water /= BCE_L2CTX_RX_HI_WATER_MARK_SCALE;
5205
5206                 if (hi_water > 0xf)
5207                         hi_water = 0xf;
5208                 else if (hi_water == 0)
5209                         lo_water = 0;
5210                 val |= (lo_water << BCE_L2CTX_RX_LO_WATER_MARK_SHIFT) |
5211                         (hi_water << BCE_L2CTX_RX_HI_WATER_MARK_SHIFT);
5212         }
5213
5214         CTX_WR(sc, GET_CID_ADDR(RX_CID), BCE_L2CTX_RX_CTX_TYPE, val);
5215
5216         /* Setup the MQ BIN mapping for l2_ctx_host_bseq. */
5217         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
5218                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
5219                 val = REG_RD(sc, BCE_MQ_MAP_L2_5);
5220                 REG_WR(sc, BCE_MQ_MAP_L2_5, val | BCE_MQ_MAP_L2_5_ARM);
5221         }
5222
5223         /* Point the hardware to the first page in the chain. */
5224         val = BCE_ADDR_HI(sc->rx_bd_chain_paddr[0]);
5225         CTX_WR(sc, GET_CID_ADDR(RX_CID), BCE_L2CTX_RX_NX_BDHADDR_HI, val);
5226         val = BCE_ADDR_LO(sc->rx_bd_chain_paddr[0]);
5227         CTX_WR(sc, GET_CID_ADDR(RX_CID), BCE_L2CTX_RX_NX_BDHADDR_LO, val);
5228
5229         DBEXIT(BCE_VERBOSE_RESET | BCE_VERBOSE_RECV | BCE_VERBOSE_CTX);
5230 }
5231
5232
5233 /****************************************************************************/
5234 /* Allocate memory and initialize the RX data structures.                   */
5235 /*                                                                          */
5236 /* Returns:                                                                 */
5237 /*   0 for success, positive value for failure.                             */
5238 /****************************************************************************/
5239 static int
5240 bce_init_rx_chain(struct bce_softc *sc)
5241 {
5242         struct rx_bd *rxbd;
5243         int i, rc = 0;
5244
5245         DBENTER(BCE_VERBOSE_RESET | BCE_VERBOSE_RECV | BCE_VERBOSE_LOAD |
5246                 BCE_VERBOSE_CTX);
5247
5248         /* Initialize the RX producer and consumer indices. */
5249         sc->rx_prod        = 0;
5250         sc->rx_cons        = 0;
5251         sc->rx_prod_bseq   = 0;
5252         sc->free_rx_bd     = USABLE_RX_BD;
5253         sc->max_rx_bd      = USABLE_RX_BD;
5254         DBRUN(sc->rx_low_watermark = sc->max_rx_bd);
5255         DBRUN(sc->rx_empty_count = 0);
5256
5257         /* Initialize the RX next pointer chain entries. */
5258         for (i = 0; i < RX_PAGES; i++) {
5259                 int j;
5260
5261                 rxbd = &sc->rx_bd_chain[i][USABLE_RX_BD_PER_PAGE];
5262
5263                 /* Check if we've reached the last page. */
5264                 if (i == (RX_PAGES - 1))
5265                         j = 0;
5266                 else
5267                         j = i + 1;
5268
5269                 /* Setup the chain page pointers. */
5270                 rxbd->rx_bd_haddr_hi = htole32(BCE_ADDR_HI(sc->rx_bd_chain_paddr[j]));
5271                 rxbd->rx_bd_haddr_lo = htole32(BCE_ADDR_LO(sc->rx_bd_chain_paddr[j]));
5272         }
5273
5274 /* Fill up the RX chain. */
5275         bce_fill_rx_chain(sc);
5276
5277         for (i = 0; i < RX_PAGES; i++) {
5278                 bus_dmamap_sync(
5279                         sc->rx_bd_chain_tag,
5280                 sc->rx_bd_chain_map[i],
5281                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
5282         }
5283
5284         bce_init_rx_context(sc);
5285
5286         DBRUNMSG(BCE_EXTREME_RECV, bce_dump_rx_chain(sc, 0, TOTAL_RX_BD));
5287         DBEXIT(BCE_VERBOSE_RESET | BCE_VERBOSE_RECV | BCE_VERBOSE_LOAD |
5288                 BCE_VERBOSE_CTX);
5289         /* ToDo: Are there possible failure modes here? */
5290         return(rc);
5291 }
5292
5293
5294 /****************************************************************************/
5295 /* Add mbufs to the RX chain until its full or an mbuf allocation error     */
5296 /* occurs.                                                                  */
5297 /*                                                                          */
5298 /* Returns:                                                                 */
5299 /*   Nothing                                                                */
5300 /****************************************************************************/
5301 static void
5302 bce_fill_rx_chain(struct bce_softc *sc)
5303 {
5304         u16 prod, prod_idx;
5305         u32 prod_bseq;
5306
5307         DBENTER(BCE_VERBOSE_RESET | BCE_EXTREME_RECV | BCE_VERBOSE_LOAD |
5308                 BCE_VERBOSE_CTX);
5309
5310         /* Get the RX chain producer indices. */
5311         prod      = sc->rx_prod;
5312         prod_bseq = sc->rx_prod_bseq;
5313
5314         /* Keep filling the RX chain until it's full. */
5315         while (sc->free_rx_bd > 0) {
5316                 prod_idx = RX_CHAIN_IDX(prod);
5317                 if (bce_get_rx_buf(sc, NULL, &prod, &prod_idx, &prod_bseq)) {
5318                         /* Bail out if we can't add an mbuf to the chain. */
5319                         break;
5320                 }
5321                 prod = NEXT_RX_BD(prod);
5322         }
5323
5324         /* Save the RX chain producer indices. */
5325         sc->rx_prod      = prod;
5326         sc->rx_prod_bseq = prod_bseq;
5327
5328         DBRUNIF(((prod & USABLE_RX_BD_PER_PAGE) == USABLE_RX_BD_PER_PAGE),
5329                 BCE_PRINTF("%s(): Invalid rx_prod value: 0x%04X\n",
5330                 __FUNCTION__, sc->rx_prod));
5331
5332         /* Write the mailbox and tell the chip about the waiting rx_bd's. */
5333         REG_WR16(sc, MB_GET_CID_ADDR(RX_CID) + BCE_L2MQ_RX_HOST_BDIDX,
5334                 sc->rx_prod);
5335         REG_WR(sc, MB_GET_CID_ADDR(RX_CID) + BCE_L2MQ_RX_HOST_BSEQ,
5336                 sc->rx_prod_bseq);
5337
5338         DBEXIT(BCE_VERBOSE_RESET | BCE_EXTREME_RECV | BCE_VERBOSE_LOAD |
5339                 BCE_VERBOSE_CTX);
5340 }
5341
5342
5343 /****************************************************************************/
5344 /* Free memory and clear the RX data structures.                            */
5345 /*                                                                          */
5346 /* Returns:                                                                 */
5347 /*   Nothing.                                                               */
5348 /****************************************************************************/
5349 static void
5350 bce_free_rx_chain(struct bce_softc *sc)
5351 {
5352         int i;
5353
5354         DBENTER(BCE_VERBOSE_RESET | BCE_VERBOSE_RECV | BCE_VERBOSE_UNLOAD);
5355
5356         /* Free any mbufs still in the RX mbuf chain. */
5357         for (i = 0; i < TOTAL_RX_BD; i++) {
5358                 if (sc->rx_mbuf_ptr[i] != NULL) {
5359                         if (sc->rx_mbuf_map[i] != NULL)
5360                                 bus_dmamap_sync(sc->rx_mbuf_tag, sc->rx_mbuf_map[i],
5361                                         BUS_DMASYNC_POSTREAD);
5362                         m_freem(sc->rx_mbuf_ptr[i]);
5363                         sc->rx_mbuf_ptr[i] = NULL;
5364                         DBRUN(sc->debug_rx_mbuf_alloc--);
5365                 }
5366         }
5367
5368         /* Clear each RX chain page. */
5369         for (i = 0; i < RX_PAGES; i++)
5370                 bzero((char *)sc->rx_bd_chain[i], BCE_RX_CHAIN_PAGE_SZ);
5371
5372         sc->free_rx_bd = sc->max_rx_bd;
5373
5374         /* Check if we lost any mbufs in the process. */
5375         DBRUNIF((sc->debug_rx_mbuf_alloc),
5376                 BCE_PRINTF("%s(): Memory leak! Lost %d mbufs from rx chain!\n",
5377                         __FUNCTION__, sc->debug_rx_mbuf_alloc));
5378
5379         DBEXIT(BCE_VERBOSE_RESET | BCE_VERBOSE_RECV | BCE_VERBOSE_UNLOAD);
5380 }
5381
5382
5383 #ifdef ZERO_COPY_SOCKETS
5384 /****************************************************************************/
5385 /* Allocate memory and initialize the page data structures.                 */
5386 /* Assumes that bce_init_rx_chain() has not already been called.            */
5387 /*                                                                          */
5388 /* Returns:                                                                 */
5389 /*   0 for success, positive value for failure.                             */
5390 /****************************************************************************/
5391 static int
5392 bce_init_pg_chain(struct bce_softc *sc)
5393 {
5394         struct rx_bd *pgbd;
5395         int i, rc = 0;
5396         u32 val;
5397
5398         DBENTER(BCE_VERBOSE_RESET | BCE_VERBOSE_RECV | BCE_VERBOSE_LOAD |
5399                 BCE_VERBOSE_CTX);
5400
5401         /* Initialize the page producer and consumer indices. */
5402         sc->pg_prod        = 0;
5403         sc->pg_cons        = 0;
5404         sc->free_pg_bd     = USABLE_PG_BD;
5405         sc->max_pg_bd      = USABLE_PG_BD;
5406         DBRUN(sc->pg_low_watermark = sc->max_pg_bd);
5407         DBRUN(sc->pg_empty_count = 0);
5408
5409         /* Initialize the page next pointer chain entries. */
5410         for (i = 0; i < PG_PAGES; i++) {
5411                 int j;
5412
5413                 pgbd = &sc->pg_bd_chain[i][USABLE_PG_BD_PER_PAGE];
5414
5415                 /* Check if we've reached the last page. */
5416                 if (i == (PG_PAGES - 1))
5417                         j = 0;
5418                 else
5419                         j = i + 1;
5420
5421                 /* Setup the chain page pointers. */
5422                 pgbd->rx_bd_haddr_hi = htole32(BCE_ADDR_HI(sc->pg_bd_chain_paddr[j]));
5423                 pgbd->rx_bd_haddr_lo = htole32(BCE_ADDR_LO(sc->pg_bd_chain_paddr[j]));
5424         }
5425
5426         /* Setup the MQ BIN mapping for host_pg_bidx. */
5427         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709)     ||
5428                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716))
5429                 REG_WR(sc, BCE_MQ_MAP_L2_3, BCE_MQ_MAP_L2_3_DEFAULT);
5430
5431         CTX_WR(sc, GET_CID_ADDR(RX_CID), BCE_L2CTX_RX_PG_BUF_SIZE, 0);
5432
5433         /* Configure the rx_bd and page chain mbuf cluster size. */
5434         val = (sc->rx_bd_mbuf_data_len << 16) | sc->pg_bd_mbuf_alloc_size;
5435         CTX_WR(sc, GET_CID_ADDR(RX_CID), BCE_L2CTX_RX_PG_BUF_SIZE, val);
5436
5437         /* Configure the context reserved for jumbo support. */
5438         CTX_WR(sc, GET_CID_ADDR(RX_CID), BCE_L2CTX_RX_RBDC_KEY,
5439                 BCE_L2CTX_RX_RBDC_JUMBO_KEY);
5440
5441         /* Point the hardware to the first page in the page chain. */
5442         val = BCE_ADDR_HI(sc->pg_bd_chain_paddr[0]);
5443         CTX_WR(sc, GET_CID_ADDR(RX_CID), BCE_L2CTX_RX_NX_PG_BDHADDR_HI, val);
5444         val = BCE_ADDR_LO(sc->pg_bd_chain_paddr[0]);
5445         CTX_WR(sc, GET_CID_ADDR(RX_CID), BCE_L2CTX_RX_NX_PG_BDHADDR_LO, val);
5446
5447         /* Fill up the page chain. */
5448         bce_fill_pg_chain(sc);
5449
5450         for (i = 0; i < PG_PAGES; i++) {
5451                 bus_dmamap_sync(
5452                         sc->pg_bd_chain_tag,
5453                 sc->pg_bd_chain_map[i],
5454                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
5455         }
5456
5457         DBRUNMSG(BCE_EXTREME_RECV, bce_dump_pg_chain(sc, 0, TOTAL_PG_BD));
5458         DBEXIT(BCE_VERBOSE_RESET | BCE_VERBOSE_RECV | BCE_VERBOSE_LOAD |
5459                 BCE_VERBOSE_CTX);
5460         return(rc);
5461 }
5462
5463
5464 /****************************************************************************/
5465 /* Add mbufs to the page chain until its full or an mbuf allocation error   */
5466 /* occurs.                                                                  */
5467 /*                                                                          */
5468 /* Returns:                                                                 */
5469 /*   Nothing                                                                */
5470 /****************************************************************************/
5471 static void
5472 bce_fill_pg_chain(struct bce_softc *sc)
5473 {
5474         u16 prod, prod_idx;
5475
5476         DBENTER(BCE_VERBOSE_RESET | BCE_EXTREME_RECV | BCE_VERBOSE_LOAD |
5477                 BCE_VERBOSE_CTX);
5478
5479         /* Get the page chain prodcuer index. */
5480         prod = sc->pg_prod;
5481
5482         /* Keep filling the page chain until it's full. */
5483         while (sc->free_pg_bd > 0) {
5484                 prod_idx = PG_CHAIN_IDX(prod);
5485                 if (bce_get_pg_buf(sc, NULL, &prod, &prod_idx)) {
5486                         /* Bail out if we can't add an mbuf to the chain. */
5487                         break;
5488                 }
5489                 prod = NEXT_PG_BD(prod);
5490         }
5491
5492         /* Save the page chain producer index. */
5493         sc->pg_prod = prod;
5494
5495         DBRUNIF(((prod & USABLE_RX_BD_PER_PAGE) == USABLE_RX_BD_PER_PAGE),
5496                 BCE_PRINTF("%s(): Invalid pg_prod value: 0x%04X\n",
5497                 __FUNCTION__, sc->pg_prod));
5498
5499         /*
5500          * Write the mailbox and tell the chip about
5501          * the new rx_bd's in the page chain.
5502          */
5503         REG_WR16(sc, MB_GET_CID_ADDR(RX_CID) + BCE_L2MQ_RX_HOST_PG_BDIDX,
5504                 sc->pg_prod);
5505
5506         DBEXIT(BCE_VERBOSE_RESET | BCE_EXTREME_RECV | BCE_VERBOSE_LOAD |
5507                 BCE_VERBOSE_CTX);
5508 }
5509
5510
5511 /****************************************************************************/
5512 /* Free memory and clear the RX data structures.                            */
5513 /*                                                                          */
5514 /* Returns:                                                                 */
5515 /*   Nothing.                                                               */
5516 /****************************************************************************/
5517 static void
5518 bce_free_pg_chain(struct bce_softc *sc)
5519 {
5520         int i;
5521
5522         DBENTER(BCE_VERBOSE_RESET | BCE_VERBOSE_RECV | BCE_VERBOSE_UNLOAD);
5523
5524         /* Free any mbufs still in the mbuf page chain. */
5525         for (i = 0; i < TOTAL_PG_BD; i++) {
5526                 if (sc->pg_mbuf_ptr[i] != NULL) {
5527                         if (sc->pg_mbuf_map[i] != NULL)
5528                                 bus_dmamap_sync(sc->pg_mbuf_tag, sc->pg_mbuf_map[i],
5529                                         BUS_DMASYNC_POSTREAD);
5530                         m_freem(sc->pg_mbuf_ptr[i]);
5531                         sc->pg_mbuf_ptr[i] = NULL;
5532                         DBRUN(sc->debug_pg_mbuf_alloc--);
5533                 }
5534         }
5535
5536         /* Clear each page chain pages. */
5537         for (i = 0; i < PG_PAGES; i++)
5538                 bzero((char *)sc->pg_bd_chain[i], BCE_PG_CHAIN_PAGE_SZ);
5539
5540         sc->free_pg_bd = sc->max_pg_bd;
5541
5542         /* Check if we lost any mbufs in the process. */
5543         DBRUNIF((sc->debug_pg_mbuf_alloc),
5544                 BCE_PRINTF("%s(): Memory leak! Lost %d mbufs from page chain!\n",
5545                         __FUNCTION__, sc->debug_pg_mbuf_alloc));
5546
5547         DBEXIT(BCE_VERBOSE_RESET | BCE_VERBOSE_RECV | BCE_VERBOSE_UNLOAD);
5548 }
5549 #endif /* ZERO_COPY_SOCKETS */
5550
5551
5552 /****************************************************************************/
5553 /* Set media options.                                                       */
5554 /*                                                                          */
5555 /* Returns:                                                                 */
5556 /*   0 for success, positive value for failure.                             */
5557 /****************************************************************************/
5558 static int
5559 bce_ifmedia_upd(struct ifnet *ifp)
5560 {
5561         struct bce_softc *sc = ifp->if_softc;
5562
5563         DBENTER(BCE_VERBOSE);
5564
5565         BCE_LOCK(sc);
5566         bce_ifmedia_upd_locked(ifp);
5567         BCE_UNLOCK(sc);
5568
5569         DBEXIT(BCE_VERBOSE);
5570         return (0);
5571 }
5572
5573
5574 /****************************************************************************/
5575 /* Set media options.                                                       */
5576 /*                                                                          */
5577 /* Returns:                                                                 */
5578 /*   Nothing.                                                               */
5579 /****************************************************************************/
5580 static void
5581 bce_ifmedia_upd_locked(struct ifnet *ifp)
5582 {
5583         struct bce_softc *sc = ifp->if_softc;
5584         struct mii_data *mii;
5585
5586         DBENTER(BCE_VERBOSE);
5587
5588         BCE_LOCK_ASSERT(sc);
5589
5590         mii = device_get_softc(sc->bce_miibus);
5591
5592         /* Make sure the MII bus has been enumerated. */
5593         if (mii) {
5594                 sc->bce_link = 0;
5595                 if (mii->mii_instance) {
5596                         struct mii_softc *miisc;
5597
5598                         LIST_FOREACH(miisc, &mii->mii_phys, mii_list)
5599                                 mii_phy_reset(miisc);
5600                 }
5601                 mii_mediachg(mii);
5602         }
5603
5604         DBEXIT(BCE_VERBOSE);
5605 }
5606
5607
5608 /****************************************************************************/
5609 /* Reports current media status.                                            */
5610 /*                                                                          */
5611 /* Returns:                                                                 */
5612 /*   Nothing.                                                               */
5613 /****************************************************************************/
5614 static void
5615 bce_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
5616 {
5617         struct bce_softc *sc = ifp->if_softc;
5618         struct mii_data *mii;
5619
5620         DBENTER(BCE_VERBOSE);
5621
5622         BCE_LOCK(sc);
5623
5624         mii = device_get_softc(sc->bce_miibus);
5625
5626         mii_pollstat(mii);
5627         ifmr->ifm_active = mii->mii_media_active;
5628         ifmr->ifm_status = mii->mii_media_status;
5629
5630         BCE_UNLOCK(sc);
5631
5632         DBEXIT(BCE_VERBOSE);
5633 }
5634
5635
5636 /****************************************************************************/
5637 /* Handles PHY generated interrupt events.                                  */
5638 /*                                                                          */
5639 /* Returns:                                                                 */
5640 /*   Nothing.                                                               */
5641 /****************************************************************************/
5642 static void
5643 bce_phy_intr(struct bce_softc *sc)
5644 {
5645         u32 new_link_state, old_link_state;
5646
5647         DBENTER(BCE_VERBOSE_PHY | BCE_VERBOSE_INTR);
5648
5649         new_link_state = sc->status_block->status_attn_bits &
5650                 STATUS_ATTN_BITS_LINK_STATE;
5651         old_link_state = sc->status_block->status_attn_bits_ack &
5652                 STATUS_ATTN_BITS_LINK_STATE;
5653
5654         /* Handle any changes if the link state has changed. */
5655         if (new_link_state != old_link_state) {
5656
5657                 /* Update the status_attn_bits_ack field in the status block. */
5658                 if (new_link_state) {
5659                         REG_WR(sc, BCE_PCICFG_STATUS_BIT_SET_CMD,
5660                                 STATUS_ATTN_BITS_LINK_STATE);
5661                         DBPRINT(sc, BCE_INFO_PHY, "%s(): Link is now UP.\n",
5662                                 __FUNCTION__);
5663                 }
5664                 else {
5665                         REG_WR(sc, BCE_PCICFG_STATUS_BIT_CLEAR_CMD,
5666                                 STATUS_ATTN_BITS_LINK_STATE);
5667                         DBPRINT(sc, BCE_INFO_PHY, "%s(): Link is now DOWN.\n",
5668                                 __FUNCTION__);
5669                 }
5670
5671                 /*
5672                  * Assume link is down and allow
5673                  * tick routine to update the state
5674                  * based on the actual media state.
5675                  */
5676                 sc->bce_link = 0;
5677                 callout_stop(&sc->bce_tick_callout);
5678                 bce_tick(sc);
5679         }
5680
5681         /* Acknowledge the link change interrupt. */
5682         REG_WR(sc, BCE_EMAC_STATUS, BCE_EMAC_STATUS_LINK_CHANGE);
5683
5684         DBEXIT(BCE_VERBOSE_PHY | BCE_VERBOSE_INTR);
5685 }
5686
5687
5688 /****************************************************************************/
5689 /* Reads the receive consumer value from the status block (skipping over    */
5690 /* chain page pointer if necessary).                                        */
5691 /*                                                                          */
5692 /* Returns:                                                                 */
5693 /*   hw_cons                                                                */
5694 /****************************************************************************/
5695 static inline u16
5696 bce_get_hw_rx_cons(struct bce_softc *sc)
5697 {
5698         u16 hw_cons;
5699
5700         rmb();
5701         hw_cons = sc->status_block->status_rx_quick_consumer_index0;
5702         if ((hw_cons & USABLE_RX_BD_PER_PAGE) == USABLE_RX_BD_PER_PAGE)
5703                 hw_cons++;
5704
5705         return hw_cons;
5706 }
5707
5708 /****************************************************************************/
5709 /* Handles received frame interrupt events.                                 */
5710 /*                                                                          */
5711 /* Returns:                                                                 */
5712 /*   Nothing.                                                               */
5713 /****************************************************************************/
5714 static void
5715 bce_rx_intr(struct bce_softc *sc)
5716 {
5717         struct ifnet *ifp = sc->bce_ifp;
5718         struct l2_fhdr *l2fhdr;
5719         unsigned int pkt_len;
5720         u16 sw_rx_cons, sw_rx_cons_idx, hw_rx_cons;
5721         u32 status;
5722 #ifdef ZERO_COPY_SOCKETS
5723         unsigned int rem_len;
5724         u16 sw_pg_cons, sw_pg_cons_idx;
5725 #endif
5726
5727         DBENTER(BCE_VERBOSE_RECV | BCE_VERBOSE_INTR);
5728         DBRUN(sc->rx_interrupts++);
5729         DBPRINT(sc, BCE_EXTREME_RECV, "%s(enter): rx_prod = 0x%04X, "
5730                 "rx_cons = 0x%04X, rx_prod_bseq = 0x%08X\n",
5731                 __FUNCTION__, sc->rx_prod, sc->rx_cons, sc->rx_prod_bseq);
5732
5733         /* Prepare the RX chain pages to be accessed by the host CPU. */
5734         for (int i = 0; i < RX_PAGES; i++)
5735                 bus_dmamap_sync(sc->rx_bd_chain_tag,
5736                     sc->rx_bd_chain_map[i], BUS_DMASYNC_POSTWRITE);
5737
5738 #ifdef ZERO_COPY_SOCKETS
5739         /* Prepare the page chain pages to be accessed by the host CPU. */
5740         for (int i = 0; i < PG_PAGES; i++)
5741                 bus_dmamap_sync(sc->pg_bd_chain_tag,
5742                     sc->pg_bd_chain_map[i], BUS_DMASYNC_POSTWRITE);
5743 #endif
5744
5745         /* Get the hardware's view of the RX consumer index. */
5746         hw_rx_cons = sc->hw_rx_cons = bce_get_hw_rx_cons(sc);
5747
5748         /* Get working copies of the driver's view of the consumer indices. */
5749         sw_rx_cons = sc->rx_cons;
5750 #ifdef ZERO_COPY_SOCKETS
5751         sw_pg_cons = sc->pg_cons;
5752 #endif
5753
5754         /* Update some debug statistics counters */
5755         DBRUNIF((sc->free_rx_bd < sc->rx_low_watermark),
5756                 sc->rx_low_watermark = sc->free_rx_bd);
5757         DBRUNIF((sc->free_rx_bd == sc->max_rx_bd), sc->rx_empty_count++);
5758
5759         /* Scan through the receive chain as long as there is work to do */
5760         /* ToDo: Consider setting a limit on the number of packets processed. */
5761         rmb();
5762         while (sw_rx_cons != hw_rx_cons) {
5763                 struct mbuf *m0;
5764
5765                 /* Convert the producer/consumer indices to an actual rx_bd index. */
5766                 sw_rx_cons_idx = RX_CHAIN_IDX(sw_rx_cons);
5767
5768                 /* Unmap the mbuf from DMA space. */
5769                 bus_dmamap_sync(sc->rx_mbuf_tag,
5770                     sc->rx_mbuf_map[sw_rx_cons_idx],
5771                 BUS_DMASYNC_POSTREAD);
5772                 bus_dmamap_unload(sc->rx_mbuf_tag,
5773                     sc->rx_mbuf_map[sw_rx_cons_idx]);
5774
5775                 /* Remove the mbuf from the RX chain. */
5776                 m0 = sc->rx_mbuf_ptr[sw_rx_cons_idx];
5777                 sc->rx_mbuf_ptr[sw_rx_cons_idx] = NULL;
5778                 DBRUN(sc->debug_rx_mbuf_alloc--);
5779                 sc->free_rx_bd++;
5780
5781                 /*
5782                  * Frames received on the NetXteme II are prepended     with an
5783                  * l2_fhdr structure which provides status information about
5784                  * the received frame (including VLAN tags and checksum info).
5785                  * The frames are also automatically adjusted to align the IP
5786                  * header (i.e. two null bytes are inserted before the Ethernet
5787                  * header).  As a result the data DMA'd by the controller into
5788                  * the mbuf is as follows:
5789                  *
5790                  * +---------+-----+---------------------+-----+
5791                  * | l2_fhdr | pad | packet data         | FCS |
5792                  * +---------+-----+---------------------+-----+
5793                  *
5794                  * The l2_fhdr needs to be checked and skipped and the FCS needs
5795                  * to be stripped before sending the packet up the stack.
5796                  */
5797                 l2fhdr  = mtod(m0, struct l2_fhdr *);
5798
5799                 /* Get the packet data + FCS length and the status. */
5800                 pkt_len = l2fhdr->l2_fhdr_pkt_len;
5801                 status  = l2fhdr->l2_fhdr_status;
5802
5803                 /*
5804                  * Skip over the l2_fhdr and pad, resulting in the
5805                  * following data in the mbuf:
5806                  * +---------------------+-----+
5807                  * | packet data         | FCS |
5808                  * +---------------------+-----+
5809                  */
5810                 m_adj(m0, sizeof(struct l2_fhdr) + ETHER_ALIGN);
5811
5812 #ifdef ZERO_COPY_SOCKETS
5813                 /*
5814                  * Check whether the received frame fits in a single
5815                  * mbuf or not (i.e. packet data + FCS <=
5816                  * sc->rx_bd_mbuf_data_len bytes).
5817                  */
5818                 if (pkt_len > m0->m_len) {
5819                         /*
5820                          * The received frame is larger than a single mbuf.
5821                          * If the frame was a TCP frame then only the TCP
5822                          * header is placed in the mbuf, the remaining
5823                          * payload (including FCS) is placed in the page
5824                          * chain, the SPLIT flag is set, and the header
5825                          * length is placed in the IP checksum field.
5826                          * If the frame is not a TCP frame then the mbuf
5827                          * is filled and the remaining bytes are placed
5828                          * in the page chain.
5829                          */
5830
5831                         DBPRINT(sc, BCE_INFO_RECV, "%s(): Found a large packet.\n",
5832                                 __FUNCTION__);
5833
5834                         /*
5835                          * When the page chain is enabled and the TCP
5836                          * header has been split from the TCP payload,
5837                          * the ip_xsum structure will reflect the length
5838                          * of the TCP header, not the IP checksum.  Set
5839                          * the packet length of the mbuf accordingly.
5840                          */
5841                         if (status & L2_FHDR_STATUS_SPLIT)
5842                                 m0->m_len = l2fhdr->l2_fhdr_ip_xsum;
5843
5844                         rem_len = pkt_len - m0->m_len;
5845
5846                         /* Pull mbufs off the page chain for the remaining data. */
5847                         while (rem_len > 0) {
5848                                 struct mbuf *m_pg;
5849
5850                                 sw_pg_cons_idx = PG_CHAIN_IDX(sw_pg_cons);
5851
5852                                 /* Remove the mbuf from the page chain. */
5853                                 m_pg = sc->pg_mbuf_ptr[sw_pg_cons_idx];
5854                                 sc->pg_mbuf_ptr[sw_pg_cons_idx] = NULL;
5855                                 DBRUN(sc->debug_pg_mbuf_alloc--);
5856                                 sc->free_pg_bd++;
5857
5858                                 /* Unmap the page chain mbuf from DMA space. */
5859                                 bus_dmamap_sync(sc->pg_mbuf_tag,
5860                                         sc->pg_mbuf_map[sw_pg_cons_idx],
5861                                         BUS_DMASYNC_POSTREAD);
5862                                 bus_dmamap_unload(sc->pg_mbuf_tag,
5863                                         sc->pg_mbuf_map[sw_pg_cons_idx]);
5864
5865                                 /* Adjust the mbuf length. */
5866                                 if (rem_len < m_pg->m_len) {
5867                                         /* The mbuf chain is complete. */
5868                                         m_pg->m_len = rem_len;
5869                                         rem_len = 0;
5870                                 } else {
5871                                         /* More packet data is waiting. */
5872                                         rem_len -= m_pg->m_len;
5873                                 }
5874
5875                                 /* Concatenate the mbuf cluster to the mbuf. */
5876                                 m_cat(m0, m_pg);
5877
5878                                 sw_pg_cons = NEXT_PG_BD(sw_pg_cons);
5879                         }
5880
5881                         /* Set the total packet length. */
5882                         m0->m_pkthdr.len = pkt_len;
5883
5884                 } else {
5885                         /*
5886                          * The received packet is small and fits in a
5887                          * single mbuf (i.e. the l2_fhdr + pad + packet +
5888                          * FCS <= MHLEN).  In other words, the packet is
5889                          * 154 bytes or less in size.
5890                          */
5891
5892                         DBPRINT(sc, BCE_INFO_RECV, "%s(): Found a small packet.\n",
5893                                 __FUNCTION__);
5894
5895                         /* Set the total packet length. */
5896                         m0->m_pkthdr.len = m0->m_len = pkt_len;
5897                 }
5898 #else
5899         /* Set the total packet length. */
5900                 m0->m_pkthdr.len = m0->m_len = pkt_len;
5901 #endif
5902
5903                 /* Remove the trailing Ethernet FCS. */
5904                 m_adj(m0, -ETHER_CRC_LEN);
5905
5906                 /* Check that the resulting mbuf chain is valid. */
5907                 DBRUN(m_sanity(m0, FALSE));
5908                 DBRUNIF(((m0->m_len < ETHER_HDR_LEN) |
5909                         (m0->m_pkthdr.len > BCE_MAX_JUMBO_ETHER_MTU_VLAN)),
5910                         BCE_PRINTF("Invalid Ethernet frame size!\n");
5911                         m_print(m0, 128));
5912
5913                 DBRUNIF(DB_RANDOMTRUE(l2fhdr_error_sim_control),
5914                         BCE_PRINTF("Simulating l2_fhdr status error.\n");
5915                         sc->l2fhdr_error_sim_count++;
5916                         status = status | L2_FHDR_ERRORS_PHY_DECODE);
5917
5918                 /* Check the received frame for errors. */
5919                 if (status & (L2_FHDR_ERRORS_BAD_CRC |
5920                         L2_FHDR_ERRORS_PHY_DECODE | L2_FHDR_ERRORS_ALIGNMENT |
5921                         L2_FHDR_ERRORS_TOO_SHORT  | L2_FHDR_ERRORS_GIANT_FRAME)) {
5922
5923                         /* Log the error and release the mbuf. */
5924                         ifp->if_ierrors++;
5925                         sc->l2fhdr_error_count++;
5926
5927                         m_freem(m0);
5928                         m0 = NULL;
5929                         goto bce_rx_int_next_rx;
5930                 }
5931
5932                 /* Send the packet to the appropriate interface. */
5933                 m0->m_pkthdr.rcvif = ifp;
5934
5935                 /* Assume no hardware checksum. */
5936                 m0->m_pkthdr.csum_flags = 0;
5937
5938                 /* Validate the checksum if offload enabled. */
5939                 if (ifp->if_capenable & IFCAP_RXCSUM) {
5940
5941                         /* Check for an IP datagram. */
5942                         if (!(status & L2_FHDR_STATUS_SPLIT) &&
5943                                 (status & L2_FHDR_STATUS_IP_DATAGRAM)) {
5944                                 m0->m_pkthdr.csum_flags |= CSUM_IP_CHECKED;
5945
5946                                 /* Check if the IP checksum is valid. */
5947                                 if ((l2fhdr->l2_fhdr_ip_xsum ^ 0xffff) == 0)
5948                                         m0->m_pkthdr.csum_flags |= CSUM_IP_VALID;
5949                         }
5950
5951                         /* Check for a valid TCP/UDP frame. */
5952                         if (status & (L2_FHDR_STATUS_TCP_SEGMENT |
5953                                 L2_FHDR_STATUS_UDP_DATAGRAM)) {
5954
5955                                 /* Check for a good TCP/UDP checksum. */
5956                                 if ((status & (L2_FHDR_ERRORS_TCP_XSUM |
5957                                               L2_FHDR_ERRORS_UDP_XSUM)) == 0) {
5958                                         m0->m_pkthdr.csum_data =
5959                                             l2fhdr->l2_fhdr_tcp_udp_xsum;
5960                                         m0->m_pkthdr.csum_flags |= (CSUM_DATA_VALID
5961                                                 | CSUM_PSEUDO_HDR);
5962                                 }
5963                         }
5964                 }
5965
5966                 /* Attach the VLAN tag. */
5967                 if (status & L2_FHDR_STATUS_L2_VLAN_TAG) {
5968 #if __FreeBSD_version < 700000
5969                         VLAN_INPUT_TAG(ifp, m0, l2fhdr->l2_fhdr_vlan_tag, continue);
5970 #else
5971                         m0->m_pkthdr.ether_vtag = l2fhdr->l2_fhdr_vlan_tag;
5972                         m0->m_flags |= M_VLANTAG;
5973 #endif
5974                 }
5975
5976                 /* Increment received packet statistics. */
5977                 ifp->if_ipackets++;
5978
5979 bce_rx_int_next_rx:
5980                 sw_rx_cons = NEXT_RX_BD(sw_rx_cons);
5981
5982                 /* If we have a packet, pass it up the stack */
5983                 if (m0) {
5984                         /* Make sure we don't lose our place when we release the lock. */
5985                         sc->rx_cons = sw_rx_cons;
5986 #ifdef ZERO_COPY_SOCKETS
5987                         sc->pg_cons = sw_pg_cons;
5988 #endif
5989
5990                         BCE_UNLOCK(sc);
5991                         (*ifp->if_input)(ifp, m0);
5992                         BCE_LOCK(sc);
5993
5994                         /* Recover our place. */
5995                         sw_rx_cons = sc->rx_cons;
5996 #ifdef ZERO_COPY_SOCKETS
5997                         sw_pg_cons = sc->pg_cons;
5998 #endif
5999                 }
6000
6001                 /* Refresh hw_cons to see if there's new work */
6002                 if (sw_rx_cons == hw_rx_cons)
6003                         hw_rx_cons = sc->hw_rx_cons = bce_get_hw_rx_cons(sc);
6004         }
6005
6006         /* No new packets to process.  Refill the RX and page chains and exit. */
6007 #ifdef ZERO_COPY_SOCKETS
6008         sc->pg_cons = sw_pg_cons;
6009         bce_fill_pg_chain(sc);
6010 #endif
6011
6012         sc->rx_cons = sw_rx_cons;
6013         bce_fill_rx_chain(sc);
6014
6015         for (int i = 0; i < RX_PAGES; i++)
6016                 bus_dmamap_sync(sc->rx_bd_chain_tag,
6017                     sc->rx_bd_chain_map[i], BUS_DMASYNC_PREWRITE);
6018
6019 #ifdef ZERO_COPY_SOCKETS
6020         for (int i = 0; i < PG_PAGES; i++)
6021                 bus_dmamap_sync(sc->pg_bd_chain_tag,
6022                     sc->pg_bd_chain_map[i], BUS_DMASYNC_PREWRITE);
6023 #endif
6024
6025         DBPRINT(sc, BCE_EXTREME_RECV, "%s(exit): rx_prod = 0x%04X, "
6026                 "rx_cons = 0x%04X, rx_prod_bseq = 0x%08X\n",
6027                 __FUNCTION__, sc->rx_prod, sc->rx_cons, sc->rx_prod_bseq);
6028         DBEXIT(BCE_VERBOSE_RECV | BCE_VERBOSE_INTR);
6029 }
6030
6031
6032 /****************************************************************************/
6033 /* Reads the transmit consumer value from the status block (skipping over   */
6034 /* chain page pointer if necessary).                                        */
6035 /*                                                                          */
6036 /* Returns:                                                                 */
6037 /*   hw_cons                                                                */
6038 /****************************************************************************/
6039 static inline u16
6040 bce_get_hw_tx_cons(struct bce_softc *sc)
6041 {
6042         u16 hw_cons;
6043
6044         mb();
6045         hw_cons = sc->status_block->status_tx_quick_consumer_index0;
6046         if ((hw_cons & USABLE_TX_BD_PER_PAGE) == USABLE_TX_BD_PER_PAGE)
6047                 hw_cons++;
6048
6049         return hw_cons;
6050 }
6051
6052
6053 /****************************************************************************/
6054 /* Handles transmit completion interrupt events.                            */
6055 /*                                                                          */
6056 /* Returns:                                                                 */
6057 /*   Nothing.                                                               */
6058 /****************************************************************************/
6059 static void
6060 bce_tx_intr(struct bce_softc *sc)
6061 {
6062         struct ifnet *ifp = sc->bce_ifp;
6063         u16 hw_tx_cons, sw_tx_cons, sw_tx_chain_cons;
6064
6065         DBENTER(BCE_VERBOSE_SEND | BCE_VERBOSE_INTR);
6066         DBRUN(sc->tx_interrupts++);
6067         DBPRINT(sc, BCE_EXTREME_SEND, "%s(enter): tx_prod = 0x%04X, "
6068                 "tx_cons = 0x%04X, tx_prod_bseq = 0x%08X\n",
6069                 __FUNCTION__, sc->tx_prod, sc->tx_cons, sc->tx_prod_bseq);
6070
6071         BCE_LOCK_ASSERT(sc);
6072
6073         /* Get the hardware's view of the TX consumer index. */
6074         hw_tx_cons = sc->hw_tx_cons = bce_get_hw_tx_cons(sc);
6075         sw_tx_cons = sc->tx_cons;
6076
6077         /* Prevent speculative reads from getting ahead of the status block. */
6078         bus_space_barrier(sc->bce_btag, sc->bce_bhandle, 0, 0,
6079                 BUS_SPACE_BARRIER_READ);
6080
6081         /* Cycle through any completed TX chain page entries. */
6082         while (sw_tx_cons != hw_tx_cons) {
6083 #ifdef BCE_DEBUG
6084                 struct tx_bd *txbd = NULL;
6085 #endif
6086                 sw_tx_chain_cons = TX_CHAIN_IDX(sw_tx_cons);
6087
6088                 DBPRINT(sc, BCE_INFO_SEND,
6089                         "%s(): hw_tx_cons = 0x%04X, sw_tx_cons = 0x%04X, "
6090                         "sw_tx_chain_cons = 0x%04X\n",
6091                         __FUNCTION__, hw_tx_cons, sw_tx_cons, sw_tx_chain_cons);
6092
6093                 DBRUNIF((sw_tx_chain_cons > MAX_TX_BD),
6094                         BCE_PRINTF("%s(%d): TX chain consumer out of range! "
6095                                 " 0x%04X > 0x%04X\n", __FILE__, __LINE__, sw_tx_chain_cons,
6096                                 (int) MAX_TX_BD);
6097                         bce_breakpoint(sc));
6098
6099                 DBRUN(txbd = &sc->tx_bd_chain[TX_PAGE(sw_tx_chain_cons)]
6100                                 [TX_IDX(sw_tx_chain_cons)]);
6101
6102                 DBRUNIF((txbd == NULL),
6103                         BCE_PRINTF("%s(%d): Unexpected NULL tx_bd[0x%04X]!\n",
6104                                 __FILE__, __LINE__, sw_tx_chain_cons);
6105                         bce_breakpoint(sc));
6106
6107                 DBRUNMSG(BCE_INFO_SEND, BCE_PRINTF("%s(): ", __FUNCTION__);
6108                         bce_dump_txbd(sc, sw_tx_chain_cons, txbd));
6109
6110                 /*
6111                  * Free the associated mbuf. Remember
6112                  * that only the last tx_bd of a packet
6113                  * has an mbuf pointer and DMA map.
6114                  */
6115                 if (sc->tx_mbuf_ptr[sw_tx_chain_cons] != NULL) {
6116
6117                         /* Validate that this is the last tx_bd. */
6118                         DBRUNIF((!(txbd->tx_bd_flags & TX_BD_FLAGS_END)),
6119                                 BCE_PRINTF("%s(%d): tx_bd END flag not set but "
6120                                 "txmbuf == NULL!\n", __FILE__, __LINE__);
6121                                 bce_breakpoint(sc));
6122
6123                         DBRUNMSG(BCE_INFO_SEND,
6124                                 BCE_PRINTF("%s(): Unloading map/freeing mbuf "
6125                                         "from tx_bd[0x%04X]\n", __FUNCTION__, sw_tx_chain_cons));
6126
6127                         /* Unmap the mbuf. */
6128                         bus_dmamap_unload(sc->tx_mbuf_tag,
6129                             sc->tx_mbuf_map[sw_tx_chain_cons]);
6130
6131                         /* Free the mbuf. */
6132                         m_freem(sc->tx_mbuf_ptr[sw_tx_chain_cons]);
6133                         sc->tx_mbuf_ptr[sw_tx_chain_cons] = NULL;
6134                         DBRUN(sc->debug_tx_mbuf_alloc--);
6135
6136                         ifp->if_opackets++;
6137                 }
6138
6139                 sc->used_tx_bd--;
6140                 sw_tx_cons = NEXT_TX_BD(sw_tx_cons);
6141
6142                 /* Refresh hw_cons to see if there's new work. */
6143                 hw_tx_cons = sc->hw_tx_cons = bce_get_hw_tx_cons(sc);
6144
6145                 /* Prevent speculative reads from getting ahead of the status block. */
6146                 bus_space_barrier(sc->bce_btag, sc->bce_bhandle, 0, 0,
6147                         BUS_SPACE_BARRIER_READ);
6148         }
6149
6150         /* Clear the TX timeout timer. */
6151         sc->watchdog_timer = 0;
6152
6153         /* Clear the tx hardware queue full flag. */
6154         if (sc->used_tx_bd < sc->max_tx_bd) {
6155                 DBRUNIF((ifp->if_drv_flags & IFF_DRV_OACTIVE),
6156                         DBPRINT(sc, BCE_INFO_SEND,
6157                                 "%s(): Open TX chain! %d/%d (used/total)\n",
6158                                 __FUNCTION__, sc->used_tx_bd, sc->max_tx_bd));
6159                 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
6160         }
6161
6162         sc->tx_cons = sw_tx_cons;
6163
6164         DBPRINT(sc, BCE_EXTREME_SEND, "%s(exit): tx_prod = 0x%04X, "
6165                 "tx_cons = 0x%04X, tx_prod_bseq = 0x%08X\n",
6166                 __FUNCTION__, sc->tx_prod, sc->tx_cons, sc->tx_prod_bseq);
6167         DBEXIT(BCE_VERBOSE_SEND | BCE_VERBOSE_INTR);
6168 }
6169
6170
6171 /****************************************************************************/
6172 /* Disables interrupt generation.                                           */
6173 /*                                                                          */
6174 /* Returns:                                                                 */
6175 /*   Nothing.                                                               */
6176 /****************************************************************************/
6177 static void
6178 bce_disable_intr(struct bce_softc *sc)
6179 {
6180         DBENTER(BCE_VERBOSE_INTR);
6181
6182         REG_WR(sc, BCE_PCICFG_INT_ACK_CMD, BCE_PCICFG_INT_ACK_CMD_MASK_INT);
6183         REG_RD(sc, BCE_PCICFG_INT_ACK_CMD);
6184
6185         DBEXIT(BCE_VERBOSE_INTR);
6186 }
6187
6188
6189 /****************************************************************************/
6190 /* Enables interrupt generation.                                            */
6191 /*                                                                          */
6192 /* Returns:                                                                 */
6193 /*   Nothing.                                                               */
6194 /****************************************************************************/
6195 static void
6196 bce_enable_intr(struct bce_softc *sc, int coal_now)
6197 {
6198         DBENTER(BCE_VERBOSE_INTR);
6199
6200         REG_WR(sc, BCE_PCICFG_INT_ACK_CMD,
6201                BCE_PCICFG_INT_ACK_CMD_INDEX_VALID |
6202                BCE_PCICFG_INT_ACK_CMD_MASK_INT | sc->last_status_idx);
6203
6204         REG_WR(sc, BCE_PCICFG_INT_ACK_CMD,
6205                BCE_PCICFG_INT_ACK_CMD_INDEX_VALID | sc->last_status_idx);
6206
6207         /* Force an immediate interrupt (whether there is new data or not). */
6208         if (coal_now)
6209                 REG_WR(sc, BCE_HC_COMMAND, sc->hc_command | BCE_HC_COMMAND_COAL_NOW);
6210
6211         DBEXIT(BCE_VERBOSE_INTR);
6212 }
6213
6214
6215 /****************************************************************************/
6216 /* Handles controller initialization.                                       */
6217 /*                                                                          */
6218 /* Returns:                                                                 */
6219 /*   Nothing.                                                               */
6220 /****************************************************************************/
6221 static void
6222 bce_init_locked(struct bce_softc *sc)
6223 {
6224         struct ifnet *ifp;
6225         u32 ether_mtu = 0;
6226
6227         DBENTER(BCE_VERBOSE_RESET);
6228
6229         BCE_LOCK_ASSERT(sc);
6230
6231         ifp = sc->bce_ifp;
6232
6233         /* Check if the driver is still running and bail out if it is. */
6234         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
6235                 goto bce_init_locked_exit;
6236
6237         bce_stop(sc);
6238
6239         if (bce_reset(sc, BCE_DRV_MSG_CODE_RESET)) {
6240                 BCE_PRINTF("%s(%d): Controller reset failed!\n",
6241                         __FILE__, __LINE__);
6242                 goto bce_init_locked_exit;
6243         }
6244
6245         if (bce_chipinit(sc)) {
6246                 BCE_PRINTF("%s(%d): Controller initialization failed!\n",
6247                         __FILE__, __LINE__);
6248                 goto bce_init_locked_exit;
6249         }
6250
6251         if (bce_blockinit(sc)) {
6252                 BCE_PRINTF("%s(%d): Block initialization failed!\n",
6253                         __FILE__, __LINE__);
6254                 goto bce_init_locked_exit;
6255         }
6256
6257         /* Load our MAC address. */
6258         bcopy(IF_LLADDR(sc->bce_ifp), sc->eaddr, ETHER_ADDR_LEN);
6259         bce_set_mac_addr(sc);
6260
6261         /*
6262          * Calculate and program the hardware Ethernet MTU
6263          * size. Be generous on the receive if we have room.
6264          */
6265 #ifdef ZERO_COPY_SOCKETS
6266         if (ifp->if_mtu <= (sc->rx_bd_mbuf_data_len + sc->pg_bd_mbuf_alloc_size))
6267                 ether_mtu = sc->rx_bd_mbuf_data_len + sc->pg_bd_mbuf_alloc_size;
6268 #else
6269         if (ifp->if_mtu <= sc->rx_bd_mbuf_data_len)
6270                 ether_mtu = sc->rx_bd_mbuf_data_len;
6271 #endif
6272         else
6273                 ether_mtu = ifp->if_mtu;
6274
6275         ether_mtu += ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + ETHER_CRC_LEN;
6276
6277         DBPRINT(sc, BCE_INFO_MISC, "%s(): setting h/w mtu = %d\n", __FUNCTION__,
6278                 ether_mtu);
6279
6280         /* Program the mtu, enabling jumbo frame support if necessary. */
6281         if (ether_mtu > (ETHER_MAX_LEN + ETHER_VLAN_ENCAP_LEN))
6282                 REG_WR(sc, BCE_EMAC_RX_MTU_SIZE,
6283                         min(ether_mtu, BCE_MAX_JUMBO_ETHER_MTU) |
6284                         BCE_EMAC_RX_MTU_SIZE_JUMBO_ENA);
6285         else
6286                 REG_WR(sc, BCE_EMAC_RX_MTU_SIZE, ether_mtu);
6287
6288         DBPRINT(sc, BCE_INFO_LOAD,
6289                 "%s(): rx_bd_mbuf_alloc_size = %d, rx_bce_mbuf_data_len = %d, "
6290                 "rx_bd_mbuf_align_pad = %d\n", __FUNCTION__,
6291                 sc->rx_bd_mbuf_alloc_size, sc->rx_bd_mbuf_data_len,
6292                 sc->rx_bd_mbuf_align_pad);
6293
6294         /* Program appropriate promiscuous/multicast filtering. */
6295         bce_set_rx_mode(sc);
6296
6297 #ifdef ZERO_COPY_SOCKETS
6298         DBPRINT(sc, BCE_INFO_LOAD, "%s(): pg_bd_mbuf_alloc_size = %d\n",
6299                 __FUNCTION__, sc->pg_bd_mbuf_alloc_size);
6300
6301         /* Init page buffer descriptor chain. */
6302         bce_init_pg_chain(sc);
6303 #endif
6304
6305         /* Init RX buffer descriptor chain. */
6306         bce_init_rx_chain(sc);
6307
6308         /* Init TX buffer descriptor chain. */
6309         bce_init_tx_chain(sc);
6310
6311         /* Enable host interrupts. */
6312         bce_enable_intr(sc, 1);
6313
6314         bce_ifmedia_upd_locked(ifp);
6315
6316         ifp->if_drv_flags |= IFF_DRV_RUNNING;
6317         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
6318
6319         callout_reset(&sc->bce_tick_callout, hz, bce_tick, sc);
6320
6321 bce_init_locked_exit:
6322         DBEXIT(BCE_VERBOSE_RESET);
6323 }
6324
6325
6326 /****************************************************************************/
6327 /* Initialize the controller just enough so that any management firmware    */
6328 /* running on the device will continue to operate correctly.                */
6329 /*                                                                          */
6330 /* Returns:                                                                 */
6331 /*   Nothing.                                                               */
6332 /****************************************************************************/
6333 static void
6334 bce_mgmt_init_locked(struct bce_softc *sc)
6335 {
6336         struct ifnet *ifp;
6337
6338         DBENTER(BCE_VERBOSE_RESET);
6339
6340         BCE_LOCK_ASSERT(sc);
6341
6342         /* Bail out if management firmware is not running. */
6343         if (!(sc->bce_flags & BCE_MFW_ENABLE_FLAG)) {
6344                 DBPRINT(sc, BCE_VERBOSE_SPECIAL,
6345                         "No management firmware running...\n");
6346                 goto bce_mgmt_init_locked_exit;
6347         }
6348
6349         ifp = sc->bce_ifp;
6350
6351         /* Enable all critical blocks in the MAC. */
6352         REG_WR(sc, BCE_MISC_ENABLE_SET_BITS, BCE_MISC_ENABLE_DEFAULT);
6353         REG_RD(sc, BCE_MISC_ENABLE_SET_BITS);
6354         DELAY(20);
6355
6356         bce_ifmedia_upd_locked(ifp);
6357
6358 bce_mgmt_init_locked_exit:
6359         DBEXIT(BCE_VERBOSE_RESET);
6360 }
6361
6362
6363 /****************************************************************************/
6364 /* Handles controller initialization when called from an unlocked routine.  */
6365 /*                                                                          */
6366 /* Returns:                                                                 */
6367 /*   Nothing.                                                               */
6368 /****************************************************************************/
6369 static void
6370 bce_init(void *xsc)
6371 {
6372         struct bce_softc *sc = xsc;
6373
6374         DBENTER(BCE_VERBOSE_RESET);
6375
6376         BCE_LOCK(sc);
6377         bce_init_locked(sc);
6378         BCE_UNLOCK(sc);
6379
6380         DBEXIT(BCE_VERBOSE_RESET);
6381 }
6382
6383
6384 /****************************************************************************/
6385 /* Encapsultes an mbuf cluster into the tx_bd chain structure and makes the */
6386 /* memory visible to the controller.                                        */
6387 /*                                                                          */
6388 /* Returns:                                                                 */
6389 /*   0 for success, positive value for failure.                             */
6390 /* Modified:                                                                */
6391 /*   m_head: May be set to NULL if MBUF is excessively fragmented.          */
6392 /****************************************************************************/
6393 static int
6394 bce_tx_encap(struct bce_softc *sc, struct mbuf **m_head)
6395 {
6396         bus_dma_segment_t segs[BCE_MAX_SEGMENTS];
6397         bus_dmamap_t map;
6398         struct tx_bd *txbd = NULL;
6399         struct mbuf *m0;
6400         struct ether_vlan_header *eh;
6401         struct ip *ip;
6402         struct tcphdr *th;
6403         u16 prod, chain_prod, etype, mss = 0, vlan_tag = 0, flags = 0;
6404         u32 prod_bseq;
6405         int hdr_len = 0, e_hlen = 0, ip_hlen = 0, tcp_hlen = 0, ip_len = 0;
6406
6407 #ifdef BCE_DEBUG
6408         u16 debug_prod;
6409 #endif
6410         int i, error, nsegs, rc = 0;
6411
6412         DBENTER(BCE_VERBOSE_SEND);
6413         DBPRINT(sc, BCE_INFO_SEND,
6414                 "%s(enter): tx_prod = 0x%04X, tx_chain_prod = %04X, "
6415                 "tx_prod_bseq = 0x%08X\n",
6416                 __FUNCTION__, sc->tx_prod, (u16) TX_CHAIN_IDX(sc->tx_prod),
6417                 sc->tx_prod_bseq);
6418
6419         /* Transfer any checksum offload flags to the bd. */
6420         m0 = *m_head;
6421         if (m0->m_pkthdr.csum_flags) {
6422                 if (m0->m_pkthdr.csum_flags & CSUM_IP)
6423                         flags |= TX_BD_FLAGS_IP_CKSUM;
6424                 if (m0->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP))
6425                         flags |= TX_BD_FLAGS_TCP_UDP_CKSUM;
6426                 if (m0->m_pkthdr.csum_flags & CSUM_TSO) {
6427                         /* For TSO the controller needs two pieces of info, */
6428                         /* the MSS and the IP+TCP options length.           */
6429                         mss = htole16(m0->m_pkthdr.tso_segsz);
6430
6431                         /* Map the header and find the Ethernet type & header length */
6432                         eh = mtod(m0, struct ether_vlan_header *);
6433                         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
6434                                 etype = ntohs(eh->evl_proto);
6435                                 e_hlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
6436                         } else {
6437                                 etype = ntohs(eh->evl_encap_proto);
6438                                 e_hlen = ETHER_HDR_LEN;
6439                         }
6440
6441                         /* Check for supported TSO Ethernet types (only IPv4 for now) */
6442                         switch (etype) {
6443                                 case ETHERTYPE_IP:
6444                                         ip = (struct ip *)(m0->m_data + e_hlen);
6445
6446                                         /* TSO only supported for TCP protocol */
6447                                         if (ip->ip_p != IPPROTO_TCP) {
6448                                                 BCE_PRINTF("%s(%d): TSO enabled for non-TCP frame!.\n",
6449                                                         __FILE__, __LINE__);
6450                                                 goto bce_tx_encap_skip_tso;
6451                                         }
6452
6453                                         /* Get IP header length in bytes (min 20) */
6454                                         ip_hlen = ip->ip_hl << 2;
6455
6456                                         /* Get the TCP header length in bytes (min 20) */
6457                                         th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
6458                                         tcp_hlen = (th->th_off << 2);
6459
6460                                         /* IP header length and checksum will be calc'd by hardware */
6461                                         ip_len = ip->ip_len;
6462                                         ip->ip_len = 0;
6463                                         ip->ip_sum = 0;
6464                                         break;
6465                                 case ETHERTYPE_IPV6:
6466                                         BCE_PRINTF("%s(%d): TSO over IPv6 not supported!.\n",
6467                                                 __FILE__, __LINE__);
6468                                         goto bce_tx_encap_skip_tso;
6469                                 default:
6470                                         BCE_PRINTF("%s(%d): TSO enabled for unsupported protocol!.\n",
6471                                                 __FILE__, __LINE__);
6472                                         goto bce_tx_encap_skip_tso;
6473                         }
6474
6475                         hdr_len = e_hlen + ip_hlen + tcp_hlen;
6476
6477                         DBPRINT(sc, BCE_EXTREME_SEND,
6478                                 "%s(): hdr_len = %d, e_hlen = %d, ip_hlen = %d, tcp_hlen = %d, ip_len = %d\n",
6479                                  __FUNCTION__, hdr_len, e_hlen, ip_hlen, tcp_hlen, ip_len);
6480
6481                         /* Set the LSO flag in the TX BD */
6482                         flags |= TX_BD_FLAGS_SW_LSO;
6483                         /* Set the length of IP + TCP options (in 32 bit words) */
6484                         flags |= (((ip_hlen + tcp_hlen - 40) >> 2) << 8);
6485
6486 bce_tx_encap_skip_tso:
6487                         DBRUN(sc->requested_tso_frames++);
6488                 }
6489         }
6490
6491         /* Transfer any VLAN tags to the bd. */
6492         if (m0->m_flags & M_VLANTAG) {
6493                 flags |= TX_BD_FLAGS_VLAN_TAG;
6494                 vlan_tag = m0->m_pkthdr.ether_vtag;
6495         }
6496
6497         /* Map the mbuf into DMAable memory. */
6498         prod = sc->tx_prod;
6499         chain_prod = TX_CHAIN_IDX(prod);
6500         map = sc->tx_mbuf_map[chain_prod];
6501
6502         /* Map the mbuf into our DMA address space. */
6503         error = bus_dmamap_load_mbuf_sg(sc->tx_mbuf_tag, map, m0,
6504             segs, &nsegs, BUS_DMA_NOWAIT);
6505
6506         /* Check if the DMA mapping was successful */
6507         if (error == EFBIG) {
6508
6509                 sc->fragmented_mbuf_count++;
6510
6511                 /* Try to defrag the mbuf. */
6512                 m0 = m_defrag(*m_head, M_DONTWAIT);
6513                 if (m0 == NULL) {
6514                         /* Defrag was unsuccessful */
6515                         m_freem(*m_head);
6516                         *m_head = NULL;
6517                         sc->mbuf_alloc_failed_count++;
6518                         rc = ENOBUFS;
6519                         goto bce_tx_encap_exit;
6520                 }
6521
6522                 /* Defrag was successful, try mapping again */
6523                 *m_head = m0;
6524                 error = bus_dmamap_load_mbuf_sg(sc->tx_mbuf_tag, map, m0,
6525                     segs, &nsegs, BUS_DMA_NOWAIT);
6526
6527                 /* Still getting an error after a defrag. */
6528                 if (error == ENOMEM) {
6529                         /* Insufficient DMA buffers available. */
6530                         sc->dma_map_addr_tx_failed_count++;
6531                         rc = error;
6532                         goto bce_tx_encap_exit;
6533                 } else if (error != 0) {
6534                         /* Still can't map the mbuf, release it and return an error. */
6535                         BCE_PRINTF(
6536                             "%s(%d): Unknown error mapping mbuf into TX chain!\n",
6537                             __FILE__, __LINE__);
6538                         m_freem(m0);
6539                         *m_head = NULL;
6540                         sc->dma_map_addr_tx_failed_count++;
6541                         rc = ENOBUFS;
6542                         goto bce_tx_encap_exit;
6543                 }
6544         } else if (error == ENOMEM) {
6545                 /* Insufficient DMA buffers available. */
6546                 sc->dma_map_addr_tx_failed_count++;
6547                 rc = error;
6548                 goto bce_tx_encap_exit;
6549         } else if (error != 0) {
6550                 m_freem(m0);
6551                 *m_head = NULL;
6552                 sc->dma_map_addr_tx_failed_count++;
6553                 rc = error;
6554                 goto bce_tx_encap_exit;
6555         }
6556
6557         /* Make sure there's room in the chain */
6558         if (nsegs > (sc->max_tx_bd - sc->used_tx_bd)) {
6559                 bus_dmamap_unload(sc->tx_mbuf_tag, map);
6560                 rc = ENOBUFS;
6561                 goto bce_tx_encap_exit;
6562         }
6563
6564         /* prod points to an empty tx_bd at this point. */
6565         prod_bseq  = sc->tx_prod_bseq;
6566
6567 #ifdef BCE_DEBUG
6568         debug_prod = chain_prod;
6569 #endif
6570
6571         DBPRINT(sc, BCE_INFO_SEND,
6572                 "%s(start): prod = 0x%04X, chain_prod = 0x%04X, "
6573                 "prod_bseq = 0x%08X\n",
6574                 __FUNCTION__, prod, chain_prod, prod_bseq);
6575
6576         /*
6577          * Cycle through each mbuf segment that makes up
6578          * the outgoing frame, gathering the mapping info
6579          * for that segment and creating a tx_bd for
6580          * the mbuf.
6581          */
6582         for (i = 0; i < nsegs ; i++) {
6583
6584                 chain_prod = TX_CHAIN_IDX(prod);
6585                 txbd= &sc->tx_bd_chain[TX_PAGE(chain_prod)][TX_IDX(chain_prod)];
6586
6587                 txbd->tx_bd_haddr_lo = htole32(BCE_ADDR_LO(segs[i].ds_addr));
6588                 txbd->tx_bd_haddr_hi = htole32(BCE_ADDR_HI(segs[i].ds_addr));
6589                 txbd->tx_bd_mss_nbytes = htole32(mss << 16) | htole16(segs[i].ds_len);
6590                 txbd->tx_bd_vlan_tag = htole16(vlan_tag);
6591                 txbd->tx_bd_flags = htole16(flags);
6592                 prod_bseq += segs[i].ds_len;
6593                 if (i == 0)
6594                         txbd->tx_bd_flags |= htole16(TX_BD_FLAGS_START);
6595                 prod = NEXT_TX_BD(prod);
6596         }
6597
6598         /* Set the END flag on the last TX buffer descriptor. */
6599         txbd->tx_bd_flags |= htole16(TX_BD_FLAGS_END);
6600
6601         DBRUNMSG(BCE_EXTREME_SEND, bce_dump_tx_chain(sc, debug_prod, nsegs));
6602
6603         DBPRINT(sc, BCE_INFO_SEND,
6604                 "%s( end ): prod = 0x%04X, chain_prod = 0x%04X, "
6605                 "prod_bseq = 0x%08X\n",
6606                 __FUNCTION__, prod, chain_prod, prod_bseq);
6607
6608         /*
6609          * Ensure that the mbuf pointer for this transmission
6610          * is placed at the array index of the last
6611          * descriptor in this chain.  This is done
6612          * because a single map is used for all
6613          * segments of the mbuf and we don't want to
6614          * unload the map before all of the segments
6615          * have been freed.
6616          */
6617         sc->tx_mbuf_ptr[chain_prod] = m0;
6618         sc->used_tx_bd += nsegs;
6619
6620         /* Update some debug statistic counters */
6621         DBRUNIF((sc->used_tx_bd > sc->tx_hi_watermark),
6622                 sc->tx_hi_watermark = sc->used_tx_bd);
6623         DBRUNIF((sc->used_tx_bd == sc->max_tx_bd), sc->tx_full_count++);
6624         DBRUNIF(sc->debug_tx_mbuf_alloc++);
6625
6626         DBRUNMSG(BCE_EXTREME_SEND, bce_dump_tx_mbuf_chain(sc, chain_prod, 1));
6627
6628         /* prod points to the next free tx_bd at this point. */
6629         sc->tx_prod = prod;
6630         sc->tx_prod_bseq = prod_bseq;
6631
6632         DBPRINT(sc, BCE_INFO_SEND,
6633                 "%s(exit): prod = 0x%04X, chain_prod = %04X, "
6634                 "prod_bseq = 0x%08X\n",
6635                 __FUNCTION__, sc->tx_prod, (u16) TX_CHAIN_IDX(sc->tx_prod),
6636                 sc->tx_prod_bseq);
6637
6638 bce_tx_encap_exit:
6639         DBEXIT(BCE_VERBOSE_SEND);
6640         return(rc);
6641 }
6642
6643
6644 /****************************************************************************/
6645 /* Main transmit routine when called from another routine with a lock.      */
6646 /*                                                                          */
6647 /* Returns:                                                                 */
6648 /*   Nothing.                                                               */
6649 /****************************************************************************/
6650 static void
6651 bce_start_locked(struct ifnet *ifp)
6652 {
6653         struct bce_softc *sc = ifp->if_softc;
6654         struct mbuf *m_head = NULL;
6655         int count = 0;
6656         u16 tx_prod, tx_chain_prod;
6657
6658         DBENTER(BCE_VERBOSE_SEND | BCE_VERBOSE_CTX);
6659
6660         BCE_LOCK_ASSERT(sc);
6661
6662         /* prod points to the next free tx_bd. */
6663         tx_prod = sc->tx_prod;
6664         tx_chain_prod = TX_CHAIN_IDX(tx_prod);
6665
6666         DBPRINT(sc, BCE_INFO_SEND,
6667                 "%s(enter): tx_prod = 0x%04X, tx_chain_prod = 0x%04X, "
6668                 "tx_prod_bseq = 0x%08X\n",
6669                 __FUNCTION__, tx_prod, tx_chain_prod, sc->tx_prod_bseq);
6670
6671         /* If there's no link or the transmit queue is empty then just exit. */
6672         if (!sc->bce_link) {
6673                 DBPRINT(sc, BCE_INFO_SEND, "%s(): No link.\n",
6674                         __FUNCTION__);
6675                 goto bce_start_locked_exit;
6676         }
6677
6678         if (IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
6679                 DBPRINT(sc, BCE_INFO_SEND, "%s(): Transmit queue empty.\n",
6680                         __FUNCTION__);
6681                 goto bce_start_locked_exit;
6682         }
6683
6684         /*
6685          * Keep adding entries while there is space in the ring.
6686          */
6687         while (sc->used_tx_bd < sc->max_tx_bd) {
6688
6689                 /* Check for any frames to send. */
6690                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
6691
6692                 /* Stop when the transmit queue is empty. */
6693                 if (m_head == NULL)
6694                         break;
6695
6696                 /*
6697                  * Pack the data into the transmit ring. If we
6698                  * don't have room, place the mbuf back at the
6699                  * head of the queue and set the OACTIVE flag
6700                  * to wait for the NIC to drain the chain.
6701                  */
6702                 if (bce_tx_encap(sc, &m_head)) {
6703                         /* No room, put the frame back on the transmit queue. */
6704                         if (m_head != NULL)
6705                                 IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
6706                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
6707                         DBPRINT(sc, BCE_INFO_SEND,
6708                                 "TX chain is closed for business! Total tx_bd used = %d\n",
6709                                 sc->used_tx_bd);
6710                         break;
6711                 }
6712
6713                 count++;
6714
6715                 /* Send a copy of the frame to any BPF listeners. */
6716                 ETHER_BPF_MTAP(ifp, m_head);
6717         }
6718
6719         /* Exit if no packets were dequeued. */
6720         if (count == 0) {
6721                 DBPRINT(sc, BCE_VERBOSE_SEND, "%s(): No packets were dequeued\n",
6722                         __FUNCTION__);
6723                 goto bce_start_locked_exit;
6724         }
6725
6726         DBPRINT(sc, BCE_VERBOSE_SEND, "%s(): Inserted %d frames into send queue.\n",
6727                 __FUNCTION__, count);
6728
6729         REG_WR(sc, BCE_MQ_COMMAND, REG_RD(sc, BCE_MQ_COMMAND) | BCE_MQ_COMMAND_NO_MAP_ERROR);
6730
6731         /* Write the mailbox and tell the chip about the waiting tx_bd's. */
6732         DBPRINT(sc, BCE_VERBOSE_SEND, "%s(): MB_GET_CID_ADDR(TX_CID) = 0x%08X; "
6733                 "BCE_L2MQ_TX_HOST_BIDX = 0x%08X, sc->tx_prod = 0x%04X\n",
6734                 __FUNCTION__,
6735                 MB_GET_CID_ADDR(TX_CID), BCE_L2MQ_TX_HOST_BIDX, sc->tx_prod);
6736         REG_WR16(sc, MB_GET_CID_ADDR(TX_CID) + BCE_L2MQ_TX_HOST_BIDX, sc->tx_prod);
6737         DBPRINT(sc, BCE_VERBOSE_SEND, "%s(): MB_GET_CID_ADDR(TX_CID) = 0x%08X; "
6738                 "BCE_L2MQ_TX_HOST_BSEQ = 0x%08X, sc->tx_prod_bseq = 0x%04X\n",
6739                 __FUNCTION__,
6740                 MB_GET_CID_ADDR(TX_CID), BCE_L2MQ_TX_HOST_BSEQ, sc->tx_prod_bseq);
6741         REG_WR(sc, MB_GET_CID_ADDR(TX_CID) + BCE_L2MQ_TX_HOST_BSEQ, sc->tx_prod_bseq);
6742
6743         /* Set the tx timeout. */
6744         sc->watchdog_timer = BCE_TX_TIMEOUT;
6745
6746         DBRUNMSG(BCE_VERBOSE_SEND, bce_dump_ctx(sc, TX_CID));
6747         DBRUNMSG(BCE_VERBOSE_SEND, bce_dump_mq_regs(sc));
6748
6749 bce_start_locked_exit:
6750         DBEXIT(BCE_VERBOSE_SEND | BCE_VERBOSE_CTX);
6751         return;
6752 }
6753
6754
6755 /****************************************************************************/
6756 /* Main transmit routine when called from another routine without a lock.   */
6757 /*                                                                          */
6758 /* Returns:                                                                 */
6759 /*   Nothing.                                                               */
6760 /****************************************************************************/
6761 static void
6762 bce_start(struct ifnet *ifp)
6763 {
6764         struct bce_softc *sc = ifp->if_softc;
6765
6766         DBENTER(BCE_VERBOSE_SEND);
6767
6768         BCE_LOCK(sc);
6769         bce_start_locked(ifp);
6770         BCE_UNLOCK(sc);
6771
6772         DBEXIT(BCE_VERBOSE_SEND);
6773 }
6774
6775
6776 /****************************************************************************/
6777 /* Handles any IOCTL calls from the operating system.                       */
6778 /*                                                                          */
6779 /* Returns:                                                                 */
6780 /*   0 for success, positive value for failure.                             */
6781 /****************************************************************************/
6782 static int
6783 bce_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
6784 {
6785         struct bce_softc *sc = ifp->if_softc;
6786         struct ifreq *ifr = (struct ifreq *) data;
6787         struct mii_data *mii;
6788         int mask, error = 0;
6789
6790         DBENTER(BCE_VERBOSE_MISC);
6791
6792         switch(command) {
6793
6794                 /* Set the interface MTU. */
6795                 case SIOCSIFMTU:
6796                         /* Check that the MTU setting is supported. */
6797                         if ((ifr->ifr_mtu < BCE_MIN_MTU) ||
6798                                 (ifr->ifr_mtu > BCE_MAX_JUMBO_MTU)) {
6799                                 error = EINVAL;
6800                                 break;
6801                         }
6802
6803                         DBPRINT(sc, BCE_INFO_MISC,
6804                                 "SIOCSIFMTU: Changing MTU from %d to %d\n",
6805                                 (int) ifp->if_mtu, (int) ifr->ifr_mtu);
6806
6807                         BCE_LOCK(sc);
6808                         ifp->if_mtu = ifr->ifr_mtu;
6809                         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
6810 #ifdef ZERO_COPY_SOCKETS
6811                         /* No buffer allocation size changes are necessary. */
6812 #else
6813                         /* Recalculate our buffer allocation sizes. */
6814                         if ((ifp->if_mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + ETHER_CRC_LEN) > MCLBYTES) {
6815                                 sc->rx_bd_mbuf_alloc_size = MJUM9BYTES;
6816                                 sc->rx_bd_mbuf_align_pad  = roundup2(MJUM9BYTES, 16) - MJUM9BYTES;
6817                                 sc->rx_bd_mbuf_data_len   = sc->rx_bd_mbuf_alloc_size -
6818                                         sc->rx_bd_mbuf_align_pad;
6819                         } else {
6820                                 sc->rx_bd_mbuf_alloc_size = MCLBYTES;
6821                                 sc->rx_bd_mbuf_align_pad  = roundup2(MCLBYTES, 16) - MCLBYTES;
6822                                 sc->rx_bd_mbuf_data_len   = sc->rx_bd_mbuf_alloc_size -
6823                                         sc->rx_bd_mbuf_align_pad;
6824                         }
6825 #endif
6826
6827                         bce_init_locked(sc);
6828                         BCE_UNLOCK(sc);
6829                         break;
6830
6831                 /* Set interface flags. */
6832                 case SIOCSIFFLAGS:
6833                         DBPRINT(sc, BCE_VERBOSE_SPECIAL, "Received SIOCSIFFLAGS\n");
6834
6835                         BCE_LOCK(sc);
6836
6837                         /* Check if the interface is up. */
6838                         if (ifp->if_flags & IFF_UP) {
6839                                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
6840                                         /* Change promiscuous/multicast flags as necessary. */
6841                                         bce_set_rx_mode(sc);
6842                                 } else {
6843                                         /* Start the HW */
6844                                         bce_init_locked(sc);
6845                                 }
6846                         } else {
6847                                 /* The interface is down, check if driver is running. */
6848                                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
6849                                         bce_stop(sc);
6850
6851                                         /* If MFW is running, restart the controller a bit. */
6852                                         if (sc->bce_flags & BCE_MFW_ENABLE_FLAG) {
6853                                                 bce_reset(sc, BCE_DRV_MSG_CODE_RESET);
6854                                                 bce_chipinit(sc);
6855                                                 bce_mgmt_init_locked(sc);
6856                                         }
6857                                 }
6858                         }
6859
6860                         BCE_UNLOCK(sc);
6861                         error = 0;
6862
6863                         break;
6864
6865                 /* Add/Delete multicast address */
6866                 case SIOCADDMULTI:
6867                 case SIOCDELMULTI:
6868                         DBPRINT(sc, BCE_VERBOSE_MISC, "Received SIOCADDMULTI/SIOCDELMULTI\n");
6869
6870                         BCE_LOCK(sc);
6871                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
6872                                 bce_set_rx_mode(sc);
6873                                 error = 0;
6874                         }
6875                         BCE_UNLOCK(sc);
6876
6877                         break;
6878
6879                 /* Set/Get Interface media */
6880                 case SIOCSIFMEDIA:
6881                 case SIOCGIFMEDIA:
6882                         DBPRINT(sc, BCE_VERBOSE_MISC, "Received SIOCSIFMEDIA/SIOCGIFMEDIA\n");
6883
6884                         mii = device_get_softc(sc->bce_miibus);
6885                         error = ifmedia_ioctl(ifp, ifr,
6886                             &mii->mii_media, command);
6887                         break;
6888
6889                 /* Set interface capability */
6890                 case SIOCSIFCAP:
6891                         mask = ifr->ifr_reqcap ^ ifp->if_capenable;
6892                         DBPRINT(sc, BCE_INFO_MISC, "Received SIOCSIFCAP = 0x%08X\n", (u32) mask);
6893
6894                         /* Toggle the TX checksum capabilites enable flag. */
6895                         if (mask & IFCAP_TXCSUM) {
6896                                 ifp->if_capenable ^= IFCAP_TXCSUM;
6897                                 if (IFCAP_TXCSUM & ifp->if_capenable)
6898                                         ifp->if_hwassist = BCE_IF_HWASSIST;
6899                                 else
6900                                         ifp->if_hwassist = 0;
6901                         }
6902
6903                         /* Toggle the RX checksum capabilities enable flag. */
6904                         if (mask & IFCAP_RXCSUM) {
6905                                 ifp->if_capenable ^= IFCAP_RXCSUM;
6906                                 if (IFCAP_RXCSUM & ifp->if_capenable)
6907                                         ifp->if_hwassist = BCE_IF_HWASSIST;
6908                                 else
6909                                         ifp->if_hwassist = 0;
6910                         }
6911
6912                         /* Toggle the TSO capabilities enable flag. */
6913                         if (bce_tso_enable && (mask & IFCAP_TSO4)) {
6914                                 ifp->if_capenable ^= IFCAP_TSO4;
6915                                 if (IFCAP_RXCSUM & ifp->if_capenable)
6916                                         ifp->if_hwassist = BCE_IF_HWASSIST;
6917                                 else
6918                                         ifp->if_hwassist = 0;
6919                         }
6920
6921                         /* Toggle VLAN_MTU capabilities enable flag. */
6922                         if (mask & IFCAP_VLAN_MTU) {
6923                                 BCE_PRINTF("%s(%d): Changing VLAN_MTU not supported.\n",
6924                                         __FILE__, __LINE__);
6925                         }
6926
6927                         /* Toggle VLANHWTAG capabilities enabled flag. */
6928                         if (mask & IFCAP_VLAN_HWTAGGING) {
6929                                 if (sc->bce_flags & BCE_MFW_ENABLE_FLAG)
6930                                         BCE_PRINTF("%s(%d): Cannot change VLAN_HWTAGGING while "
6931                                                 "management firmware (ASF/IPMI/UMP) is running!\n",
6932                                                 __FILE__, __LINE__);
6933                                 else
6934                                         BCE_PRINTF("%s(%d): Changing VLAN_HWTAGGING not supported!\n",
6935                                                 __FILE__, __LINE__);
6936                         }
6937
6938                         break;
6939                 default:
6940                         /* We don't know how to handle the IOCTL, pass it on. */
6941                         error = ether_ioctl(ifp, command, data);
6942                         break;
6943         }
6944
6945         DBEXIT(BCE_VERBOSE_MISC);
6946         return(error);
6947 }
6948
6949
6950 /****************************************************************************/
6951 /* Transmit timeout handler.                                                */
6952 /*                                                                          */
6953 /* Returns:                                                                 */
6954 /*   Nothing.                                                               */
6955 /****************************************************************************/
6956 static void
6957 bce_watchdog(struct bce_softc *sc)
6958 {
6959         DBENTER(BCE_EXTREME_SEND);
6960
6961         BCE_LOCK_ASSERT(sc);
6962
6963         /* If the watchdog timer hasn't expired then just exit. */
6964         if (sc->watchdog_timer == 0 || --sc->watchdog_timer)
6965                 goto bce_watchdog_exit;
6966
6967         /* If pause frames are active then don't reset the hardware. */
6968         /* ToDo: Should we reset the timer here? */
6969         if (REG_RD(sc, BCE_EMAC_TX_STATUS) & BCE_EMAC_TX_STATUS_XOFFED)
6970                 goto bce_watchdog_exit;
6971
6972         BCE_PRINTF("%s(%d): Watchdog timeout occurred, resetting!\n",
6973                 __FILE__, __LINE__);
6974
6975         DBRUNMSG(BCE_INFO,
6976                 bce_dump_driver_state(sc);
6977                 bce_dump_status_block(sc);
6978                 bce_dump_stats_block(sc);
6979                 bce_dump_ftqs(sc);
6980                 bce_dump_txp_state(sc, 0);
6981                 bce_dump_rxp_state(sc, 0);
6982                 bce_dump_tpat_state(sc, 0);
6983                 bce_dump_cp_state(sc, 0);
6984                 bce_dump_com_state(sc, 0));
6985
6986         DBRUN(bce_breakpoint(sc));
6987
6988         sc->bce_ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
6989
6990         bce_init_locked(sc);
6991         sc->bce_ifp->if_oerrors++;
6992
6993 bce_watchdog_exit:
6994         DBEXIT(BCE_EXTREME_SEND);
6995 }
6996
6997
6998 /*
6999  * Interrupt handler.
7000  */
7001 /****************************************************************************/
7002 /* Main interrupt entry point.  Verifies that the controller generated the  */
7003 /* interrupt and then calls a separate routine for handle the various       */
7004 /* interrupt causes (PHY, TX, RX).                                          */
7005 /*                                                                          */
7006 /* Returns:                                                                 */
7007 /*   0 for success, positive value for failure.                             */
7008 /****************************************************************************/
7009 static void
7010 bce_intr(void *xsc)
7011 {
7012         struct bce_softc *sc;
7013         struct ifnet *ifp;
7014         u32 status_attn_bits;
7015         u16 hw_rx_cons, hw_tx_cons;
7016
7017         sc = xsc;
7018         ifp = sc->bce_ifp;
7019
7020         DBENTER(BCE_VERBOSE_SEND | BCE_VERBOSE_RECV | BCE_VERBOSE_INTR);
7021         DBRUNMSG(BCE_VERBOSE_INTR, bce_dump_status_block(sc));
7022
7023         BCE_LOCK(sc);
7024
7025         DBRUN(sc->interrupts_generated++);
7026
7027         bus_dmamap_sync(sc->status_tag, sc->status_map,
7028             BUS_DMASYNC_POSTWRITE);
7029
7030         /*
7031          * If the hardware status block index
7032          * matches the last value read by the
7033          * driver and we haven't asserted our
7034          * interrupt then there's nothing to do.
7035          */
7036         if ((sc->status_block->status_idx == sc->last_status_idx) &&
7037                 (REG_RD(sc, BCE_PCICFG_MISC_STATUS) & BCE_PCICFG_MISC_STATUS_INTA_VALUE)) {
7038                         DBPRINT(sc, BCE_VERBOSE_INTR, "%s(): Spurious interrupt.\n",
7039                                 __FUNCTION__);
7040                         goto bce_intr_exit;
7041         }
7042
7043         /* Ack the interrupt and stop others from occuring. */
7044         REG_WR(sc, BCE_PCICFG_INT_ACK_CMD,
7045                 BCE_PCICFG_INT_ACK_CMD_USE_INT_HC_PARAM |
7046                 BCE_PCICFG_INT_ACK_CMD_MASK_INT);
7047
7048         /* Check if the hardware has finished any work. */
7049         hw_rx_cons = bce_get_hw_rx_cons(sc);
7050         hw_tx_cons = bce_get_hw_tx_cons(sc);
7051
7052         /* Keep processing data as long as there is work to do. */
7053         for (;;) {
7054
7055                 status_attn_bits = sc->status_block->status_attn_bits;
7056
7057         DBRUNIF(DB_RANDOMTRUE(unexpected_attention_sim_control),
7058                 BCE_PRINTF("Simulating unexpected status attention bit set.");
7059                 sc->unexpected_attention_sim_count++;
7060                 status_attn_bits = status_attn_bits | STATUS_ATTN_BITS_PARITY_ERROR);
7061
7062                 /* Was it a link change interrupt? */
7063                 if ((status_attn_bits & STATUS_ATTN_BITS_LINK_STATE) !=
7064                         (sc->status_block->status_attn_bits_ack & STATUS_ATTN_BITS_LINK_STATE)) {
7065                         bce_phy_intr(sc);
7066
7067                         /* Clear any transient status updates during link state change. */
7068                         REG_WR(sc, BCE_HC_COMMAND,
7069                                 sc->hc_command | BCE_HC_COMMAND_COAL_NOW_WO_INT);
7070                         REG_RD(sc, BCE_HC_COMMAND);
7071                 }
7072
7073                 /* If any other attention is asserted then the chip is toast. */
7074                 if (((status_attn_bits & ~STATUS_ATTN_BITS_LINK_STATE) !=
7075                         (sc->status_block->status_attn_bits_ack &
7076                         ~STATUS_ATTN_BITS_LINK_STATE))) {
7077
7078                 sc->unexpected_attention_count++;
7079
7080                         BCE_PRINTF("%s(%d): Fatal attention detected: 0x%08X\n",
7081                                 __FILE__, __LINE__, sc->status_block->status_attn_bits);
7082
7083                         DBRUNMSG(BCE_FATAL,
7084                                 if (unexpected_attention_sim_control == 0)
7085                                         bce_breakpoint(sc));
7086
7087                         bce_init_locked(sc);
7088                         goto bce_intr_exit;
7089                 }
7090
7091                 /* Check for any completed RX frames. */
7092                 if (hw_rx_cons != sc->hw_rx_cons)
7093                         bce_rx_intr(sc);
7094
7095                 /* Check for any completed TX frames. */
7096                 if (hw_tx_cons != sc->hw_tx_cons)
7097                         bce_tx_intr(sc);
7098
7099                 /* Save the status block index value for use during the next interrupt. */
7100                 sc->last_status_idx = sc->status_block->status_idx;
7101
7102                 /* Prevent speculative reads from getting ahead of the status block. */
7103                 bus_space_barrier(sc->bce_btag, sc->bce_bhandle, 0, 0,
7104                         BUS_SPACE_BARRIER_READ);
7105
7106                 /* If there's no work left then exit the interrupt service routine. */
7107                 hw_rx_cons = bce_get_hw_rx_cons(sc);
7108                 hw_tx_cons = bce_get_hw_tx_cons(sc);
7109
7110                 if ((hw_rx_cons == sc->hw_rx_cons) && (hw_tx_cons == sc->hw_tx_cons))
7111                         break;
7112
7113         }
7114
7115         bus_dmamap_sync(sc->status_tag, sc->status_map,
7116             BUS_DMASYNC_PREWRITE);
7117
7118         /* Re-enable interrupts. */
7119         bce_enable_intr(sc, 0);
7120
7121         /* Handle any frames that arrived while handling the interrupt. */
7122         if (ifp->if_drv_flags & IFF_DRV_RUNNING && !IFQ_DRV_IS_EMPTY(&ifp->if_snd))
7123                 bce_start_locked(ifp);
7124
7125 bce_intr_exit:
7126         BCE_UNLOCK(sc);
7127
7128         DBEXIT(BCE_VERBOSE_SEND | BCE_VERBOSE_RECV | BCE_VERBOSE_INTR);
7129 }
7130
7131
7132 /****************************************************************************/
7133 /* Programs the various packet receive modes (broadcast and multicast).     */
7134 /*                                                                          */
7135 /* Returns:                                                                 */
7136 /*   Nothing.                                                               */
7137 /****************************************************************************/
7138 static void
7139 bce_set_rx_mode(struct bce_softc *sc)
7140 {
7141         struct ifnet *ifp;
7142         struct ifmultiaddr *ifma;
7143         u32 hashes[NUM_MC_HASH_REGISTERS] = { 0, 0, 0, 0, 0, 0, 0, 0 };
7144         u32 rx_mode, sort_mode;
7145         int h, i;
7146
7147         DBENTER(BCE_VERBOSE_MISC);
7148
7149         BCE_LOCK_ASSERT(sc);
7150
7151         ifp = sc->bce_ifp;
7152
7153         /* Initialize receive mode default settings. */
7154         rx_mode   = sc->rx_mode & ~(BCE_EMAC_RX_MODE_PROMISCUOUS |
7155                             BCE_EMAC_RX_MODE_KEEP_VLAN_TAG);
7156         sort_mode = 1 | BCE_RPM_SORT_USER0_BC_EN;
7157
7158         /*
7159          * ASF/IPMI/UMP firmware requires that VLAN tag stripping
7160          * be enbled.
7161          */
7162         if (!(BCE_IF_CAPABILITIES & IFCAP_VLAN_HWTAGGING) &&
7163                 (!(sc->bce_flags & BCE_MFW_ENABLE_FLAG)))
7164                 rx_mode |= BCE_EMAC_RX_MODE_KEEP_VLAN_TAG;
7165
7166         /*
7167          * Check for promiscuous, all multicast, or selected
7168          * multicast address filtering.
7169          */
7170         if (ifp->if_flags & IFF_PROMISC) {
7171                 DBPRINT(sc, BCE_INFO_MISC, "Enabling promiscuous mode.\n");
7172
7173                 /* Enable promiscuous mode. */
7174                 rx_mode |= BCE_EMAC_RX_MODE_PROMISCUOUS;
7175                 sort_mode |= BCE_RPM_SORT_USER0_PROM_EN;
7176         } else if (ifp->if_flags & IFF_ALLMULTI) {
7177                 DBPRINT(sc, BCE_INFO_MISC, "Enabling all multicast mode.\n");
7178
7179                 /* Enable all multicast addresses. */
7180                 for (i = 0; i < NUM_MC_HASH_REGISTERS; i++) {
7181                         REG_WR(sc, BCE_EMAC_MULTICAST_HASH0 + (i * 4), 0xffffffff);
7182         }
7183                 sort_mode |= BCE_RPM_SORT_USER0_MC_EN;
7184         } else {
7185                 /* Accept one or more multicast(s). */
7186                 DBPRINT(sc, BCE_INFO_MISC, "Enabling selective multicast mode.\n");
7187
7188                 IF_ADDR_LOCK(ifp);
7189                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
7190                         if (ifma->ifma_addr->sa_family != AF_LINK)
7191                                 continue;
7192                         h = ether_crc32_le(LLADDR((struct sockaddr_dl *)
7193                             ifma->ifma_addr), ETHER_ADDR_LEN) & 0xFF;
7194                             hashes[(h & 0xE0) >> 5] |= 1 << (h & 0x1F);
7195                 }
7196                 IF_ADDR_UNLOCK(ifp);
7197
7198                 for (i = 0; i < NUM_MC_HASH_REGISTERS; i++)
7199                         REG_WR(sc, BCE_EMAC_MULTICAST_HASH0 + (i * 4), hashes[i]);
7200
7201                 sort_mode |= BCE_RPM_SORT_USER0_MC_HSH_EN;
7202         }
7203
7204         /* Only make changes if the recive mode has actually changed. */
7205         if (rx_mode != sc->rx_mode) {
7206                 DBPRINT(sc, BCE_VERBOSE_MISC, "Enabling new receive mode: 0x%08X\n",
7207                         rx_mode);
7208
7209                 sc->rx_mode = rx_mode;
7210                 REG_WR(sc, BCE_EMAC_RX_MODE, rx_mode);
7211         }
7212
7213         /* Disable and clear the exisitng sort before enabling a new sort. */
7214         REG_WR(sc, BCE_RPM_SORT_USER0, 0x0);
7215         REG_WR(sc, BCE_RPM_SORT_USER0, sort_mode);
7216         REG_WR(sc, BCE_RPM_SORT_USER0, sort_mode | BCE_RPM_SORT_USER0_ENA);
7217
7218         DBEXIT(BCE_VERBOSE_MISC);
7219 }
7220
7221
7222 /****************************************************************************/
7223 /* Called periodically to updates statistics from the controllers           */
7224 /* statistics block.                                                        */
7225 /*                                                                          */
7226 /* Returns:                                                                 */
7227 /*   Nothing.                                                               */
7228 /****************************************************************************/
7229 static void
7230 bce_stats_update(struct bce_softc *sc)
7231 {
7232         struct ifnet *ifp;
7233         struct statistics_block *stats;
7234
7235         DBENTER(BCE_EXTREME_MISC);
7236
7237         ifp = sc->bce_ifp;
7238
7239         stats = (struct statistics_block *) sc->stats_block;
7240
7241         /*
7242          * Certain controllers don't report
7243          * carrier sense errors correctly.
7244          * See errata E11_5708CA0_1165.
7245          */
7246         if (!(BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5706) &&
7247             !(BCE_CHIP_ID(sc) == BCE_CHIP_ID_5708_A0))
7248                 ifp->if_oerrors += (u_long) stats->stat_Dot3StatsCarrierSenseErrors;
7249
7250         /*
7251          * Update the sysctl statistics from the
7252          * hardware statistics.
7253          */
7254         sc->stat_IfHCInOctets =
7255                 ((u64) stats->stat_IfHCInOctets_hi << 32) +
7256                  (u64) stats->stat_IfHCInOctets_lo;
7257
7258         sc->stat_IfHCInBadOctets =
7259                 ((u64) stats->stat_IfHCInBadOctets_hi << 32) +
7260                  (u64) stats->stat_IfHCInBadOctets_lo;
7261
7262         sc->stat_IfHCOutOctets =
7263                 ((u64) stats->stat_IfHCOutOctets_hi << 32) +
7264                  (u64) stats->stat_IfHCOutOctets_lo;
7265
7266         sc->stat_IfHCOutBadOctets =
7267                 ((u64) stats->stat_IfHCOutBadOctets_hi << 32) +
7268                  (u64) stats->stat_IfHCOutBadOctets_lo;
7269
7270         sc->stat_IfHCInUcastPkts =
7271                 ((u64) stats->stat_IfHCInUcastPkts_hi << 32) +
7272                  (u64) stats->stat_IfHCInUcastPkts_lo;
7273
7274         sc->stat_IfHCInMulticastPkts =
7275                 ((u64) stats->stat_IfHCInMulticastPkts_hi << 32) +
7276                  (u64) stats->stat_IfHCInMulticastPkts_lo;
7277
7278         sc->stat_IfHCInBroadcastPkts =
7279                 ((u64) stats->stat_IfHCInBroadcastPkts_hi << 32) +
7280                  (u64) stats->stat_IfHCInBroadcastPkts_lo;
7281
7282         sc->stat_IfHCOutUcastPkts =
7283                 ((u64) stats->stat_IfHCOutUcastPkts_hi << 32) +
7284                  (u64) stats->stat_IfHCOutUcastPkts_lo;
7285
7286         sc->stat_IfHCOutMulticastPkts =
7287                 ((u64) stats->stat_IfHCOutMulticastPkts_hi << 32) +
7288                  (u64) stats->stat_IfHCOutMulticastPkts_lo;
7289
7290         sc->stat_IfHCOutBroadcastPkts =
7291                 ((u64) stats->stat_IfHCOutBroadcastPkts_hi << 32) +
7292                  (u64) stats->stat_IfHCOutBroadcastPkts_lo;
7293
7294         sc->stat_emac_tx_stat_dot3statsinternalmactransmiterrors =
7295                 stats->stat_emac_tx_stat_dot3statsinternalmactransmiterrors;
7296
7297         sc->stat_Dot3StatsCarrierSenseErrors =
7298                 stats->stat_Dot3StatsCarrierSenseErrors;
7299
7300         sc->stat_Dot3StatsFCSErrors =
7301                 stats->stat_Dot3StatsFCSErrors;
7302
7303         sc->stat_Dot3StatsAlignmentErrors =
7304                 stats->stat_Dot3StatsAlignmentErrors;
7305
7306         sc->stat_Dot3StatsSingleCollisionFrames =
7307                 stats->stat_Dot3StatsSingleCollisionFrames;
7308
7309         sc->stat_Dot3StatsMultipleCollisionFrames =
7310                 stats->stat_Dot3StatsMultipleCollisionFrames;
7311
7312         sc->stat_Dot3StatsDeferredTransmissions =
7313                 stats->stat_Dot3StatsDeferredTransmissions;
7314
7315         sc->stat_Dot3StatsExcessiveCollisions =
7316                 stats->stat_Dot3StatsExcessiveCollisions;
7317
7318         sc->stat_Dot3StatsLateCollisions =
7319                 stats->stat_Dot3StatsLateCollisions;
7320
7321         sc->stat_EtherStatsCollisions =
7322                 stats->stat_EtherStatsCollisions;
7323
7324         sc->stat_EtherStatsFragments =
7325                 stats->stat_EtherStatsFragments;
7326
7327         sc->stat_EtherStatsJabbers =
7328                 stats->stat_EtherStatsJabbers;
7329
7330         sc->stat_EtherStatsUndersizePkts =
7331                 stats->stat_EtherStatsUndersizePkts;
7332
7333         sc->stat_EtherStatsOversizePkts =
7334                 stats->stat_EtherStatsOversizePkts;
7335
7336         sc->stat_EtherStatsPktsRx64Octets =
7337                 stats->stat_EtherStatsPktsRx64Octets;
7338
7339         sc->stat_EtherStatsPktsRx65Octetsto127Octets =
7340                 stats->stat_EtherStatsPktsRx65Octetsto127Octets;
7341
7342         sc->stat_EtherStatsPktsRx128Octetsto255Octets =
7343                 stats->stat_EtherStatsPktsRx128Octetsto255Octets;
7344
7345         sc->stat_EtherStatsPktsRx256Octetsto511Octets =
7346                 stats->stat_EtherStatsPktsRx256Octetsto511Octets;
7347
7348         sc->stat_EtherStatsPktsRx512Octetsto1023Octets =
7349                 stats->stat_EtherStatsPktsRx512Octetsto1023Octets;
7350
7351         sc->stat_EtherStatsPktsRx1024Octetsto1522Octets =
7352                 stats->stat_EtherStatsPktsRx1024Octetsto1522Octets;
7353
7354         sc->stat_EtherStatsPktsRx1523Octetsto9022Octets =
7355                 stats->stat_EtherStatsPktsRx1523Octetsto9022Octets;
7356
7357         sc->stat_EtherStatsPktsTx64Octets =
7358                 stats->stat_EtherStatsPktsTx64Octets;
7359
7360         sc->stat_EtherStatsPktsTx65Octetsto127Octets =
7361                 stats->stat_EtherStatsPktsTx65Octetsto127Octets;
7362
7363         sc->stat_EtherStatsPktsTx128Octetsto255Octets =
7364                 stats->stat_EtherStatsPktsTx128Octetsto255Octets;
7365
7366         sc->stat_EtherStatsPktsTx256Octetsto511Octets =
7367                 stats->stat_EtherStatsPktsTx256Octetsto511Octets;
7368
7369         sc->stat_EtherStatsPktsTx512Octetsto1023Octets =
7370                 stats->stat_EtherStatsPktsTx512Octetsto1023Octets;
7371
7372         sc->stat_EtherStatsPktsTx1024Octetsto1522Octets =
7373                 stats->stat_EtherStatsPktsTx1024Octetsto1522Octets;
7374
7375         sc->stat_EtherStatsPktsTx1523Octetsto9022Octets =
7376                 stats->stat_EtherStatsPktsTx1523Octetsto9022Octets;
7377
7378         sc->stat_XonPauseFramesReceived =
7379                 stats->stat_XonPauseFramesReceived;
7380
7381         sc->stat_XoffPauseFramesReceived =
7382                 stats->stat_XoffPauseFramesReceived;
7383
7384         sc->stat_OutXonSent =
7385                 stats->stat_OutXonSent;
7386
7387         sc->stat_OutXoffSent =
7388                 stats->stat_OutXoffSent;
7389
7390         sc->stat_FlowControlDone =
7391                 stats->stat_FlowControlDone;
7392
7393         sc->stat_MacControlFramesReceived =
7394                 stats->stat_MacControlFramesReceived;
7395
7396         sc->stat_XoffStateEntered =
7397                 stats->stat_XoffStateEntered;
7398
7399         sc->stat_IfInFramesL2FilterDiscards =
7400                 stats->stat_IfInFramesL2FilterDiscards;
7401
7402         sc->stat_IfInRuleCheckerDiscards =
7403                 stats->stat_IfInRuleCheckerDiscards;
7404
7405         sc->stat_IfInFTQDiscards =
7406                 stats->stat_IfInFTQDiscards;
7407
7408         sc->stat_IfInMBUFDiscards =
7409                 stats->stat_IfInMBUFDiscards;
7410
7411         sc->stat_IfInRuleCheckerP4Hit =
7412                 stats->stat_IfInRuleCheckerP4Hit;
7413
7414         sc->stat_CatchupInRuleCheckerDiscards =
7415                 stats->stat_CatchupInRuleCheckerDiscards;
7416
7417         sc->stat_CatchupInFTQDiscards =
7418                 stats->stat_CatchupInFTQDiscards;
7419
7420         sc->stat_CatchupInMBUFDiscards =
7421                 stats->stat_CatchupInMBUFDiscards;
7422
7423         sc->stat_CatchupInRuleCheckerP4Hit =
7424                 stats->stat_CatchupInRuleCheckerP4Hit;
7425
7426         sc->com_no_buffers = REG_RD_IND(sc, 0x120084);
7427
7428         /*
7429          * Update the interface statistics from the
7430          * hardware statistics.
7431          */
7432         ifp->if_collisions =
7433                 (u_long) sc->stat_EtherStatsCollisions;
7434
7435         /* ToDo: This method loses soft errors. */
7436         ifp->if_ierrors =
7437                 (u_long) sc->stat_EtherStatsUndersizePkts +
7438                 (u_long) sc->stat_EtherStatsOversizePkts +
7439                 (u_long) sc->stat_IfInMBUFDiscards +
7440                 (u_long) sc->stat_Dot3StatsAlignmentErrors +
7441                 (u_long) sc->stat_Dot3StatsFCSErrors +
7442                 (u_long) sc->stat_IfInRuleCheckerDiscards +
7443                 (u_long) sc->stat_IfInFTQDiscards +
7444                 (u_long) sc->com_no_buffers;
7445
7446         /* ToDo: This method loses soft errors. */
7447         ifp->if_oerrors =
7448                 (u_long) sc->stat_emac_tx_stat_dot3statsinternalmactransmiterrors +
7449                 (u_long) sc->stat_Dot3StatsExcessiveCollisions +
7450                 (u_long) sc->stat_Dot3StatsLateCollisions;
7451
7452         /* ToDo: Add additional statistics. */
7453
7454         DBEXIT(BCE_EXTREME_MISC);
7455 }
7456
7457
7458 /****************************************************************************/
7459 /* Periodic function to notify the bootcode that the driver is still        */
7460 /* present.                                                                 */
7461 /*                                                                          */
7462 /* Returns:                                                                 */
7463 /*   Nothing.                                                               */
7464 /****************************************************************************/
7465 static void
7466 bce_pulse(void *xsc)
7467 {
7468         struct bce_softc *sc = xsc;
7469         u32 msg;
7470
7471         DBENTER(BCE_EXTREME_MISC);
7472
7473         BCE_LOCK_ASSERT(sc);
7474
7475         /* Tell the firmware that the driver is still running. */
7476         msg = (u32) ++sc->bce_fw_drv_pulse_wr_seq;
7477         REG_WR_IND(sc, sc->bce_shmem_base + BCE_DRV_PULSE_MB, msg);
7478
7479         /* Schedule the next pulse. */
7480         callout_reset(&sc->bce_pulse_callout, hz, bce_pulse, sc);
7481
7482         DBEXIT(BCE_EXTREME_MISC);
7483 }
7484
7485
7486 /****************************************************************************/
7487 /* Periodic function to perform maintenance tasks.                          */
7488 /*                                                                          */
7489 /* Returns:                                                                 */
7490 /*   Nothing.                                                               */
7491 /****************************************************************************/
7492 static void
7493 bce_tick(void *xsc)
7494 {
7495         struct bce_softc *sc = xsc;
7496         struct mii_data *mii;
7497         struct ifnet *ifp;
7498
7499         ifp = sc->bce_ifp;
7500
7501         DBENTER(BCE_EXTREME_MISC);
7502
7503         BCE_LOCK_ASSERT(sc);
7504
7505         /* Schedule the next tick. */
7506         callout_reset(&sc->bce_tick_callout, hz, bce_tick, sc);
7507
7508         /* Update the statistics from the hardware statistics block. */
7509         bce_stats_update(sc);
7510
7511         /* Top off the receive and page chains. */
7512 #ifdef ZERO_COPY_SOCKETS
7513         bce_fill_pg_chain(sc);
7514 #endif
7515         bce_fill_rx_chain(sc);
7516
7517         /* Check that chip hasn't hung. */
7518         bce_watchdog(sc);
7519
7520         /* If link is up already up then we're done. */
7521         if (sc->bce_link)
7522                 goto bce_tick_exit;
7523
7524         /* Link is down.  Check what the PHY's doing. */
7525         mii = device_get_softc(sc->bce_miibus);
7526         mii_tick(mii);
7527
7528         /* Check if the link has come up. */
7529         if ((mii->mii_media_status & IFM_ACTIVE) &&
7530             (IFM_SUBTYPE(mii->mii_media_active) != IFM_NONE)) {
7531                 DBPRINT(sc, BCE_VERBOSE_MISC, "%s(): Link up!\n", __FUNCTION__);
7532                 sc->bce_link++;
7533                 if ((IFM_SUBTYPE(mii->mii_media_active) == IFM_1000_T ||
7534                     IFM_SUBTYPE(mii->mii_media_active) == IFM_1000_SX) &&
7535                     bootverbose)
7536                         BCE_PRINTF("Gigabit link up!\n");
7537                 /* Now that link is up, handle any outstanding TX traffic. */
7538                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
7539                         DBPRINT(sc, BCE_VERBOSE_MISC, "%s(): Found pending TX traffic.\n",
7540                                  __FUNCTION__);
7541                         bce_start_locked(ifp);
7542                 }
7543         }
7544
7545 bce_tick_exit:
7546         DBEXIT(BCE_EXTREME_MISC);
7547         return;
7548 }
7549
7550
7551 #ifdef BCE_DEBUG
7552 /****************************************************************************/
7553 /* Allows the driver state to be dumped through the sysctl interface.       */
7554 /*                                                                          */
7555 /* Returns:                                                                 */
7556 /*   0 for success, positive value for failure.                             */
7557 /****************************************************************************/
7558 static int
7559 bce_sysctl_driver_state(SYSCTL_HANDLER_ARGS)
7560 {
7561         int error;
7562         int result;
7563         struct bce_softc *sc;
7564
7565         result = -1;
7566         error = sysctl_handle_int(oidp, &result, 0, req);
7567
7568         if (error || !req->newptr)
7569                 return (error);
7570
7571         if (result == 1) {
7572                 sc = (struct bce_softc *)arg1;
7573                 bce_dump_driver_state(sc);
7574         }
7575
7576         return error;
7577 }
7578
7579
7580 /****************************************************************************/
7581 /* Allows the hardware state to be dumped through the sysctl interface.     */
7582 /*                                                                          */
7583 /* Returns:                                                                 */
7584 /*   0 for success, positive value for failure.                             */
7585 /****************************************************************************/
7586 static int
7587 bce_sysctl_hw_state(SYSCTL_HANDLER_ARGS)
7588 {
7589         int error;
7590         int result;
7591         struct bce_softc *sc;
7592
7593         result = -1;
7594         error = sysctl_handle_int(oidp, &result, 0, req);
7595
7596         if (error || !req->newptr)
7597                 return (error);
7598
7599         if (result == 1) {
7600                 sc = (struct bce_softc *)arg1;
7601                 bce_dump_hw_state(sc);
7602         }
7603
7604         return error;
7605 }
7606
7607
7608 /****************************************************************************/
7609 /* Allows the bootcode state to be dumped through the sysctl interface.     */
7610 /*                                                                          */
7611 /* Returns:                                                                 */
7612 /*   0 for success, positive value for failure.                             */
7613 /****************************************************************************/
7614 static int
7615 bce_sysctl_bc_state(SYSCTL_HANDLER_ARGS)
7616 {
7617         int error;
7618         int result;
7619         struct bce_softc *sc;
7620
7621         result = -1;
7622         error = sysctl_handle_int(oidp, &result, 0, req);
7623
7624         if (error || !req->newptr)
7625                 return (error);
7626
7627         if (result == 1) {
7628                 sc = (struct bce_softc *)arg1;
7629                 bce_dump_bc_state(sc);
7630         }
7631
7632         return error;
7633 }
7634
7635
7636 /****************************************************************************/
7637 /* Provides a sysctl interface to allow dumping the RX chain.               */
7638 /*                                                                          */
7639 /* Returns:                                                                 */
7640 /*   0 for success, positive value for failure.                             */
7641 /****************************************************************************/
7642 static int
7643 bce_sysctl_dump_rx_chain(SYSCTL_HANDLER_ARGS)
7644 {
7645         int error;
7646         int result;
7647         struct bce_softc *sc;
7648
7649         result = -1;
7650         error = sysctl_handle_int(oidp, &result, 0, req);
7651
7652         if (error || !req->newptr)
7653                 return (error);
7654
7655         if (result == 1) {
7656                 sc = (struct bce_softc *)arg1;
7657                 bce_dump_rx_chain(sc, 0, TOTAL_RX_BD);
7658         }
7659
7660         return error;
7661 }
7662
7663
7664 /****************************************************************************/
7665 /* Provides a sysctl interface to allow dumping the TX chain.               */
7666 /*                                                                          */
7667 /* Returns:                                                                 */
7668 /*   0 for success, positive value for failure.                             */
7669 /****************************************************************************/
7670 static int
7671 bce_sysctl_dump_tx_chain(SYSCTL_HANDLER_ARGS)
7672 {
7673         int error;
7674         int result;
7675         struct bce_softc *sc;
7676
7677         result = -1;
7678         error = sysctl_handle_int(oidp, &result, 0, req);
7679
7680         if (error || !req->newptr)
7681                 return (error);
7682
7683         if (result == 1) {
7684                 sc = (struct bce_softc *)arg1;
7685                 bce_dump_tx_chain(sc, 0, USABLE_TX_BD);
7686         }
7687
7688         return error;
7689 }
7690
7691
7692 #ifdef ZERO_COPY_SOCKETS
7693 /****************************************************************************/
7694 /* Provides a sysctl interface to allow dumping the page chain.             */
7695 /*                                                                          */
7696 /* Returns:                                                                 */
7697 /*   0 for success, positive value for failure.                             */
7698 /****************************************************************************/
7699 static int
7700 bce_sysctl_dump_pg_chain(SYSCTL_HANDLER_ARGS)
7701 {
7702         int error;
7703         int result;
7704         struct bce_softc *sc;
7705
7706         result = -1;
7707         error = sysctl_handle_int(oidp, &result, 0, req);
7708
7709         if (error || !req->newptr)
7710                 return (error);
7711
7712         if (result == 1) {
7713                 sc = (struct bce_softc *)arg1;
7714                 bce_dump_pg_chain(sc, 0, TOTAL_PG_BD);
7715         }
7716
7717         return error;
7718 }
7719 #endif
7720
7721 /****************************************************************************/
7722 /* Provides a sysctl interface to allow reading arbitrary NVRAM offsets in  */
7723 /* the device.  DO NOT ENABLE ON PRODUCTION SYSTEMS!                        */
7724 /*                                                                          */
7725 /* Returns:                                                                 */
7726 /*   0 for success, positive value for failure.                             */
7727 /****************************************************************************/
7728 static int
7729 bce_sysctl_nvram_read(SYSCTL_HANDLER_ARGS)
7730 {
7731         struct bce_softc *sc = (struct bce_softc *)arg1;
7732         int error;
7733         u32 result;
7734         u32 val[1];
7735         u8 *data = (u8 *) val;
7736
7737         result = -1;
7738         error = sysctl_handle_int(oidp, &result, 0, req);
7739         if (error || (req->newptr == NULL))
7740                 return (error);
7741
7742         bce_nvram_read(sc, result, data, 4);
7743         BCE_PRINTF("offset 0x%08X = 0x%08X\n", result, bce_be32toh(val[0]));
7744
7745         return (error);
7746 }
7747
7748
7749 /****************************************************************************/
7750 /* Provides a sysctl interface to allow reading arbitrary registers in the  */
7751 /* device.  DO NOT ENABLE ON PRODUCTION SYSTEMS!                            */
7752 /*                                                                          */
7753 /* Returns:                                                                 */
7754 /*   0 for success, positive value for failure.                             */
7755 /****************************************************************************/
7756 static int
7757 bce_sysctl_reg_read(SYSCTL_HANDLER_ARGS)
7758 {
7759         struct bce_softc *sc = (struct bce_softc *)arg1;
7760         int error;
7761         u32 val, result;
7762
7763         result = -1;
7764         error = sysctl_handle_int(oidp, &result, 0, req);
7765         if (error || (req->newptr == NULL))
7766                 return (error);
7767
7768         /* Make sure the register is accessible. */
7769         if (result < 0x8000) {
7770                 val = REG_RD(sc, result);
7771                 BCE_PRINTF("reg 0x%08X = 0x%08X\n", result, val);
7772         } else if (result < 0x0280000) {
7773                 val = REG_RD_IND(sc, result);
7774                 BCE_PRINTF("reg 0x%08X = 0x%08X\n", result, val);
7775         }
7776
7777         return (error);
7778 }
7779
7780
7781 /****************************************************************************/
7782 /* Provides a sysctl interface to allow reading arbitrary PHY registers in  */
7783 /* the device.  DO NOT ENABLE ON PRODUCTION SYSTEMS!                        */
7784 /*                                                                          */
7785 /* Returns:                                                                 */
7786 /*   0 for success, positive value for failure.                             */
7787 /****************************************************************************/
7788 static int
7789 bce_sysctl_phy_read(SYSCTL_HANDLER_ARGS)
7790 {
7791         struct bce_softc *sc;
7792         device_t dev;
7793         int error, result;
7794         u16 val;
7795
7796         result = -1;
7797         error = sysctl_handle_int(oidp, &result, 0, req);
7798         if (error || (req->newptr == NULL))
7799                 return (error);
7800
7801         /* Make sure the register is accessible. */
7802         if (result < 0x20) {
7803                 sc = (struct bce_softc *)arg1;
7804                 dev = sc->bce_dev;
7805                 val = bce_miibus_read_reg(dev, sc->bce_phy_addr, result);
7806                 BCE_PRINTF("phy 0x%02X = 0x%04X\n", result, val);
7807         }
7808         return (error);
7809 }
7810
7811
7812 /****************************************************************************/
7813 /* Provides a sysctl interface to allow reading a CID.                      */
7814 /*                                                                          */
7815 /* Returns:                                                                 */
7816 /*   0 for success, positive value for failure.                             */
7817 /****************************************************************************/
7818 static int
7819 bce_sysctl_dump_ctx(SYSCTL_HANDLER_ARGS)
7820 {
7821         struct bce_softc *sc;
7822         int error;
7823         u16 result;
7824
7825         result = -1;
7826         error = sysctl_handle_int(oidp, &result, 0, req);
7827         if (error || (req->newptr == NULL))
7828                 return (error);
7829
7830         /* Make sure the register is accessible. */
7831         if (result <= TX_CID) {
7832                 sc = (struct bce_softc *)arg1;
7833                 bce_dump_ctx(sc, result);
7834         }
7835
7836         return (error);
7837 }
7838
7839
7840  /****************************************************************************/
7841 /* Provides a sysctl interface to forcing the driver to dump state and      */
7842 /* enter the debugger.  DO NOT ENABLE ON PRODUCTION SYSTEMS!                */
7843 /*                                                                          */
7844 /* Returns:                                                                 */
7845 /*   0 for success, positive value for failure.                             */
7846 /****************************************************************************/
7847 static int
7848 bce_sysctl_breakpoint(SYSCTL_HANDLER_ARGS)
7849 {
7850         int error;
7851         int result;
7852         struct bce_softc *sc;
7853
7854         result = -1;
7855         error = sysctl_handle_int(oidp, &result, 0, req);
7856
7857         if (error || !req->newptr)
7858                 return (error);
7859
7860         if (result == 1) {
7861                 sc = (struct bce_softc *)arg1;
7862                 bce_breakpoint(sc);
7863         }
7864
7865         return error;
7866 }
7867 #endif
7868
7869
7870 /****************************************************************************/
7871 /* Adds any sysctl parameters for tuning or debugging purposes.             */
7872 /*                                                                          */
7873 /* Returns:                                                                 */
7874 /*   0 for success, positive value for failure.                             */
7875 /****************************************************************************/
7876 static void
7877 bce_add_sysctls(struct bce_softc *sc)
7878 {
7879         struct sysctl_ctx_list *ctx;
7880         struct sysctl_oid_list *children;
7881
7882         DBENTER(BCE_VERBOSE_MISC);
7883
7884         ctx = device_get_sysctl_ctx(sc->bce_dev);
7885         children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->bce_dev));
7886
7887 #ifdef BCE_DEBUG
7888         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7889                 "l2fhdr_error_sim_control",
7890                 CTLFLAG_RW, &l2fhdr_error_sim_control,
7891                 0, "Debug control to force l2fhdr errors");
7892
7893         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7894                 "l2fhdr_error_sim_count",
7895                 CTLFLAG_RD, &sc->l2fhdr_error_sim_count,
7896                 0, "Number of simulated l2_fhdr errors");
7897 #endif
7898
7899         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7900                 "l2fhdr_error_count",
7901                 CTLFLAG_RD, &sc->l2fhdr_error_count,
7902                 0, "Number of l2_fhdr errors");
7903
7904 #ifdef BCE_DEBUG
7905         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7906                 "mbuf_alloc_failed_sim_control",
7907                 CTLFLAG_RW, &mbuf_alloc_failed_sim_control,
7908                 0, "Debug control to force mbuf allocation failures");
7909
7910         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7911                 "mbuf_alloc_failed_sim_count",
7912                 CTLFLAG_RD, &sc->mbuf_alloc_failed_sim_count,
7913                 0, "Number of simulated mbuf cluster allocation failures");
7914 #endif
7915
7916         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7917                 "mbuf_alloc_failed_count",
7918                 CTLFLAG_RD, &sc->mbuf_alloc_failed_count,
7919                 0, "Number of mbuf allocation failures");
7920
7921         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7922                 "fragmented_mbuf_count",
7923                 CTLFLAG_RD, &sc->fragmented_mbuf_count,
7924                 0, "Number of fragmented mbufs");
7925
7926 #ifdef BCE_DEBUG
7927         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7928                 "dma_map_addr_failed_sim_control",
7929                 CTLFLAG_RW, &dma_map_addr_failed_sim_control,
7930                 0, "Debug control to force DMA mapping failures");
7931
7932         /* ToDo: Figure out how to update this value in bce_dma_map_addr(). */
7933         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7934                 "dma_map_addr_failed_sim_count",
7935                 CTLFLAG_RD, &sc->dma_map_addr_failed_sim_count,
7936                 0, "Number of simulated DMA mapping failures");
7937
7938 #endif
7939
7940         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7941                 "dma_map_addr_rx_failed_count",
7942                 CTLFLAG_RD, &sc->dma_map_addr_rx_failed_count,
7943                 0, "Number of RX DMA mapping failures");
7944
7945         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7946                 "dma_map_addr_tx_failed_count",
7947                 CTLFLAG_RD, &sc->dma_map_addr_tx_failed_count,
7948                 0, "Number of TX DMA mapping failures");
7949
7950 #ifdef BCE_DEBUG
7951         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7952                 "unexpected_attention_sim_control",
7953                 CTLFLAG_RW, &unexpected_attention_sim_control,
7954                 0, "Debug control to simulate unexpected attentions");
7955
7956         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7957                 "unexpected_attention_sim_count",
7958                 CTLFLAG_RW, &sc->unexpected_attention_sim_count,
7959                 0, "Number of simulated unexpected attentions");
7960 #endif
7961
7962         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7963                 "unexpected_attention_count",
7964                 CTLFLAG_RW, &sc->unexpected_attention_count,
7965                 0, "Number of unexpected attentions");
7966
7967 #ifdef BCE_DEBUG
7968         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7969                 "debug_bootcode_running_failure",
7970                 CTLFLAG_RW, &bootcode_running_failure_sim_control,
7971                 0, "Debug control to force bootcode running failures");
7972
7973         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7974                 "rx_low_watermark",
7975                 CTLFLAG_RD, &sc->rx_low_watermark,
7976                 0, "Lowest level of free rx_bd's");
7977
7978         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7979                 "rx_empty_count",
7980                 CTLFLAG_RD, &sc->rx_empty_count,
7981                 0, "Number of times the RX chain was empty");
7982
7983         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7984                 "tx_hi_watermark",
7985                 CTLFLAG_RD, &sc->tx_hi_watermark,
7986                 0, "Highest level of used tx_bd's");
7987
7988         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7989                 "tx_full_count",
7990                 CTLFLAG_RD, &sc->tx_full_count,
7991                 0, "Number of times the TX chain was full");
7992
7993         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7994                 "requested_tso_frames",
7995                 CTLFLAG_RD, &sc->requested_tso_frames,
7996                 0, "Number of TSO frames received");
7997
7998         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
7999                 "rx_interrupts",
8000                 CTLFLAG_RD, &sc->rx_interrupts,
8001                 0, "Number of RX interrupts");
8002
8003         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8004                 "tx_interrupts",
8005                 CTLFLAG_RD, &sc->tx_interrupts,
8006                 0, "Number of TX interrupts");
8007
8008         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
8009                 "rx_intr_time",
8010                 CTLFLAG_RD, &sc->rx_intr_time,
8011                 "RX interrupt time");
8012
8013         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
8014                 "tx_intr_time",
8015                 CTLFLAG_RD, &sc->tx_intr_time,
8016                 "TX interrupt time");
8017 #endif
8018
8019         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
8020                 "stat_IfHcInOctets",
8021                 CTLFLAG_RD, &sc->stat_IfHCInOctets,
8022                 "Bytes received");
8023
8024         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
8025                 "stat_IfHCInBadOctets",
8026                 CTLFLAG_RD, &sc->stat_IfHCInBadOctets,
8027                 "Bad bytes received");
8028
8029         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
8030                 "stat_IfHCOutOctets",
8031                 CTLFLAG_RD, &sc->stat_IfHCOutOctets,
8032                 "Bytes sent");
8033
8034         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
8035                 "stat_IfHCOutBadOctets",
8036                 CTLFLAG_RD, &sc->stat_IfHCOutBadOctets,
8037                 "Bad bytes sent");
8038
8039         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
8040                 "stat_IfHCInUcastPkts",
8041                 CTLFLAG_RD, &sc->stat_IfHCInUcastPkts,
8042                 "Unicast packets received");
8043
8044         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
8045                 "stat_IfHCInMulticastPkts",
8046                 CTLFLAG_RD, &sc->stat_IfHCInMulticastPkts,
8047                 "Multicast packets received");
8048
8049         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
8050                 "stat_IfHCInBroadcastPkts",
8051                 CTLFLAG_RD, &sc->stat_IfHCInBroadcastPkts,
8052                 "Broadcast packets received");
8053
8054         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
8055                 "stat_IfHCOutUcastPkts",
8056                 CTLFLAG_RD, &sc->stat_IfHCOutUcastPkts,
8057                 "Unicast packets sent");
8058
8059         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
8060                 "stat_IfHCOutMulticastPkts",
8061                 CTLFLAG_RD, &sc->stat_IfHCOutMulticastPkts,
8062                 "Multicast packets sent");
8063
8064         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
8065                 "stat_IfHCOutBroadcastPkts",
8066                 CTLFLAG_RD, &sc->stat_IfHCOutBroadcastPkts,
8067                 "Broadcast packets sent");
8068
8069         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8070                 "stat_emac_tx_stat_dot3statsinternalmactransmiterrors",
8071                 CTLFLAG_RD, &sc->stat_emac_tx_stat_dot3statsinternalmactransmiterrors,
8072                 0, "Internal MAC transmit errors");
8073
8074         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8075                 "stat_Dot3StatsCarrierSenseErrors",
8076                 CTLFLAG_RD, &sc->stat_Dot3StatsCarrierSenseErrors,
8077                 0, "Carrier sense errors");
8078
8079         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8080                 "stat_Dot3StatsFCSErrors",
8081                 CTLFLAG_RD, &sc->stat_Dot3StatsFCSErrors,
8082                 0, "Frame check sequence errors");
8083
8084         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8085                 "stat_Dot3StatsAlignmentErrors",
8086                 CTLFLAG_RD, &sc->stat_Dot3StatsAlignmentErrors,
8087                 0, "Alignment errors");
8088
8089         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8090                 "stat_Dot3StatsSingleCollisionFrames",
8091                 CTLFLAG_RD, &sc->stat_Dot3StatsSingleCollisionFrames,
8092                 0, "Single Collision Frames");
8093
8094         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8095                 "stat_Dot3StatsMultipleCollisionFrames",
8096                 CTLFLAG_RD, &sc->stat_Dot3StatsMultipleCollisionFrames,
8097                 0, "Multiple Collision Frames");
8098
8099         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8100                 "stat_Dot3StatsDeferredTransmissions",
8101                 CTLFLAG_RD, &sc->stat_Dot3StatsDeferredTransmissions,
8102                 0, "Deferred Transmissions");
8103
8104         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8105                 "stat_Dot3StatsExcessiveCollisions",
8106                 CTLFLAG_RD, &sc->stat_Dot3StatsExcessiveCollisions,
8107                 0, "Excessive Collisions");
8108
8109         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8110                 "stat_Dot3StatsLateCollisions",
8111                 CTLFLAG_RD, &sc->stat_Dot3StatsLateCollisions,
8112                 0, "Late Collisions");
8113
8114         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8115                 "stat_EtherStatsCollisions",
8116                 CTLFLAG_RD, &sc->stat_EtherStatsCollisions,
8117                 0, "Collisions");
8118
8119         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8120                 "stat_EtherStatsFragments",
8121                 CTLFLAG_RD, &sc->stat_EtherStatsFragments,
8122                 0, "Fragments");
8123
8124         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8125                 "stat_EtherStatsJabbers",
8126                 CTLFLAG_RD, &sc->stat_EtherStatsJabbers,
8127                 0, "Jabbers");
8128
8129         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8130                 "stat_EtherStatsUndersizePkts",
8131                 CTLFLAG_RD, &sc->stat_EtherStatsUndersizePkts,
8132                 0, "Undersize packets");
8133
8134         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8135                 "stat_EtherStatsOversizePkts",
8136                 CTLFLAG_RD, &sc->stat_EtherStatsOversizePkts,
8137                 0, "stat_EtherStatsOversizePkts");
8138
8139         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8140                 "stat_EtherStatsPktsRx64Octets",
8141                 CTLFLAG_RD, &sc->stat_EtherStatsPktsRx64Octets,
8142                 0, "Bytes received in 64 byte packets");
8143
8144         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8145                 "stat_EtherStatsPktsRx65Octetsto127Octets",
8146                 CTLFLAG_RD, &sc->stat_EtherStatsPktsRx65Octetsto127Octets,
8147                 0, "Bytes received in 65 to 127 byte packets");
8148
8149         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8150                 "stat_EtherStatsPktsRx128Octetsto255Octets",
8151                 CTLFLAG_RD, &sc->stat_EtherStatsPktsRx128Octetsto255Octets,
8152                 0, "Bytes received in 128 to 255 byte packets");
8153
8154         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8155                 "stat_EtherStatsPktsRx256Octetsto511Octets",
8156                 CTLFLAG_RD, &sc->stat_EtherStatsPktsRx256Octetsto511Octets,
8157                 0, "Bytes received in 256 to 511 byte packets");
8158
8159         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8160                 "stat_EtherStatsPktsRx512Octetsto1023Octets",
8161                 CTLFLAG_RD, &sc->stat_EtherStatsPktsRx512Octetsto1023Octets,
8162                 0, "Bytes received in 512 to 1023 byte packets");
8163
8164         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8165                 "stat_EtherStatsPktsRx1024Octetsto1522Octets",
8166                 CTLFLAG_RD, &sc->stat_EtherStatsPktsRx1024Octetsto1522Octets,
8167                 0, "Bytes received in 1024 t0 1522 byte packets");
8168
8169         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8170                 "stat_EtherStatsPktsRx1523Octetsto9022Octets",
8171                 CTLFLAG_RD, &sc->stat_EtherStatsPktsRx1523Octetsto9022Octets,
8172                 0, "Bytes received in 1523 to 9022 byte packets");
8173
8174         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8175                 "stat_EtherStatsPktsTx64Octets",
8176                 CTLFLAG_RD, &sc->stat_EtherStatsPktsTx64Octets,
8177                 0, "Bytes sent in 64 byte packets");
8178
8179         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8180                 "stat_EtherStatsPktsTx65Octetsto127Octets",
8181                 CTLFLAG_RD, &sc->stat_EtherStatsPktsTx65Octetsto127Octets,
8182                 0, "Bytes sent in 65 to 127 byte packets");
8183
8184         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8185                 "stat_EtherStatsPktsTx128Octetsto255Octets",
8186                 CTLFLAG_RD, &sc->stat_EtherStatsPktsTx128Octetsto255Octets,
8187                 0, "Bytes sent in 128 to 255 byte packets");
8188
8189         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8190                 "stat_EtherStatsPktsTx256Octetsto511Octets",
8191                 CTLFLAG_RD, &sc->stat_EtherStatsPktsTx256Octetsto511Octets,
8192                 0, "Bytes sent in 256 to 511 byte packets");
8193
8194         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8195                 "stat_EtherStatsPktsTx512Octetsto1023Octets",
8196                 CTLFLAG_RD, &sc->stat_EtherStatsPktsTx512Octetsto1023Octets,
8197                 0, "Bytes sent in 512 to 1023 byte packets");
8198
8199         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8200                 "stat_EtherStatsPktsTx1024Octetsto1522Octets",
8201                 CTLFLAG_RD, &sc->stat_EtherStatsPktsTx1024Octetsto1522Octets,
8202                 0, "Bytes sent in 1024 to 1522 byte packets");
8203
8204         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8205                 "stat_EtherStatsPktsTx1523Octetsto9022Octets",
8206                 CTLFLAG_RD, &sc->stat_EtherStatsPktsTx1523Octetsto9022Octets,
8207                 0, "Bytes sent in 1523 to 9022 byte packets");
8208
8209         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8210                 "stat_XonPauseFramesReceived",
8211                 CTLFLAG_RD, &sc->stat_XonPauseFramesReceived,
8212                 0, "XON pause frames receved");
8213
8214         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8215                 "stat_XoffPauseFramesReceived",
8216                 CTLFLAG_RD, &sc->stat_XoffPauseFramesReceived,
8217                 0, "XOFF pause frames received");
8218
8219         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8220                 "stat_OutXonSent",
8221                 CTLFLAG_RD, &sc->stat_OutXonSent,
8222                 0, "XON pause frames sent");
8223
8224         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8225                 "stat_OutXoffSent",
8226                 CTLFLAG_RD, &sc->stat_OutXoffSent,
8227                 0, "XOFF pause frames sent");
8228
8229         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8230                 "stat_FlowControlDone",
8231                 CTLFLAG_RD, &sc->stat_FlowControlDone,
8232                 0, "Flow control done");
8233
8234         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8235                 "stat_MacControlFramesReceived",
8236                 CTLFLAG_RD, &sc->stat_MacControlFramesReceived,
8237                 0, "MAC control frames received");
8238
8239         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8240                 "stat_XoffStateEntered",
8241                 CTLFLAG_RD, &sc->stat_XoffStateEntered,
8242                 0, "XOFF state entered");
8243
8244         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8245                 "stat_IfInFramesL2FilterDiscards",
8246                 CTLFLAG_RD, &sc->stat_IfInFramesL2FilterDiscards,
8247                 0, "Received L2 packets discarded");
8248
8249         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8250                 "stat_IfInRuleCheckerDiscards",
8251                 CTLFLAG_RD, &sc->stat_IfInRuleCheckerDiscards,
8252                 0, "Received packets discarded by rule");
8253
8254         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8255                 "stat_IfInFTQDiscards",
8256                 CTLFLAG_RD, &sc->stat_IfInFTQDiscards,
8257                 0, "Received packet FTQ discards");
8258
8259         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8260                 "stat_IfInMBUFDiscards",
8261                 CTLFLAG_RD, &sc->stat_IfInMBUFDiscards,
8262                 0, "Received packets discarded due to lack of controller buffer memory");
8263
8264         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8265                 "stat_IfInRuleCheckerP4Hit",
8266                 CTLFLAG_RD, &sc->stat_IfInRuleCheckerP4Hit,
8267                 0, "Received packets rule checker hits");
8268
8269         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8270                 "stat_CatchupInRuleCheckerDiscards",
8271                 CTLFLAG_RD, &sc->stat_CatchupInRuleCheckerDiscards,
8272                 0, "Received packets discarded in Catchup path");
8273
8274         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8275                 "stat_CatchupInFTQDiscards",
8276                 CTLFLAG_RD, &sc->stat_CatchupInFTQDiscards,
8277                 0, "Received packets discarded in FTQ in Catchup path");
8278
8279         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8280                 "stat_CatchupInMBUFDiscards",
8281                 CTLFLAG_RD, &sc->stat_CatchupInMBUFDiscards,
8282                 0, "Received packets discarded in controller buffer memory in Catchup path");
8283
8284         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8285                 "stat_CatchupInRuleCheckerP4Hit",
8286                 CTLFLAG_RD, &sc->stat_CatchupInRuleCheckerP4Hit,
8287                 0, "Received packets rule checker hits in Catchup path");
8288
8289         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8290                 "com_no_buffers",
8291                 CTLFLAG_RD, &sc->com_no_buffers,
8292                 0, "Valid packets received but no RX buffers available");
8293
8294 #ifdef BCE_DEBUG
8295         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
8296                 "driver_state", CTLTYPE_INT | CTLFLAG_RW,
8297                 (void *)sc, 0,
8298                 bce_sysctl_driver_state, "I", "Drive state information");
8299
8300         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
8301                 "hw_state", CTLTYPE_INT | CTLFLAG_RW,
8302                 (void *)sc, 0,
8303                 bce_sysctl_hw_state, "I", "Hardware state information");
8304
8305         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
8306                 "bc_state", CTLTYPE_INT | CTLFLAG_RW,
8307                 (void *)sc, 0,
8308                 bce_sysctl_bc_state, "I", "Bootcode state information");
8309
8310         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
8311                 "dump_rx_chain", CTLTYPE_INT | CTLFLAG_RW,
8312                 (void *)sc, 0,
8313                 bce_sysctl_dump_rx_chain, "I", "Dump rx_bd chain");
8314
8315         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
8316                 "dump_tx_chain", CTLTYPE_INT | CTLFLAG_RW,
8317                 (void *)sc, 0,
8318                 bce_sysctl_dump_tx_chain, "I", "Dump tx_bd chain");
8319
8320 #ifdef ZERO_COPY_SOCKETS
8321         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
8322                 "dump_pg_chain", CTLTYPE_INT | CTLFLAG_RW,
8323                 (void *)sc, 0,
8324                 bce_sysctl_dump_pg_chain, "I", "Dump page chain");
8325 #endif
8326         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
8327                 "dump_ctx", CTLTYPE_INT | CTLFLAG_RW,
8328                 (void *)sc, 0,
8329                 bce_sysctl_dump_ctx, "I", "Dump context memory");
8330
8331         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
8332                 "breakpoint", CTLTYPE_INT | CTLFLAG_RW,
8333                 (void *)sc, 0,
8334                 bce_sysctl_breakpoint, "I", "Driver breakpoint");
8335
8336         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
8337                 "reg_read", CTLTYPE_INT | CTLFLAG_RW,
8338                 (void *)sc, 0,
8339                 bce_sysctl_reg_read, "I", "Register read");
8340
8341         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
8342                 "nvram_read", CTLTYPE_INT | CTLFLAG_RW,
8343                 (void *)sc, 0,
8344                 bce_sysctl_nvram_read, "I", "NVRAM read");
8345
8346         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
8347                 "phy_read", CTLTYPE_INT | CTLFLAG_RW,
8348                 (void *)sc, 0,
8349                 bce_sysctl_phy_read, "I", "PHY register read");
8350
8351 #endif
8352
8353         DBEXIT(BCE_VERBOSE_MISC);
8354 }
8355
8356
8357 /****************************************************************************/
8358 /* BCE Debug Routines                                                       */
8359 /****************************************************************************/
8360 #ifdef BCE_DEBUG
8361
8362 /****************************************************************************/
8363 /* Freezes the controller to allow for a cohesive state dump.               */
8364 /*                                                                          */
8365 /* Returns:                                                                 */
8366 /*   Nothing.                                                               */
8367 /****************************************************************************/
8368 static void
8369 bce_freeze_controller(struct bce_softc *sc)
8370 {
8371         u32 val;
8372         val = REG_RD(sc, BCE_MISC_COMMAND);
8373         val |= BCE_MISC_COMMAND_DISABLE_ALL;
8374         REG_WR(sc, BCE_MISC_COMMAND, val);
8375 }
8376
8377
8378 /****************************************************************************/
8379 /* Unfreezes the controller after a freeze operation.  This may not always  */
8380 /* work and the controller will require a reset!                            */
8381 /*                                                                          */
8382 /* Returns:                                                                 */
8383 /*   Nothing.                                                               */
8384 /****************************************************************************/
8385 static void
8386 bce_unfreeze_controller(struct bce_softc *sc)
8387 {
8388         u32 val;
8389         val = REG_RD(sc, BCE_MISC_COMMAND);
8390         val |= BCE_MISC_COMMAND_ENABLE_ALL;
8391         REG_WR(sc, BCE_MISC_COMMAND, val);
8392 }
8393
8394
8395 /****************************************************************************/
8396 /* Prints out Ethernet frame information from an mbuf.                      */
8397 /*                                                                          */
8398 /* Partially decode an Ethernet frame to look at some important headers.    */
8399 /*                                                                          */
8400 /* Returns:                                                                 */
8401 /*   Nothing.                                                               */
8402 /****************************************************************************/
8403 static void
8404 bce_dump_enet(struct bce_softc *sc, struct mbuf *m)
8405 {
8406         struct ether_vlan_header *eh;
8407         u16 etype;
8408         int ehlen;
8409         struct ip *ip;
8410         struct tcphdr *th;
8411         struct udphdr *uh;
8412         struct arphdr *ah;
8413
8414                 BCE_PRINTF(
8415                         "-----------------------------"
8416                         " Frame Decode "
8417                         "-----------------------------\n");
8418
8419         eh = mtod(m, struct ether_vlan_header *);
8420
8421         /* Handle VLAN encapsulation if present. */
8422         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
8423                 etype = ntohs(eh->evl_proto);
8424                 ehlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
8425         } else {
8426                 etype = ntohs(eh->evl_encap_proto);
8427                 ehlen = ETHER_HDR_LEN;
8428         }
8429
8430         /* ToDo: Add VLAN output. */
8431         BCE_PRINTF("enet: dest = %6D, src = %6D, type = 0x%04X, hlen = %d\n",
8432                 eh->evl_dhost, ":", eh->evl_shost, ":", etype, ehlen);
8433
8434         switch (etype) {
8435                 case ETHERTYPE_IP:
8436                         ip = (struct ip *)(m->m_data + ehlen);
8437                         BCE_PRINTF("--ip: dest = 0x%08X , src = 0x%08X, len = %d bytes, "
8438                                 "protocol = 0x%02X, xsum = 0x%04X\n",
8439                                 ntohl(ip->ip_dst.s_addr), ntohl(ip->ip_src.s_addr),
8440                                 ntohs(ip->ip_len), ip->ip_p, ntohs(ip->ip_sum));
8441
8442                         switch (ip->ip_p) {
8443                                 case IPPROTO_TCP:
8444                                         th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2));
8445                                         BCE_PRINTF("-tcp: dest = %d, src = %d, hlen = %d bytes, "
8446                                                 "flags = 0x%b, csum = 0x%04X\n",
8447                                                 ntohs(th->th_dport), ntohs(th->th_sport), (th->th_off << 2),
8448                                                 th->th_flags, "\20\10CWR\07ECE\06URG\05ACK\04PSH\03RST\02SYN\01FIN",
8449                                                 ntohs(th->th_sum));
8450                                         break;
8451                                 case IPPROTO_UDP:
8452                             uh = (struct udphdr *)((caddr_t)ip + (ip->ip_hl << 2));
8453                                         BCE_PRINTF("-udp: dest = %d, src = %d, len = %d bytes, "
8454                                                 "csum = 0x%04X\n", ntohs(uh->uh_dport), ntohs(uh->uh_sport),
8455                                                 ntohs(uh->uh_ulen), ntohs(uh->uh_sum));
8456                                         break;
8457                                 case IPPROTO_ICMP:
8458                                         BCE_PRINTF("icmp:\n");
8459                                         break;
8460                                 default:
8461                                         BCE_PRINTF("----: Other IP protocol.\n");
8462                         }
8463                         break;
8464                 case ETHERTYPE_IPV6:
8465                         BCE_PRINTF("ipv6: No decode supported.\n");
8466                         break;
8467                 case ETHERTYPE_ARP:
8468                         BCE_PRINTF("-arp: ");
8469                         ah = (struct arphdr *) (m->m_data + ehlen);
8470                         switch (ntohs(ah->ar_op)) {
8471                                 case ARPOP_REVREQUEST:
8472                                         printf("reverse ARP request\n");
8473                                         break;
8474                                 case ARPOP_REVREPLY:
8475                                         printf("reverse ARP reply\n");
8476                                         break;
8477                                 case ARPOP_REQUEST:
8478                                         printf("ARP request\n");
8479                                         break;
8480                                 case ARPOP_REPLY:
8481                                         printf("ARP reply\n");
8482                                         break;
8483                                 default:
8484                                         printf("other ARP operation\n");
8485                         }
8486                         break;
8487                 default:
8488                         BCE_PRINTF("----: Other protocol.\n");
8489         }
8490
8491         BCE_PRINTF(
8492                 "-----------------------------"
8493                 "--------------"
8494                 "-----------------------------\n");
8495 }
8496
8497
8498 /****************************************************************************/
8499 /* Prints out information about an mbuf.                                    */
8500 /*                                                                          */
8501 /* Returns:                                                                 */
8502 /*   Nothing.                                                               */
8503 /****************************************************************************/
8504 static __attribute__ ((noinline)) void
8505 bce_dump_mbuf(struct bce_softc *sc, struct mbuf *m)
8506 {
8507         struct mbuf *mp = m;
8508
8509         if (m == NULL) {
8510                 BCE_PRINTF("mbuf: null pointer\n");
8511                 return;
8512         }
8513
8514         while (mp) {
8515                 BCE_PRINTF("mbuf: %p, m_len = %d, m_flags = 0x%b, m_data = %p\n",
8516                         mp, mp->m_len, mp->m_flags,
8517                         "\20\1M_EXT\2M_PKTHDR\3M_EOR\4M_RDONLY",
8518                         mp->m_data);
8519
8520                 if (mp->m_flags & M_PKTHDR) {
8521                         BCE_PRINTF("- m_pkthdr: len = %d, flags = 0x%b, csum_flags = %b\n",
8522                                 mp->m_pkthdr.len, mp->m_flags,
8523                                 "\20\12M_BCAST\13M_MCAST\14M_FRAG\15M_FIRSTFRAG"
8524                                 "\16M_LASTFRAG\21M_VLANTAG\22M_PROMISC\23M_NOFREE",
8525                                 mp->m_pkthdr.csum_flags,
8526                                 "\20\1CSUM_IP\2CSUM_TCP\3CSUM_UDP\4CSUM_IP_FRAGS"
8527                                 "\5CSUM_FRAGMENT\6CSUM_TSO\11CSUM_IP_CHECKED"
8528                                 "\12CSUM_IP_VALID\13CSUM_DATA_VALID\14CSUM_PSEUDO_HDR");
8529                 }
8530
8531                 if (mp->m_flags & M_EXT) {
8532                         BCE_PRINTF("- m_ext: %p, ext_size = %d, type = ",
8533                                 mp->m_ext.ext_buf, mp->m_ext.ext_size);
8534                         switch (mp->m_ext.ext_type) {
8535                                 case EXT_CLUSTER:    printf("EXT_CLUSTER\n"); break;
8536                                 case EXT_SFBUF:      printf("EXT_SFBUF\n"); break;
8537                                 case EXT_JUMBO9:     printf("EXT_JUMBO9\n"); break;
8538                                 case EXT_JUMBO16:    printf("EXT_JUMBO16\n"); break;
8539                                 case EXT_PACKET:     printf("EXT_PACKET\n"); break;
8540                                 case EXT_MBUF:       printf("EXT_MBUF\n"); break;
8541                                 case EXT_NET_DRV:    printf("EXT_NET_DRV\n"); break;
8542                                 case EXT_MOD_TYPE:   printf("EXT_MDD_TYPE\n"); break;
8543                                 case EXT_DISPOSABLE: printf("EXT_DISPOSABLE\n"); break;
8544                                 case EXT_EXTREF:     printf("EXT_EXTREF\n"); break;
8545                                 default:             printf("UNKNOWN\n");
8546                         }
8547                 }
8548
8549                 mp = mp->m_next;
8550         }
8551 }
8552
8553
8554 /****************************************************************************/
8555 /* Prints out the mbufs in the TX mbuf chain.                               */
8556 /*                                                                          */
8557 /* Returns:                                                                 */
8558 /*   Nothing.                                                               */
8559 /****************************************************************************/
8560 static __attribute__ ((noinline)) void
8561 bce_dump_tx_mbuf_chain(struct bce_softc *sc, u16 chain_prod, int count)
8562 {
8563         struct mbuf *m;
8564
8565         BCE_PRINTF(
8566                 "----------------------------"
8567                 "  tx mbuf data  "
8568                 "----------------------------\n");
8569
8570         for (int i = 0; i < count; i++) {
8571                 m = sc->tx_mbuf_ptr[chain_prod];
8572                 BCE_PRINTF("txmbuf[0x%04X]\n", chain_prod);
8573                 bce_dump_mbuf(sc, m);
8574                 chain_prod = TX_CHAIN_IDX(NEXT_TX_BD(chain_prod));
8575         }
8576
8577         BCE_PRINTF(
8578                 "----------------------------"
8579                 "----------------"
8580                 "----------------------------\n");
8581 }
8582
8583
8584 /****************************************************************************/
8585 /* Prints out the mbufs in the RX mbuf chain.                               */
8586 /*                                                                          */
8587 /* Returns:                                                                 */
8588 /*   Nothing.                                                               */
8589 /****************************************************************************/
8590 static __attribute__ ((noinline)) void
8591 bce_dump_rx_mbuf_chain(struct bce_softc *sc, u16 chain_prod, int count)
8592 {
8593         struct mbuf *m;
8594
8595         BCE_PRINTF(
8596                 "----------------------------"
8597                 "  rx mbuf data  "
8598                 "----------------------------\n");
8599
8600         for (int i = 0; i < count; i++) {
8601                 m = sc->rx_mbuf_ptr[chain_prod];
8602                 BCE_PRINTF("rxmbuf[0x%04X]\n", chain_prod);
8603                 bce_dump_mbuf(sc, m);
8604                 chain_prod = RX_CHAIN_IDX(NEXT_RX_BD(chain_prod));
8605         }
8606
8607
8608         BCE_PRINTF(
8609                 "----------------------------"
8610                 "----------------"
8611                 "----------------------------\n");
8612 }
8613
8614
8615 #ifdef ZERO_COPY_SOCKETS
8616 /****************************************************************************/
8617 /* Prints out the mbufs in the mbuf page chain.                             */
8618 /*                                                                          */
8619 /* Returns:                                                                 */
8620 /*   Nothing.                                                               */
8621 /****************************************************************************/
8622 static __attribute__ ((noinline)) void
8623 bce_dump_pg_mbuf_chain(struct bce_softc *sc, u16 chain_prod, int count)
8624 {
8625         struct mbuf *m;
8626
8627         BCE_PRINTF(
8628                 "----------------------------"
8629                 "  pg mbuf data  "
8630                 "----------------------------\n");
8631
8632         for (int i = 0; i < count; i++) {
8633                 m = sc->pg_mbuf_ptr[chain_prod];
8634                 BCE_PRINTF("pgmbuf[0x%04X]\n", chain_prod);
8635                 bce_dump_mbuf(sc, m);
8636                 chain_prod = PG_CHAIN_IDX(NEXT_PG_BD(chain_prod));
8637         }
8638
8639
8640         BCE_PRINTF(
8641                 "----------------------------"
8642                 "----------------"
8643                 "----------------------------\n");
8644 }
8645 #endif
8646
8647
8648 /****************************************************************************/
8649 /* Prints out a tx_bd structure.                                            */
8650 /*                                                                          */
8651 /* Returns:                                                                 */
8652 /*   Nothing.                                                               */
8653 /****************************************************************************/
8654 static __attribute__ ((noinline)) void
8655 bce_dump_txbd(struct bce_softc *sc, int idx, struct tx_bd *txbd)
8656 {
8657         if (idx > MAX_TX_BD)
8658                 /* Index out of range. */
8659                 BCE_PRINTF("tx_bd[0x%04X]: Invalid tx_bd index!\n", idx);
8660         else if ((idx & USABLE_TX_BD_PER_PAGE) == USABLE_TX_BD_PER_PAGE)
8661                 /* TX Chain page pointer. */
8662                 BCE_PRINTF("tx_bd[0x%04X]: haddr = 0x%08X:%08X, chain page pointer\n",
8663                         idx, txbd->tx_bd_haddr_hi, txbd->tx_bd_haddr_lo);
8664         else {
8665                         /* Normal tx_bd entry. */
8666                         BCE_PRINTF("tx_bd[0x%04X]: haddr = 0x%08X:%08X, nbytes = 0x%08X, "
8667                                 "vlan tag= 0x%04X, flags = 0x%04X (", idx,
8668                                 txbd->tx_bd_haddr_hi, txbd->tx_bd_haddr_lo,
8669                                 txbd->tx_bd_mss_nbytes, txbd->tx_bd_vlan_tag,
8670                                 txbd->tx_bd_flags);
8671
8672                         if (txbd->tx_bd_flags & TX_BD_FLAGS_CONN_FAULT)
8673                                 printf(" CONN_FAULT");
8674
8675                         if (txbd->tx_bd_flags & TX_BD_FLAGS_TCP_UDP_CKSUM)
8676                                 printf(" TCP_UDP_CKSUM");
8677
8678                         if (txbd->tx_bd_flags & TX_BD_FLAGS_IP_CKSUM)
8679                                 printf(" IP_CKSUM");
8680
8681                         if (txbd->tx_bd_flags & TX_BD_FLAGS_VLAN_TAG)
8682                                 printf("  VLAN");
8683
8684                         if (txbd->tx_bd_flags & TX_BD_FLAGS_COAL_NOW)
8685                                 printf(" COAL_NOW");
8686
8687                         if (txbd->tx_bd_flags & TX_BD_FLAGS_DONT_GEN_CRC)
8688                                 printf(" DONT_GEN_CRC");
8689
8690                         if (txbd->tx_bd_flags & TX_BD_FLAGS_START)
8691                                 printf(" START");
8692
8693                         if (txbd->tx_bd_flags & TX_BD_FLAGS_END)
8694                                 printf(" END");
8695
8696                         if (txbd->tx_bd_flags & TX_BD_FLAGS_SW_LSO)
8697                                 printf(" LSO");
8698
8699                         if (txbd->tx_bd_flags & TX_BD_FLAGS_SW_OPTION_WORD)
8700                                 printf(" OPTION_WORD");
8701
8702                         if (txbd->tx_bd_flags & TX_BD_FLAGS_SW_FLAGS)
8703                                 printf(" FLAGS");
8704
8705                         if (txbd->tx_bd_flags & TX_BD_FLAGS_SW_SNAP)
8706                                 printf(" SNAP");
8707
8708                         printf(" )\n");
8709                 }
8710
8711 }
8712
8713
8714 /****************************************************************************/
8715 /* Prints out a rx_bd structure.                                            */
8716 /*                                                                          */
8717 /* Returns:                                                                 */
8718 /*   Nothing.                                                               */
8719 /****************************************************************************/
8720 static __attribute__ ((noinline)) void
8721 bce_dump_rxbd(struct bce_softc *sc, int idx, struct rx_bd *rxbd)
8722 {
8723         if (idx > MAX_RX_BD)
8724                 /* Index out of range. */
8725                 BCE_PRINTF("rx_bd[0x%04X]: Invalid rx_bd index!\n", idx);
8726         else if ((idx & USABLE_RX_BD_PER_PAGE) == USABLE_RX_BD_PER_PAGE)
8727                 /* RX Chain page pointer. */
8728                 BCE_PRINTF("rx_bd[0x%04X]: haddr = 0x%08X:%08X, chain page pointer\n",
8729                         idx, rxbd->rx_bd_haddr_hi, rxbd->rx_bd_haddr_lo);
8730         else
8731                 /* Normal rx_bd entry. */
8732                 BCE_PRINTF("rx_bd[0x%04X]: haddr = 0x%08X:%08X, nbytes = 0x%08X, "
8733                         "flags = 0x%08X\n", idx,
8734                         rxbd->rx_bd_haddr_hi, rxbd->rx_bd_haddr_lo,
8735                         rxbd->rx_bd_len, rxbd->rx_bd_flags);
8736 }
8737
8738
8739 #ifdef ZERO_COPY_SOCKETS
8740 /****************************************************************************/
8741 /* Prints out a rx_bd structure in the page chain.                          */
8742 /*                                                                          */
8743 /* Returns:                                                                 */
8744 /*   Nothing.                                                               */
8745 /****************************************************************************/
8746 static __attribute__ ((noinline)) void
8747 bce_dump_pgbd(struct bce_softc *sc, int idx, struct rx_bd *pgbd)
8748 {
8749         if (idx > MAX_PG_BD)
8750                 /* Index out of range. */
8751                 BCE_PRINTF("pg_bd[0x%04X]: Invalid pg_bd index!\n", idx);
8752         else if ((idx & USABLE_PG_BD_PER_PAGE) == USABLE_PG_BD_PER_PAGE)
8753                 /* Page Chain page pointer. */
8754                 BCE_PRINTF("px_bd[0x%04X]: haddr = 0x%08X:%08X, chain page pointer\n",
8755                         idx, pgbd->rx_bd_haddr_hi, pgbd->rx_bd_haddr_lo);
8756         else
8757                 /* Normal rx_bd entry. */
8758                 BCE_PRINTF("pg_bd[0x%04X]: haddr = 0x%08X:%08X, nbytes = 0x%08X, "
8759                         "flags = 0x%08X\n", idx,
8760                         pgbd->rx_bd_haddr_hi, pgbd->rx_bd_haddr_lo,
8761                         pgbd->rx_bd_len, pgbd->rx_bd_flags);
8762 }
8763 #endif
8764
8765
8766 /****************************************************************************/
8767 /* Prints out a l2_fhdr structure.                                          */
8768 /*                                                                          */
8769 /* Returns:                                                                 */
8770 /*   Nothing.                                                               */
8771 /****************************************************************************/
8772 static __attribute__ ((noinline)) void
8773 bce_dump_l2fhdr(struct bce_softc *sc, int idx, struct l2_fhdr *l2fhdr)
8774 {
8775         BCE_PRINTF("l2_fhdr[0x%04X]: status = 0x%b, "
8776                 "pkt_len = %d, vlan = 0x%04x, ip_xsum/hdr_len = 0x%04X, "
8777                 "tcp_udp_xsum = 0x%04X\n", idx,
8778                 l2fhdr->l2_fhdr_status, BCE_L2FHDR_PRINTFB,
8779                 l2fhdr->l2_fhdr_pkt_len, l2fhdr->l2_fhdr_vlan_tag,
8780                 l2fhdr->l2_fhdr_ip_xsum, l2fhdr->l2_fhdr_tcp_udp_xsum);
8781 }
8782
8783
8784 /****************************************************************************/
8785 /* Prints out context memory info.  (Only useful for CID 0 to 16.)          */
8786 /*                                                                          */
8787 /* Returns:                                                                 */
8788 /*   Nothing.                                                               */
8789 /****************************************************************************/
8790 static __attribute__ ((noinline)) void
8791 bce_dump_ctx(struct bce_softc *sc, u16 cid)
8792 {
8793         if (cid <= TX_CID) {
8794                 BCE_PRINTF(
8795                         "----------------------------"
8796                         "    CTX Data    "
8797                         "----------------------------\n");
8798
8799                 BCE_PRINTF("     0x%04X - (CID) Context ID\n", cid);
8800
8801                 if (cid == RX_CID) {
8802                         BCE_PRINTF(" 0x%08X - (L2CTX_RX_HOST_BDIDX) host rx "
8803                                 "producer index\n",
8804                                 CTX_RD(sc, GET_CID_ADDR(cid), BCE_L2CTX_RX_HOST_BDIDX));
8805                         BCE_PRINTF(" 0x%08X - (L2CTX_RX_HOST_BSEQ) host byte sequence\n",
8806                                 CTX_RD(sc, GET_CID_ADDR(cid), BCE_L2CTX_RX_HOST_BSEQ));
8807                         BCE_PRINTF(" 0x%08X - (L2CTX_RX_NX_BSEQ) h/w byte sequence\n",
8808                                 CTX_RD(sc, GET_CID_ADDR(cid), BCE_L2CTX_RX_NX_BSEQ));
8809                         BCE_PRINTF(" 0x%08X - (L2CTX_RX_NX_BDHADDR_HI) h/w buffer "
8810                                 "descriptor address\n",
8811                                 CTX_RD(sc, GET_CID_ADDR(cid), BCE_L2CTX_RX_NX_BDHADDR_HI));
8812                         BCE_PRINTF(" 0x%08X - (L2CTX_RX_NX_BDHADDR_LO) h/w buffer "
8813                                 "descriptor address\n",
8814                                 CTX_RD(sc, GET_CID_ADDR(cid), BCE_L2CTX_RX_NX_BDHADDR_LO));
8815                         BCE_PRINTF(" 0x%08X - (L2CTX_RX_NX_BDIDX) h/w rx consumer index\n",
8816                                 CTX_RD(sc, GET_CID_ADDR(cid), BCE_L2CTX_RX_NX_BDIDX));
8817                         BCE_PRINTF(" 0x%08X - (L2CTX_RX_HOST_PG_BDIDX) host page "
8818                                 "producer index\n",
8819                                 CTX_RD(sc, GET_CID_ADDR(cid), BCE_L2CTX_RX_HOST_PG_BDIDX));
8820                         BCE_PRINTF(" 0x%08X - (L2CTX_RX_PG_BUF_SIZE) host rx_bd/page "
8821                                 "buffer size\n",
8822                                 CTX_RD(sc, GET_CID_ADDR(cid), BCE_L2CTX_RX_PG_BUF_SIZE));
8823                         BCE_PRINTF(" 0x%08X - (L2CTX_RX_NX_PG_BDHADDR_HI) h/w page "
8824                                 "chain address\n",
8825                                 CTX_RD(sc, GET_CID_ADDR(cid), BCE_L2CTX_RX_NX_PG_BDHADDR_HI));
8826                         BCE_PRINTF(" 0x%08X - (L2CTX_RX_NX_PG_BDHADDR_LO) h/w page "
8827                                 "chain address\n",
8828                                 CTX_RD(sc, GET_CID_ADDR(cid), BCE_L2CTX_RX_NX_PG_BDHADDR_LO));
8829                         BCE_PRINTF(" 0x%08X - (L2CTX_RX_NX_PG_BDIDX) h/w page "
8830                                 "consumer index\n",
8831                                 CTX_RD(sc, GET_CID_ADDR(cid), BCE_L2CTX_RX_NX_PG_BDIDX));
8832                 } else if (cid == TX_CID) {
8833                         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
8834                                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
8835                                 BCE_PRINTF(" 0x%08X - (L2CTX_TX_TYPE_XI) ctx type\n",
8836                                         CTX_RD(sc, GET_CID_ADDR(cid), BCE_L2CTX_TX_TYPE_XI));
8837                                 BCE_PRINTF(" 0x%08X - (L2CTX_CMD_TX_TYPE_XI) ctx cmd\n",
8838                                         CTX_RD(sc, GET_CID_ADDR(cid), BCE_L2CTX_TX_CMD_TYPE_XI));
8839                                 BCE_PRINTF(" 0x%08X - (L2CTX_TX_TBDR_BDHADDR_HI_XI) h/w buffer "
8840                                         "descriptor address\n", CTX_RD(sc,
8841                                         GET_CID_ADDR(cid), BCE_L2CTX_TX_TBDR_BHADDR_HI_XI));
8842                                 BCE_PRINTF(" 0x%08X - (L2CTX_TX_TBDR_BHADDR_LO_XI) h/w buffer "
8843                                         "descriptor address\n", CTX_RD(sc,
8844                                         GET_CID_ADDR(cid), BCE_L2CTX_TX_TBDR_BHADDR_LO_XI));
8845                                 BCE_PRINTF(" 0x%08X - (L2CTX_TX_HOST_BIDX_XI) host producer "
8846                                         "index\n", CTX_RD(sc, GET_CID_ADDR(cid),
8847                                         BCE_L2CTX_TX_HOST_BIDX_XI));
8848                                 BCE_PRINTF(" 0x%08X - (L2CTX_TX_HOST_BSEQ_XI) host byte "
8849                                         "sequence\n", CTX_RD(sc, GET_CID_ADDR(cid),
8850                                         BCE_L2CTX_TX_HOST_BSEQ_XI));
8851                         } else {
8852                                 BCE_PRINTF(" 0x%08X - (L2CTX_TX_TYPE) ctx type\n",
8853                                         CTX_RD(sc, GET_CID_ADDR(cid), BCE_L2CTX_TX_TYPE));
8854                                 BCE_PRINTF(" 0x%08X - (L2CTX_TX_CMD_TYPE) ctx cmd\n",
8855                                         CTX_RD(sc, GET_CID_ADDR(cid), BCE_L2CTX_TX_CMD_TYPE));
8856                                 BCE_PRINTF(" 0x%08X - (L2CTX_TX_TBDR_BDHADDR_HI) h/w buffer "
8857                                         "descriptor address\n", CTX_RD(sc, GET_CID_ADDR(cid),
8858                                         BCE_L2CTX_TX_TBDR_BHADDR_HI));
8859                                 BCE_PRINTF(" 0x%08X - (L2CTX_TX_TBDR_BHADDR_LO) h/w buffer "
8860                                         "descriptor address\n", CTX_RD(sc, GET_CID_ADDR(cid),
8861                                         BCE_L2CTX_TX_TBDR_BHADDR_LO));
8862                                 BCE_PRINTF(" 0x%08X - (L2CTX_TX_HOST_BIDX) host producer "
8863                                         "index\n", CTX_RD(sc, GET_CID_ADDR(cid),
8864                                         BCE_L2CTX_TX_HOST_BIDX));
8865                                 BCE_PRINTF(" 0x%08X - (L2CTX_TX_HOST_BSEQ) host byte "
8866                                         "sequence\n", CTX_RD(sc, GET_CID_ADDR(cid),
8867                                         BCE_L2CTX_TX_HOST_BSEQ));
8868                         }
8869                 } else
8870                         BCE_PRINTF(" Unknown CID\n");
8871
8872                 BCE_PRINTF(
8873                         "----------------------------"
8874                         "    Raw CTX     "
8875                         "----------------------------\n");
8876
8877                 for (int i = 0x0; i < 0x300; i += 0x10) {
8878                         BCE_PRINTF("0x%04X: 0x%08X 0x%08X 0x%08X 0x%08X\n", i,
8879                                 CTX_RD(sc, GET_CID_ADDR(cid), i),
8880                                 CTX_RD(sc, GET_CID_ADDR(cid), i + 0x4),
8881                                 CTX_RD(sc, GET_CID_ADDR(cid), i + 0x8),
8882                                 CTX_RD(sc, GET_CID_ADDR(cid), i + 0xc));
8883                 }
8884
8885
8886                 BCE_PRINTF(
8887                         "----------------------------"
8888                         "----------------"
8889                         "----------------------------\n");
8890         }
8891 }
8892
8893
8894 /****************************************************************************/
8895 /* Prints out the FTQ data.                                                 */
8896 /*                                                                          */
8897 /* Returns:                                                                */
8898 /*   Nothing.                                                               */
8899 /****************************************************************************/
8900 static __attribute__ ((noinline)) void
8901 bce_dump_ftqs(struct bce_softc *sc)
8902 {
8903         u32 cmd, ctl, cur_depth, max_depth, valid_cnt, val;
8904
8905         BCE_PRINTF(
8906                 "----------------------------"
8907                 "    FTQ Data    "
8908                 "----------------------------\n");
8909
8910         BCE_PRINTF("   FTQ    Command    Control   Depth_Now  Max_Depth  Valid_Cnt \n");
8911         BCE_PRINTF(" ------- ---------- ---------- ---------- ---------- ----------\n");
8912
8913         /* Setup the generic statistic counters for the FTQ valid count. */
8914         val = (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_RV2PPQ_VALID_CNT << 24) |
8915                 (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_RXPCQ_VALID_CNT  << 16) |
8916                 (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_RXPQ_VALID_CNT   <<  8) |
8917                 (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_RLUPQ_VALID_CNT);
8918         REG_WR(sc, BCE_HC_STAT_GEN_SEL_0, val);
8919
8920         val = (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_TSCHQ_VALID_CNT  << 24) |
8921                 (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_RDMAQ_VALID_CNT  << 16) |
8922                 (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_RV2PTQ_VALID_CNT <<  8) |
8923                 (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_RV2PMQ_VALID_CNT);
8924         REG_WR(sc, BCE_HC_STAT_GEN_SEL_1, val);
8925
8926         val = (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_TPATQ_VALID_CNT  << 24) |
8927                 (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_TDMAQ_VALID_CNT  << 16) |
8928                 (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_TXPQ_VALID_CNT   <<  8) |
8929                 (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_TBDRQ_VALID_CNT);
8930         REG_WR(sc, BCE_HC_STAT_GEN_SEL_2, val);
8931
8932         val = (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_COMQ_VALID_CNT   << 24) |
8933                 (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_COMTQ_VALID_CNT  << 16) |
8934                 (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_COMXQ_VALID_CNT  <<  8) |
8935                 (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_TASQ_VALID_CNT);
8936         REG_WR(sc, BCE_HC_STAT_GEN_SEL_3, val);
8937
8938         /* Input queue to the Receive Lookup state machine */
8939         cmd = REG_RD(sc, BCE_RLUP_FTQ_CMD);
8940         ctl = REG_RD(sc, BCE_RLUP_FTQ_CTL);
8941         cur_depth = (ctl & BCE_RLUP_FTQ_CTL_CUR_DEPTH) >> 22;
8942         max_depth = (ctl & BCE_RLUP_FTQ_CTL_MAX_DEPTH) >> 12;
8943         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT0);
8944         BCE_PRINTF(" RLUP    0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
8945                 cmd, ctl, cur_depth, max_depth, valid_cnt);
8946
8947         /* Input queue to the Receive Processor */
8948         cmd = REG_RD_IND(sc, BCE_RXP_FTQ_CMD);
8949         ctl = REG_RD_IND(sc, BCE_RXP_FTQ_CTL);
8950         cur_depth = (ctl & BCE_RXP_FTQ_CTL_CUR_DEPTH) >> 22;
8951         max_depth = (ctl & BCE_RXP_FTQ_CTL_MAX_DEPTH) >> 12;
8952         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT1);
8953         BCE_PRINTF(" RXP     0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
8954                 cmd, ctl, cur_depth, max_depth, valid_cnt);
8955
8956         /* Input queue to the Recevie Processor */
8957         cmd = REG_RD_IND(sc, BCE_RXP_CFTQ_CMD);
8958         ctl = REG_RD_IND(sc, BCE_RXP_CFTQ_CTL);
8959         cur_depth = (ctl & BCE_RXP_CFTQ_CTL_CUR_DEPTH) >> 22;
8960         max_depth = (ctl & BCE_RXP_CFTQ_CTL_MAX_DEPTH) >> 12;
8961         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT2);
8962         BCE_PRINTF(" RXPC    0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
8963                 cmd, ctl, cur_depth, max_depth, valid_cnt);
8964
8965         /* Input queue to the Receive Virtual to Physical state machine */
8966         cmd = REG_RD(sc, BCE_RV2P_PFTQ_CMD);
8967         ctl = REG_RD(sc, BCE_RV2P_PFTQ_CTL);
8968         cur_depth = (ctl & BCE_RV2P_PFTQ_CTL_CUR_DEPTH) >> 22;
8969         max_depth = (ctl & BCE_RV2P_PFTQ_CTL_MAX_DEPTH) >> 12;
8970         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT3);
8971         BCE_PRINTF(" RV2PP   0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
8972                 cmd, ctl, cur_depth, max_depth, valid_cnt);
8973
8974         /* Input queue to the Recevie Virtual to Physical state machine */
8975         cmd = REG_RD(sc, BCE_RV2P_MFTQ_CMD);
8976         ctl = REG_RD(sc, BCE_RV2P_MFTQ_CTL);
8977         cur_depth = (ctl & BCE_RV2P_MFTQ_CTL_CUR_DEPTH) >> 22;
8978         max_depth = (ctl & BCE_RV2P_MFTQ_CTL_MAX_DEPTH) >> 12;
8979         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT4);
8980         BCE_PRINTF(" RV2PM   0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
8981                 cmd, ctl, cur_depth, max_depth, valid_cnt);
8982
8983         /* Input queue to the Receive Virtual to Physical state machine */
8984         cmd = REG_RD(sc, BCE_RV2P_TFTQ_CMD);
8985         ctl = REG_RD(sc, BCE_RV2P_TFTQ_CTL);
8986         cur_depth = (ctl & BCE_RV2P_TFTQ_CTL_CUR_DEPTH) >> 22;
8987         max_depth = (ctl & BCE_RV2P_TFTQ_CTL_MAX_DEPTH) >> 12;
8988         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT5);
8989         BCE_PRINTF(" RV2PT   0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
8990                 cmd, ctl, cur_depth, max_depth, valid_cnt);
8991
8992         /* Input queue to the Receive DMA state machine */
8993         cmd = REG_RD(sc, BCE_RDMA_FTQ_CMD);
8994         ctl = REG_RD(sc, BCE_RDMA_FTQ_CTL);
8995         cur_depth = (ctl & BCE_RDMA_FTQ_CTL_CUR_DEPTH) >> 22;
8996         max_depth = (ctl & BCE_RDMA_FTQ_CTL_MAX_DEPTH) >> 12;
8997         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT6);
8998         BCE_PRINTF(" RDMA    0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
8999                 cmd, ctl, cur_depth, max_depth, valid_cnt);
9000
9001         /* Input queue to the Transmit Scheduler state machine */
9002         cmd = REG_RD(sc, BCE_TSCH_FTQ_CMD);
9003         ctl = REG_RD(sc, BCE_TSCH_FTQ_CTL);
9004         cur_depth = (ctl & BCE_TSCH_FTQ_CTL_CUR_DEPTH) >> 22;
9005         max_depth = (ctl & BCE_TSCH_FTQ_CTL_MAX_DEPTH) >> 12;
9006         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT7);
9007         BCE_PRINTF(" TSCH    0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
9008                 cmd, ctl, cur_depth, max_depth, valid_cnt);
9009
9010         /* Input queue to the Transmit Buffer Descriptor state machine */
9011         cmd = REG_RD(sc, BCE_TBDR_FTQ_CMD);
9012         ctl = REG_RD(sc, BCE_TBDR_FTQ_CTL);
9013         cur_depth = (ctl & BCE_TBDR_FTQ_CTL_CUR_DEPTH) >> 22;
9014         max_depth = (ctl & BCE_TBDR_FTQ_CTL_MAX_DEPTH) >> 12;
9015         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT8);
9016         BCE_PRINTF(" TBDR    0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
9017                 cmd, ctl, cur_depth, max_depth, valid_cnt);
9018
9019         /* Input queue to the Transmit Processor */
9020         cmd = REG_RD_IND(sc, BCE_TXP_FTQ_CMD);
9021         ctl = REG_RD_IND(sc, BCE_TXP_FTQ_CTL);
9022         cur_depth = (ctl & BCE_TXP_FTQ_CTL_CUR_DEPTH) >> 22;
9023         max_depth = (ctl & BCE_TXP_FTQ_CTL_MAX_DEPTH) >> 12;
9024         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT9);
9025         BCE_PRINTF(" TXP     0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
9026                 cmd, ctl, cur_depth, max_depth, valid_cnt);
9027
9028         /* Input queue to the Transmit DMA state machine */
9029         cmd = REG_RD(sc, BCE_TDMA_FTQ_CMD);
9030         ctl = REG_RD(sc, BCE_TDMA_FTQ_CTL);
9031         cur_depth = (ctl & BCE_TDMA_FTQ_CTL_CUR_DEPTH) >> 22;
9032         max_depth = (ctl & BCE_TDMA_FTQ_CTL_MAX_DEPTH) >> 12;
9033         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT10);
9034         BCE_PRINTF(" TDMA    0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
9035                 cmd, ctl, cur_depth, max_depth, valid_cnt);
9036
9037         /* Input queue to the Transmit Patch-Up Processor */
9038         cmd = REG_RD_IND(sc, BCE_TPAT_FTQ_CMD);
9039         ctl = REG_RD_IND(sc, BCE_TPAT_FTQ_CTL);
9040         cur_depth = (ctl & BCE_TPAT_FTQ_CTL_CUR_DEPTH) >> 22;
9041         max_depth = (ctl & BCE_TPAT_FTQ_CTL_MAX_DEPTH) >> 12;
9042         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT11);
9043         BCE_PRINTF(" TPAT    0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
9044                 cmd, ctl, cur_depth, max_depth, valid_cnt);
9045
9046         /* Input queue to the Transmit Assembler state machine */
9047         cmd = REG_RD_IND(sc, BCE_TAS_FTQ_CMD);
9048         ctl = REG_RD_IND(sc, BCE_TAS_FTQ_CTL);
9049         cur_depth = (ctl & BCE_TAS_FTQ_CTL_CUR_DEPTH) >> 22;
9050         max_depth = (ctl & BCE_TAS_FTQ_CTL_MAX_DEPTH) >> 12;
9051         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT12);
9052         BCE_PRINTF(" TAS     0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
9053                 cmd, ctl, cur_depth, max_depth, valid_cnt);
9054
9055         /* Input queue to the Completion Processor */
9056         cmd = REG_RD_IND(sc, BCE_COM_COMXQ_FTQ_CMD);
9057         ctl = REG_RD_IND(sc, BCE_COM_COMXQ_FTQ_CTL);
9058         cur_depth = (ctl & BCE_COM_COMXQ_FTQ_CTL_CUR_DEPTH) >> 22;
9059         max_depth = (ctl & BCE_COM_COMXQ_FTQ_CTL_MAX_DEPTH) >> 12;
9060         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT13);
9061         BCE_PRINTF(" COMX    0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
9062                 cmd, ctl, cur_depth, max_depth, valid_cnt);
9063
9064         /* Input queue to the Completion Processor */
9065         cmd = REG_RD_IND(sc, BCE_COM_COMTQ_FTQ_CMD);
9066         ctl = REG_RD_IND(sc, BCE_COM_COMTQ_FTQ_CTL);
9067         cur_depth = (ctl & BCE_COM_COMTQ_FTQ_CTL_CUR_DEPTH) >> 22;
9068         max_depth = (ctl & BCE_COM_COMTQ_FTQ_CTL_MAX_DEPTH) >> 12;
9069         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT14);
9070         BCE_PRINTF(" COMT    0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
9071                 cmd, ctl, cur_depth, max_depth, valid_cnt);
9072
9073         /* Input queue to the Completion Processor */
9074         cmd = REG_RD_IND(sc, BCE_COM_COMQ_FTQ_CMD);
9075         ctl = REG_RD_IND(sc, BCE_COM_COMQ_FTQ_CTL);
9076         cur_depth = (ctl & BCE_COM_COMQ_FTQ_CTL_CUR_DEPTH) >> 22;
9077         max_depth = (ctl & BCE_COM_COMQ_FTQ_CTL_MAX_DEPTH) >> 12;
9078         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT15);
9079         BCE_PRINTF(" COMX    0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
9080                 cmd, ctl, cur_depth, max_depth, valid_cnt);
9081
9082         /* Setup the generic statistic counters for the FTQ valid count. */
9083         val = (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_CSQ_VALID_CNT  << 16) |
9084                 (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_CPQ_VALID_CNT  <<  8) |
9085                 (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_MGMQ_VALID_CNT);
9086
9087         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709)     ||
9088                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716))
9089                 val = val | (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_RV2PCSQ_VALID_CNT_XI << 24);
9090                 REG_WR(sc, BCE_HC_STAT_GEN_SEL_0, val);
9091
9092         /* Input queue to the Management Control Processor */
9093         cmd = REG_RD_IND(sc, BCE_MCP_MCPQ_FTQ_CMD);
9094         ctl = REG_RD_IND(sc, BCE_MCP_MCPQ_FTQ_CTL);
9095         cur_depth = (ctl & BCE_MCP_MCPQ_FTQ_CTL_CUR_DEPTH) >> 22;
9096         max_depth = (ctl & BCE_MCP_MCPQ_FTQ_CTL_MAX_DEPTH) >> 12;
9097         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT0);
9098         BCE_PRINTF(" MCP     0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
9099                 cmd, ctl, cur_depth, max_depth, valid_cnt);
9100
9101         /* Input queue to the Command Processor */
9102         cmd = REG_RD_IND(sc, BCE_CP_CPQ_FTQ_CMD);
9103         ctl = REG_RD_IND(sc, BCE_CP_CPQ_FTQ_CTL);
9104         cur_depth = (ctl & BCE_CP_CPQ_FTQ_CTL_CUR_DEPTH) >> 22;
9105         max_depth = (ctl & BCE_CP_CPQ_FTQ_CTL_MAX_DEPTH) >> 12;
9106         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT1);
9107         BCE_PRINTF(" CP      0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
9108                 cmd, ctl, cur_depth, max_depth, valid_cnt);
9109
9110         /* Input queue to the Completion Scheduler state machine */
9111         cmd = REG_RD(sc, BCE_CSCH_CH_FTQ_CMD);
9112         ctl = REG_RD(sc, BCE_CSCH_CH_FTQ_CTL);
9113         cur_depth = (ctl & BCE_CSCH_CH_FTQ_CTL_CUR_DEPTH) >> 22;
9114         max_depth = (ctl & BCE_CSCH_CH_FTQ_CTL_MAX_DEPTH) >> 12;
9115         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT2);
9116         BCE_PRINTF(" CS      0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
9117                 cmd, ctl, cur_depth, max_depth, valid_cnt);
9118
9119         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
9120                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
9121                 /* Input queue to the Receive Virtual to Physical Command Scheduler */
9122                 cmd = REG_RD(sc, BCE_RV2PCSR_FTQ_CMD);
9123                 ctl = REG_RD(sc, BCE_RV2PCSR_FTQ_CTL);
9124                 cur_depth = (ctl & 0xFFC00000) >> 22;
9125                 max_depth = (ctl & 0x003FF000) >> 12;
9126                 valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT3);
9127                 BCE_PRINTF(" RV2PCSR 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
9128                         cmd, ctl, cur_depth, max_depth, valid_cnt);
9129         }
9130
9131         BCE_PRINTF(
9132                 "----------------------------"
9133                 "----------------"
9134                 "----------------------------\n");
9135 }
9136
9137
9138 /****************************************************************************/
9139 /* Prints out the TX chain.                                                 */
9140 /*                                                                          */
9141 /* Returns:                                                                 */
9142 /*   Nothing.                                                               */
9143 /****************************************************************************/
9144 static __attribute__ ((noinline)) void
9145 bce_dump_tx_chain(struct bce_softc *sc, u16 tx_prod, int count)
9146 {
9147         struct tx_bd *txbd;
9148
9149         /* First some info about the tx_bd chain structure. */
9150         BCE_PRINTF(
9151                 "----------------------------"
9152                 "  tx_bd  chain  "
9153                 "----------------------------\n");
9154
9155         BCE_PRINTF("page size      = 0x%08X, tx chain pages        = 0x%08X\n",
9156                 (u32) BCM_PAGE_SIZE, (u32) TX_PAGES);
9157
9158         BCE_PRINTF("tx_bd per page = 0x%08X, usable tx_bd per page = 0x%08X\n",
9159                 (u32) TOTAL_TX_BD_PER_PAGE, (u32) USABLE_TX_BD_PER_PAGE);
9160
9161         BCE_PRINTF("total tx_bd    = 0x%08X\n", (u32) TOTAL_TX_BD);
9162
9163         BCE_PRINTF(
9164                 "----------------------------"
9165                 "   tx_bd data   "
9166                 "----------------------------\n");
9167
9168         /* Now print out the tx_bd's themselves. */
9169         for (int i = 0; i < count; i++) {
9170                 txbd = &sc->tx_bd_chain[TX_PAGE(tx_prod)][TX_IDX(tx_prod)];
9171                 bce_dump_txbd(sc, tx_prod, txbd);
9172                 tx_prod = NEXT_TX_BD(tx_prod);
9173         }
9174
9175         BCE_PRINTF(
9176                 "----------------------------"
9177                 "----------------"
9178                 "----------------------------\n");
9179 }
9180
9181
9182 /****************************************************************************/
9183 /* Prints out the RX chain.                                                 */
9184 /*                                                                          */
9185 /* Returns:                                                                 */
9186 /*   Nothing.                                                               */
9187 /****************************************************************************/
9188 static __attribute__ ((noinline)) void
9189 bce_dump_rx_chain(struct bce_softc *sc, u16 rx_prod, int count)
9190 {
9191         struct rx_bd *rxbd;
9192
9193         /* First some info about the rx_bd chain structure. */
9194         BCE_PRINTF(
9195                 "----------------------------"
9196                 "  rx_bd  chain  "
9197                 "----------------------------\n");
9198
9199         BCE_PRINTF("page size      = 0x%08X, rx chain pages        = 0x%08X\n",
9200                 (u32) BCM_PAGE_SIZE, (u32) RX_PAGES);
9201
9202         BCE_PRINTF("rx_bd per page = 0x%08X, usable rx_bd per page = 0x%08X\n",
9203                 (u32) TOTAL_RX_BD_PER_PAGE, (u32) USABLE_RX_BD_PER_PAGE);
9204
9205         BCE_PRINTF("total rx_bd    = 0x%08X\n", (u32) TOTAL_RX_BD);
9206
9207         BCE_PRINTF(
9208                 "----------------------------"
9209                 "   rx_bd data   "
9210                 "----------------------------\n");
9211
9212         /* Now print out the rx_bd's themselves. */
9213         for (int i = 0; i < count; i++) {
9214                 rxbd = &sc->rx_bd_chain[RX_PAGE(rx_prod)][RX_IDX(rx_prod)];
9215                 bce_dump_rxbd(sc, rx_prod, rxbd);
9216                 rx_prod = RX_CHAIN_IDX(rx_prod + 1);
9217         }
9218
9219         BCE_PRINTF(
9220                 "----------------------------"
9221                 "----------------"
9222                 "----------------------------\n");
9223 }
9224
9225
9226 #ifdef ZERO_COPY_SOCKETS
9227 /****************************************************************************/
9228 /* Prints out the page chain.                                               */
9229 /*                                                                          */
9230 /* Returns:                                                                 */
9231 /*   Nothing.                                                               */
9232 /****************************************************************************/
9233 static __attribute__ ((noinline)) void
9234 bce_dump_pg_chain(struct bce_softc *sc, u16 pg_prod, int count)
9235 {
9236         struct rx_bd *pgbd;
9237
9238         /* First some info about the page chain structure. */
9239         BCE_PRINTF(
9240                 "----------------------------"
9241                 "   page chain   "
9242                 "----------------------------\n");
9243
9244         BCE_PRINTF("page size      = 0x%08X, pg chain pages        = 0x%08X\n",
9245                 (u32) BCM_PAGE_SIZE, (u32) PG_PAGES);
9246
9247         BCE_PRINTF("rx_bd per page = 0x%08X, usable rx_bd per page = 0x%08X\n",
9248                 (u32) TOTAL_PG_BD_PER_PAGE, (u32) USABLE_PG_BD_PER_PAGE);
9249
9250         BCE_PRINTF("total rx_bd    = 0x%08X, max_pg_bd             = 0x%08X\n",
9251                 (u32) TOTAL_PG_BD, (u32) MAX_PG_BD);
9252
9253         BCE_PRINTF(
9254                 "----------------------------"
9255                 "   page data    "
9256                 "----------------------------\n");
9257
9258         /* Now print out the rx_bd's themselves. */
9259         for (int i = 0; i < count; i++) {
9260                 pgbd = &sc->pg_bd_chain[PG_PAGE(pg_prod)][PG_IDX(pg_prod)];
9261                 bce_dump_pgbd(sc, pg_prod, pgbd);
9262                 pg_prod = PG_CHAIN_IDX(pg_prod + 1);
9263         }
9264
9265         BCE_PRINTF(
9266                 "----------------------------"
9267                 "----------------"
9268                 "----------------------------\n");
9269 }
9270 #endif
9271
9272
9273 /****************************************************************************/
9274 /* Prints out the status block from host memory.                            */
9275 /*                                                                          */
9276 /* Returns:                                                                 */
9277 /*   Nothing.                                                               */
9278 /****************************************************************************/
9279 static __attribute__ ((noinline)) void
9280 bce_dump_status_block(struct bce_softc *sc)
9281 {
9282         struct status_block *sblk;
9283
9284         sblk = sc->status_block;
9285
9286         BCE_PRINTF(
9287                 "----------------------------"
9288                 "  Status Block  "
9289                 "----------------------------\n");
9290
9291         BCE_PRINTF("    0x%08X - attn_bits\n",
9292                 sblk->status_attn_bits);
9293
9294         BCE_PRINTF("    0x%08X - attn_bits_ack\n",
9295                 sblk->status_attn_bits_ack);
9296
9297         BCE_PRINTF("0x%04X(0x%04X) - rx_cons0\n",
9298                 sblk->status_rx_quick_consumer_index0,
9299                 (u16) RX_CHAIN_IDX(sblk->status_rx_quick_consumer_index0));
9300
9301         BCE_PRINTF("0x%04X(0x%04X) - tx_cons0\n",
9302                 sblk->status_tx_quick_consumer_index0,
9303                 (u16) TX_CHAIN_IDX(sblk->status_tx_quick_consumer_index0));
9304
9305         BCE_PRINTF("        0x%04X - status_idx\n", sblk->status_idx);
9306
9307         /* Theses indices are not used for normal L2 drivers. */
9308         if (sblk->status_rx_quick_consumer_index1)
9309                 BCE_PRINTF("0x%04X(0x%04X) - rx_cons1\n",
9310                         sblk->status_rx_quick_consumer_index1,
9311                         (u16) RX_CHAIN_IDX(sblk->status_rx_quick_consumer_index1));
9312
9313         if (sblk->status_tx_quick_consumer_index1)
9314                 BCE_PRINTF("0x%04X(0x%04X) - tx_cons1\n",
9315                         sblk->status_tx_quick_consumer_index1,
9316                         (u16) TX_CHAIN_IDX(sblk->status_tx_quick_consumer_index1));
9317
9318         if (sblk->status_rx_quick_consumer_index2)
9319                 BCE_PRINTF("0x%04X(0x%04X)- rx_cons2\n",
9320                         sblk->status_rx_quick_consumer_index2,
9321                         (u16) RX_CHAIN_IDX(sblk->status_rx_quick_consumer_index2));
9322
9323         if (sblk->status_tx_quick_consumer_index2)
9324                 BCE_PRINTF("0x%04X(0x%04X) - tx_cons2\n",
9325                         sblk->status_tx_quick_consumer_index2,
9326                         (u16) TX_CHAIN_IDX(sblk->status_tx_quick_consumer_index2));
9327
9328         if (sblk->status_rx_quick_consumer_index3)
9329                 BCE_PRINTF("0x%04X(0x%04X) - rx_cons3\n",
9330                         sblk->status_rx_quick_consumer_index3,
9331                         (u16) RX_CHAIN_IDX(sblk->status_rx_quick_consumer_index3));
9332
9333         if (sblk->status_tx_quick_consumer_index3)
9334                 BCE_PRINTF("0x%04X(0x%04X) - tx_cons3\n",
9335                         sblk->status_tx_quick_consumer_index3,
9336                         (u16) TX_CHAIN_IDX(sblk->status_tx_quick_consumer_index3));
9337
9338         if (sblk->status_rx_quick_consumer_index4 ||
9339                 sblk->status_rx_quick_consumer_index5)
9340                 BCE_PRINTF("rx_cons4  = 0x%08X, rx_cons5      = 0x%08X\n",
9341                         sblk->status_rx_quick_consumer_index4,
9342                         sblk->status_rx_quick_consumer_index5);
9343
9344         if (sblk->status_rx_quick_consumer_index6 ||
9345                 sblk->status_rx_quick_consumer_index7)
9346                 BCE_PRINTF("rx_cons6  = 0x%08X, rx_cons7      = 0x%08X\n",
9347                         sblk->status_rx_quick_consumer_index6,
9348                         sblk->status_rx_quick_consumer_index7);
9349
9350         if (sblk->status_rx_quick_consumer_index8 ||
9351                 sblk->status_rx_quick_consumer_index9)
9352                 BCE_PRINTF("rx_cons8  = 0x%08X, rx_cons9      = 0x%08X\n",
9353                         sblk->status_rx_quick_consumer_index8,
9354                         sblk->status_rx_quick_consumer_index9);
9355
9356         if (sblk->status_rx_quick_consumer_index10 ||
9357                 sblk->status_rx_quick_consumer_index11)
9358                 BCE_PRINTF("rx_cons10 = 0x%08X, rx_cons11     = 0x%08X\n",
9359                         sblk->status_rx_quick_consumer_index10,
9360                         sblk->status_rx_quick_consumer_index11);
9361
9362         if (sblk->status_rx_quick_consumer_index12 ||
9363                 sblk->status_rx_quick_consumer_index13)
9364                 BCE_PRINTF("rx_cons12 = 0x%08X, rx_cons13     = 0x%08X\n",
9365                         sblk->status_rx_quick_consumer_index12,
9366                         sblk->status_rx_quick_consumer_index13);
9367
9368         if (sblk->status_rx_quick_consumer_index14 ||
9369                 sblk->status_rx_quick_consumer_index15)
9370                 BCE_PRINTF("rx_cons14 = 0x%08X, rx_cons15     = 0x%08X\n",
9371                         sblk->status_rx_quick_consumer_index14,
9372                         sblk->status_rx_quick_consumer_index15);
9373
9374         if (sblk->status_completion_producer_index ||
9375                 sblk->status_cmd_consumer_index)
9376                 BCE_PRINTF("com_prod  = 0x%08X, cmd_cons      = 0x%08X\n",
9377                         sblk->status_completion_producer_index,
9378                         sblk->status_cmd_consumer_index);
9379
9380         BCE_PRINTF(
9381                 "----------------------------"
9382                 "----------------"
9383                 "----------------------------\n");
9384 }
9385
9386
9387 /****************************************************************************/
9388 /* Prints out the statistics block from host memory.                        */
9389 /*                                                                          */
9390 /* Returns:                                                                 */
9391 /*   Nothing.                                                               */
9392 /****************************************************************************/
9393 static __attribute__ ((noinline)) void
9394 bce_dump_stats_block(struct bce_softc *sc)
9395 {
9396         struct statistics_block *sblk;
9397
9398         sblk = sc->stats_block;
9399
9400         BCE_PRINTF(
9401                 "---------------"
9402                 " Stats Block  (All Stats Not Shown Are 0) "
9403                 "---------------\n");
9404
9405         if (sblk->stat_IfHCInOctets_hi
9406                 || sblk->stat_IfHCInOctets_lo)
9407                 BCE_PRINTF("0x%08X:%08X : "
9408                         "IfHcInOctets\n",
9409                         sblk->stat_IfHCInOctets_hi,
9410                         sblk->stat_IfHCInOctets_lo);
9411
9412         if (sblk->stat_IfHCInBadOctets_hi
9413                 || sblk->stat_IfHCInBadOctets_lo)
9414                 BCE_PRINTF("0x%08X:%08X : "
9415                         "IfHcInBadOctets\n",
9416                         sblk->stat_IfHCInBadOctets_hi,
9417                         sblk->stat_IfHCInBadOctets_lo);
9418
9419         if (sblk->stat_IfHCOutOctets_hi
9420                 || sblk->stat_IfHCOutOctets_lo)
9421                 BCE_PRINTF("0x%08X:%08X : "
9422                         "IfHcOutOctets\n",
9423                         sblk->stat_IfHCOutOctets_hi,
9424                         sblk->stat_IfHCOutOctets_lo);
9425
9426         if (sblk->stat_IfHCOutBadOctets_hi
9427                 || sblk->stat_IfHCOutBadOctets_lo)
9428                 BCE_PRINTF("0x%08X:%08X : "
9429                         "IfHcOutBadOctets\n",
9430                         sblk->stat_IfHCOutBadOctets_hi,
9431                         sblk->stat_IfHCOutBadOctets_lo);
9432
9433         if (sblk->stat_IfHCInUcastPkts_hi
9434                 || sblk->stat_IfHCInUcastPkts_lo)
9435                 BCE_PRINTF("0x%08X:%08X : "
9436                         "IfHcInUcastPkts\n",
9437                         sblk->stat_IfHCInUcastPkts_hi,
9438                         sblk->stat_IfHCInUcastPkts_lo);
9439
9440         if (sblk->stat_IfHCInBroadcastPkts_hi
9441                 || sblk->stat_IfHCInBroadcastPkts_lo)
9442                 BCE_PRINTF("0x%08X:%08X : "
9443                         "IfHcInBroadcastPkts\n",
9444                         sblk->stat_IfHCInBroadcastPkts_hi,
9445                         sblk->stat_IfHCInBroadcastPkts_lo);
9446
9447         if (sblk->stat_IfHCInMulticastPkts_hi
9448                 || sblk->stat_IfHCInMulticastPkts_lo)
9449                 BCE_PRINTF("0x%08X:%08X : "
9450                         "IfHcInMulticastPkts\n",
9451                         sblk->stat_IfHCInMulticastPkts_hi,
9452                         sblk->stat_IfHCInMulticastPkts_lo);
9453
9454         if (sblk->stat_IfHCOutUcastPkts_hi
9455                 || sblk->stat_IfHCOutUcastPkts_lo)
9456                 BCE_PRINTF("0x%08X:%08X : "
9457                         "IfHcOutUcastPkts\n",
9458                         sblk->stat_IfHCOutUcastPkts_hi,
9459                         sblk->stat_IfHCOutUcastPkts_lo);
9460
9461         if (sblk->stat_IfHCOutBroadcastPkts_hi
9462                 || sblk->stat_IfHCOutBroadcastPkts_lo)
9463                 BCE_PRINTF("0x%08X:%08X : "
9464                         "IfHcOutBroadcastPkts\n",
9465                         sblk->stat_IfHCOutBroadcastPkts_hi,
9466                         sblk->stat_IfHCOutBroadcastPkts_lo);
9467
9468         if (sblk->stat_IfHCOutMulticastPkts_hi
9469                 || sblk->stat_IfHCOutMulticastPkts_lo)
9470                 BCE_PRINTF("0x%08X:%08X : "
9471                         "IfHcOutMulticastPkts\n",
9472                         sblk->stat_IfHCOutMulticastPkts_hi,
9473                         sblk->stat_IfHCOutMulticastPkts_lo);
9474
9475         if (sblk->stat_emac_tx_stat_dot3statsinternalmactransmiterrors)
9476                 BCE_PRINTF("         0x%08X : "
9477                         "emac_tx_stat_dot3statsinternalmactransmiterrors\n",
9478                         sblk->stat_emac_tx_stat_dot3statsinternalmactransmiterrors);
9479
9480         if (sblk->stat_Dot3StatsCarrierSenseErrors)
9481                 BCE_PRINTF("         0x%08X : Dot3StatsCarrierSenseErrors\n",
9482                         sblk->stat_Dot3StatsCarrierSenseErrors);
9483
9484         if (sblk->stat_Dot3StatsFCSErrors)
9485                 BCE_PRINTF("         0x%08X : Dot3StatsFCSErrors\n",
9486                         sblk->stat_Dot3StatsFCSErrors);
9487
9488         if (sblk->stat_Dot3StatsAlignmentErrors)
9489                 BCE_PRINTF("         0x%08X : Dot3StatsAlignmentErrors\n",
9490                         sblk->stat_Dot3StatsAlignmentErrors);
9491
9492         if (sblk->stat_Dot3StatsSingleCollisionFrames)
9493                 BCE_PRINTF("         0x%08X : Dot3StatsSingleCollisionFrames\n",
9494                         sblk->stat_Dot3StatsSingleCollisionFrames);
9495
9496         if (sblk->stat_Dot3StatsMultipleCollisionFrames)
9497                 BCE_PRINTF("         0x%08X : Dot3StatsMultipleCollisionFrames\n",
9498                         sblk->stat_Dot3StatsMultipleCollisionFrames);
9499
9500         if (sblk->stat_Dot3StatsDeferredTransmissions)
9501                 BCE_PRINTF("         0x%08X : Dot3StatsDeferredTransmissions\n",
9502                         sblk->stat_Dot3StatsDeferredTransmissions);
9503
9504         if (sblk->stat_Dot3StatsExcessiveCollisions)
9505                 BCE_PRINTF("         0x%08X : Dot3StatsExcessiveCollisions\n",
9506                         sblk->stat_Dot3StatsExcessiveCollisions);
9507
9508         if (sblk->stat_Dot3StatsLateCollisions)
9509                 BCE_PRINTF("         0x%08X : Dot3StatsLateCollisions\n",
9510                         sblk->stat_Dot3StatsLateCollisions);
9511
9512         if (sblk->stat_EtherStatsCollisions)
9513                 BCE_PRINTF("         0x%08X : EtherStatsCollisions\n",
9514                         sblk->stat_EtherStatsCollisions);
9515
9516         if (sblk->stat_EtherStatsFragments)
9517                 BCE_PRINTF("         0x%08X : EtherStatsFragments\n",
9518                         sblk->stat_EtherStatsFragments);
9519
9520         if (sblk->stat_EtherStatsJabbers)
9521                 BCE_PRINTF("         0x%08X : EtherStatsJabbers\n",
9522                         sblk->stat_EtherStatsJabbers);
9523
9524         if (sblk->stat_EtherStatsUndersizePkts)
9525                 BCE_PRINTF("         0x%08X : EtherStatsUndersizePkts\n",
9526                         sblk->stat_EtherStatsUndersizePkts);
9527
9528         if (sblk->stat_EtherStatsOversizePkts)
9529                 BCE_PRINTF("         0x%08X : EtherStatsOverrsizePkts\n",
9530                         sblk->stat_EtherStatsOversizePkts);
9531
9532         if (sblk->stat_EtherStatsPktsRx64Octets)
9533                 BCE_PRINTF("         0x%08X : EtherStatsPktsRx64Octets\n",
9534                         sblk->stat_EtherStatsPktsRx64Octets);
9535
9536         if (sblk->stat_EtherStatsPktsRx65Octetsto127Octets)
9537                 BCE_PRINTF("         0x%08X : EtherStatsPktsRx65Octetsto127Octets\n",
9538                         sblk->stat_EtherStatsPktsRx65Octetsto127Octets);
9539
9540         if (sblk->stat_EtherStatsPktsRx128Octetsto255Octets)
9541                 BCE_PRINTF("         0x%08X : EtherStatsPktsRx128Octetsto255Octets\n",
9542                         sblk->stat_EtherStatsPktsRx128Octetsto255Octets);
9543
9544         if (sblk->stat_EtherStatsPktsRx256Octetsto511Octets)
9545                 BCE_PRINTF("         0x%08X : EtherStatsPktsRx256Octetsto511Octets\n",
9546                         sblk->stat_EtherStatsPktsRx256Octetsto511Octets);
9547
9548         if (sblk->stat_EtherStatsPktsRx512Octetsto1023Octets)
9549                 BCE_PRINTF("         0x%08X : EtherStatsPktsRx512Octetsto1023Octets\n",
9550                         sblk->stat_EtherStatsPktsRx512Octetsto1023Octets);
9551
9552         if (sblk->stat_EtherStatsPktsRx1024Octetsto1522Octets)
9553                 BCE_PRINTF("         0x%08X : EtherStatsPktsRx1024Octetsto1522Octets\n",
9554                         sblk->stat_EtherStatsPktsRx1024Octetsto1522Octets);
9555
9556         if (sblk->stat_EtherStatsPktsRx1523Octetsto9022Octets)
9557                 BCE_PRINTF("         0x%08X : EtherStatsPktsRx1523Octetsto9022Octets\n",
9558                         sblk->stat_EtherStatsPktsRx1523Octetsto9022Octets);
9559
9560         if (sblk->stat_EtherStatsPktsTx64Octets)
9561                 BCE_PRINTF("         0x%08X : EtherStatsPktsTx64Octets\n",
9562                         sblk->stat_EtherStatsPktsTx64Octets);
9563
9564         if (sblk->stat_EtherStatsPktsTx65Octetsto127Octets)
9565                 BCE_PRINTF("         0x%08X : EtherStatsPktsTx65Octetsto127Octets\n",
9566                         sblk->stat_EtherStatsPktsTx65Octetsto127Octets);
9567
9568         if (sblk->stat_EtherStatsPktsTx128Octetsto255Octets)
9569                 BCE_PRINTF("         0x%08X : EtherStatsPktsTx128Octetsto255Octets\n",
9570                         sblk->stat_EtherStatsPktsTx128Octetsto255Octets);
9571
9572         if (sblk->stat_EtherStatsPktsTx256Octetsto511Octets)
9573                 BCE_PRINTF("         0x%08X : EtherStatsPktsTx256Octetsto511Octets\n",
9574                         sblk->stat_EtherStatsPktsTx256Octetsto511Octets);
9575
9576         if (sblk->stat_EtherStatsPktsTx512Octetsto1023Octets)
9577                 BCE_PRINTF("         0x%08X : EtherStatsPktsTx512Octetsto1023Octets\n",
9578                         sblk->stat_EtherStatsPktsTx512Octetsto1023Octets);
9579
9580         if (sblk->stat_EtherStatsPktsTx1024Octetsto1522Octets)
9581                 BCE_PRINTF("         0x%08X : EtherStatsPktsTx1024Octetsto1522Octets\n",
9582                         sblk->stat_EtherStatsPktsTx1024Octetsto1522Octets);
9583
9584         if (sblk->stat_EtherStatsPktsTx1523Octetsto9022Octets)
9585                 BCE_PRINTF("         0x%08X : EtherStatsPktsTx1523Octetsto9022Octets\n",
9586                         sblk->stat_EtherStatsPktsTx1523Octetsto9022Octets);
9587
9588         if (sblk->stat_XonPauseFramesReceived)
9589                 BCE_PRINTF("         0x%08X : XonPauseFramesReceived\n",
9590                         sblk->stat_XonPauseFramesReceived);
9591
9592         if (sblk->stat_XoffPauseFramesReceived)
9593            BCE_PRINTF("          0x%08X : XoffPauseFramesReceived\n",
9594                         sblk->stat_XoffPauseFramesReceived);
9595
9596         if (sblk->stat_OutXonSent)
9597                 BCE_PRINTF("         0x%08X : OutXonSent\n",
9598                         sblk->stat_OutXonSent);
9599
9600         if (sblk->stat_OutXoffSent)
9601                 BCE_PRINTF("         0x%08X : OutXoffSent\n",
9602                         sblk->stat_OutXoffSent);
9603
9604         if (sblk->stat_FlowControlDone)
9605                 BCE_PRINTF("         0x%08X : FlowControlDone\n",
9606                         sblk->stat_FlowControlDone);
9607
9608         if (sblk->stat_MacControlFramesReceived)
9609                 BCE_PRINTF("         0x%08X : MacControlFramesReceived\n",
9610                         sblk->stat_MacControlFramesReceived);
9611
9612         if (sblk->stat_XoffStateEntered)
9613                 BCE_PRINTF("         0x%08X : XoffStateEntered\n",
9614                         sblk->stat_XoffStateEntered);
9615
9616         if (sblk->stat_IfInFramesL2FilterDiscards)
9617                 BCE_PRINTF("         0x%08X : IfInFramesL2FilterDiscards\n",
9618                         sblk->stat_IfInFramesL2FilterDiscards);
9619
9620         if (sblk->stat_IfInRuleCheckerDiscards)
9621                 BCE_PRINTF("         0x%08X : IfInRuleCheckerDiscards\n",
9622                         sblk->stat_IfInRuleCheckerDiscards);
9623
9624         if (sblk->stat_IfInFTQDiscards)
9625                 BCE_PRINTF("         0x%08X : IfInFTQDiscards\n",
9626                         sblk->stat_IfInFTQDiscards);
9627
9628         if (sblk->stat_IfInMBUFDiscards)
9629                 BCE_PRINTF("         0x%08X : IfInMBUFDiscards\n",
9630                         sblk->stat_IfInMBUFDiscards);
9631
9632         if (sblk->stat_IfInRuleCheckerP4Hit)
9633                 BCE_PRINTF("         0x%08X : IfInRuleCheckerP4Hit\n",
9634                         sblk->stat_IfInRuleCheckerP4Hit);
9635
9636         if (sblk->stat_CatchupInRuleCheckerDiscards)
9637                 BCE_PRINTF("         0x%08X : CatchupInRuleCheckerDiscards\n",
9638                         sblk->stat_CatchupInRuleCheckerDiscards);
9639
9640         if (sblk->stat_CatchupInFTQDiscards)
9641                 BCE_PRINTF("         0x%08X : CatchupInFTQDiscards\n",
9642                         sblk->stat_CatchupInFTQDiscards);
9643
9644         if (sblk->stat_CatchupInMBUFDiscards)
9645                 BCE_PRINTF("         0x%08X : CatchupInMBUFDiscards\n",
9646                         sblk->stat_CatchupInMBUFDiscards);
9647
9648         if (sblk->stat_CatchupInRuleCheckerP4Hit)
9649                 BCE_PRINTF("         0x%08X : CatchupInRuleCheckerP4Hit\n",
9650                         sblk->stat_CatchupInRuleCheckerP4Hit);
9651
9652         BCE_PRINTF(
9653                 "----------------------------"
9654                 "----------------"
9655                 "----------------------------\n");
9656 }
9657
9658
9659 /****************************************************************************/
9660 /* Prints out a summary of the driver state.                                */
9661 /*                                                                          */
9662 /* Returns:                                                                 */
9663 /*   Nothing.                                                               */
9664 /****************************************************************************/
9665 static __attribute__ ((noinline)) void
9666 bce_dump_driver_state(struct bce_softc *sc)
9667 {
9668         u32 val_hi, val_lo;
9669
9670         BCE_PRINTF(
9671                 "-----------------------------"
9672                 " Driver State "
9673                 "-----------------------------\n");
9674
9675         val_hi = BCE_ADDR_HI(sc);
9676         val_lo = BCE_ADDR_LO(sc);
9677         BCE_PRINTF("0x%08X:%08X - (sc) driver softc structure virtual address\n",
9678                 val_hi, val_lo);
9679
9680         val_hi = BCE_ADDR_HI(sc->bce_vhandle);
9681         val_lo = BCE_ADDR_LO(sc->bce_vhandle);
9682         BCE_PRINTF("0x%08X:%08X - (sc->bce_vhandle) PCI BAR virtual address\n",
9683                 val_hi, val_lo);
9684
9685         val_hi = BCE_ADDR_HI(sc->status_block);
9686         val_lo = BCE_ADDR_LO(sc->status_block);
9687         BCE_PRINTF("0x%08X:%08X - (sc->status_block) status block virtual address\n",
9688                 val_hi, val_lo);
9689
9690         val_hi = BCE_ADDR_HI(sc->stats_block);
9691         val_lo = BCE_ADDR_LO(sc->stats_block);
9692         BCE_PRINTF("0x%08X:%08X - (sc->stats_block) statistics block virtual address\n",
9693                 val_hi, val_lo);
9694
9695         val_hi = BCE_ADDR_HI(sc->tx_bd_chain);
9696         val_lo = BCE_ADDR_LO(sc->tx_bd_chain);
9697         BCE_PRINTF(
9698                 "0x%08X:%08X - (sc->tx_bd_chain) tx_bd chain virtual adddress\n",
9699                 val_hi, val_lo);
9700
9701         val_hi = BCE_ADDR_HI(sc->rx_bd_chain);
9702         val_lo = BCE_ADDR_LO(sc->rx_bd_chain);
9703         BCE_PRINTF(
9704                 "0x%08X:%08X - (sc->rx_bd_chain) rx_bd chain virtual address\n",
9705                 val_hi, val_lo);
9706
9707 #ifdef ZERO_COPY_SOCKETS
9708         val_hi = BCE_ADDR_HI(sc->pg_bd_chain);
9709         val_lo = BCE_ADDR_LO(sc->pg_bd_chain);
9710         BCE_PRINTF(
9711                 "0x%08X:%08X - (sc->pg_bd_chain) page chain virtual address\n",
9712                 val_hi, val_lo);
9713 #endif
9714
9715         val_hi = BCE_ADDR_HI(sc->tx_mbuf_ptr);
9716         val_lo = BCE_ADDR_LO(sc->tx_mbuf_ptr);
9717         BCE_PRINTF(
9718                 "0x%08X:%08X - (sc->tx_mbuf_ptr) tx mbuf chain virtual address\n",
9719                 val_hi, val_lo);
9720
9721         val_hi = BCE_ADDR_HI(sc->rx_mbuf_ptr);
9722         val_lo = BCE_ADDR_LO(sc->rx_mbuf_ptr);
9723         BCE_PRINTF(
9724                 "0x%08X:%08X - (sc->rx_mbuf_ptr) rx mbuf chain virtual address\n",
9725                 val_hi, val_lo);
9726
9727 #ifdef ZERO_COPY_SOCKETS
9728         val_hi = BCE_ADDR_HI(sc->pg_mbuf_ptr);
9729         val_lo = BCE_ADDR_LO(sc->pg_mbuf_ptr);
9730         BCE_PRINTF(
9731                 "0x%08X:%08X - (sc->pg_mbuf_ptr) page mbuf chain virtual address\n",
9732                 val_hi, val_lo);
9733 #endif
9734
9735         BCE_PRINTF("         0x%08X - (sc->interrupts_generated) h/w intrs\n",
9736                 sc->interrupts_generated);
9737
9738         BCE_PRINTF("         0x%08X - (sc->rx_interrupts) rx interrupts handled\n",
9739                 sc->rx_interrupts);
9740
9741         BCE_PRINTF("         0x%08X - (sc->tx_interrupts) tx interrupts handled\n",
9742                 sc->tx_interrupts);
9743
9744         BCE_PRINTF("         0x%08X - (sc->last_status_idx) status block index\n",
9745                 sc->last_status_idx);
9746
9747         BCE_PRINTF("     0x%04X(0x%04X) - (sc->tx_prod) tx producer index\n",
9748                 sc->tx_prod, (u16) TX_CHAIN_IDX(sc->tx_prod));
9749
9750         BCE_PRINTF("     0x%04X(0x%04X) - (sc->tx_cons) tx consumer index\n",
9751                 sc->tx_cons, (u16) TX_CHAIN_IDX(sc->tx_cons));
9752
9753         BCE_PRINTF("         0x%08X - (sc->tx_prod_bseq) tx producer bseq index\n",
9754                 sc->tx_prod_bseq);
9755
9756         BCE_PRINTF("         0x%08X - (sc->debug_tx_mbuf_alloc) tx mbufs allocated\n",
9757                 sc->debug_tx_mbuf_alloc);
9758
9759         BCE_PRINTF("         0x%08X - (sc->used_tx_bd) used tx_bd's\n",
9760                 sc->used_tx_bd);
9761
9762         BCE_PRINTF("0x%08X/%08X - (sc->tx_hi_watermark) tx hi watermark\n",
9763                 sc->tx_hi_watermark, sc->max_tx_bd);
9764
9765         BCE_PRINTF("     0x%04X(0x%04X) - (sc->rx_prod) rx producer index\n",
9766                 sc->rx_prod, (u16) RX_CHAIN_IDX(sc->rx_prod));
9767
9768         BCE_PRINTF("     0x%04X(0x%04X) - (sc->rx_cons) rx consumer index\n",
9769                 sc->rx_cons, (u16) RX_CHAIN_IDX(sc->rx_cons));
9770
9771         BCE_PRINTF("         0x%08X - (sc->rx_prod_bseq) rx producer bseq index\n",
9772                 sc->rx_prod_bseq);
9773
9774         BCE_PRINTF("         0x%08X - (sc->debug_rx_mbuf_alloc) rx mbufs allocated\n",
9775                 sc->debug_rx_mbuf_alloc);
9776
9777         BCE_PRINTF("         0x%08X - (sc->free_rx_bd) free rx_bd's\n",
9778                 sc->free_rx_bd);
9779
9780 #ifdef ZERO_COPY_SOCKETS
9781         BCE_PRINTF("     0x%04X(0x%04X) - (sc->pg_prod) page producer index\n",
9782                 sc->pg_prod, (u16) PG_CHAIN_IDX(sc->pg_prod));
9783
9784         BCE_PRINTF("     0x%04X(0x%04X) - (sc->pg_cons) page consumer index\n",
9785                 sc->pg_cons, (u16) PG_CHAIN_IDX(sc->pg_cons));
9786
9787         BCE_PRINTF("         0x%08X - (sc->debug_pg_mbuf_alloc) page mbufs allocated\n",
9788                 sc->debug_pg_mbuf_alloc);
9789
9790         BCE_PRINTF("         0x%08X - (sc->free_pg_bd) free page rx_bd's\n",
9791                 sc->free_pg_bd);
9792
9793         BCE_PRINTF("0x%08X/%08X - (sc->pg_low_watermark) page low watermark\n",
9794                 sc->pg_low_watermark, sc->max_pg_bd);
9795 #endif
9796
9797         BCE_PRINTF("         0x%08X - (sc->mbuf_alloc_failed_count) "
9798                 "mbuf alloc failures\n",
9799                 sc->mbuf_alloc_failed_count);
9800
9801         BCE_PRINTF("         0x%08X - (sc->bce_flags) bce mac flags\n",
9802                 sc->bce_flags);
9803
9804         BCE_PRINTF("         0x%08X - (sc->bce_phy_flags) bce phy flags\n",
9805                 sc->bce_phy_flags);
9806
9807         BCE_PRINTF(
9808                 "----------------------------"
9809                 "----------------"
9810                 "----------------------------\n");
9811 }
9812
9813
9814 /****************************************************************************/
9815 /* Prints out the hardware state through a summary of important register,   */
9816 /* followed by a complete register dump.                                    */
9817 /*                                                                          */
9818 /* Returns:                                                                 */
9819 /*   Nothing.                                                               */
9820 /****************************************************************************/
9821 static __attribute__ ((noinline)) void
9822 bce_dump_hw_state(struct bce_softc *sc)
9823 {
9824         u32 val;
9825
9826         BCE_PRINTF(
9827                 "----------------------------"
9828                 " Hardware State "
9829                 "----------------------------\n");
9830
9831         BCE_PRINTF("0x%08X - bootcode version\n", sc->bce_bc_ver);
9832
9833         val = REG_RD(sc, BCE_MISC_ENABLE_STATUS_BITS);
9834         BCE_PRINTF("0x%08X - (0x%06X) misc_enable_status_bits\n",
9835                 val, BCE_MISC_ENABLE_STATUS_BITS);
9836
9837         val = REG_RD(sc, BCE_DMA_STATUS);
9838         BCE_PRINTF("0x%08X - (0x%06X) dma_status\n", val, BCE_DMA_STATUS);
9839
9840         val = REG_RD(sc, BCE_CTX_STATUS);
9841         BCE_PRINTF("0x%08X - (0x%06X) ctx_status\n", val, BCE_CTX_STATUS);
9842
9843         val = REG_RD(sc, BCE_EMAC_STATUS);
9844         BCE_PRINTF("0x%08X - (0x%06X) emac_status\n", val, BCE_EMAC_STATUS);
9845
9846         val = REG_RD(sc, BCE_RPM_STATUS);
9847         BCE_PRINTF("0x%08X - (0x%06X) rpm_status\n", val, BCE_RPM_STATUS);
9848
9849         val = REG_RD(sc, 0x2004);
9850         BCE_PRINTF("0x%08X - (0x%06X) rlup_status\n", val, 0x2004);
9851
9852         val = REG_RD(sc, BCE_RV2P_STATUS);
9853         BCE_PRINTF("0x%08X - (0x%06X) rv2p_status\n", val, BCE_RV2P_STATUS);
9854
9855         val = REG_RD(sc, 0x2c04);
9856         BCE_PRINTF("0x%08X - (0x%06X) rdma_status\n", val, 0x2c04);
9857
9858         val = REG_RD(sc, BCE_TBDR_STATUS);
9859         BCE_PRINTF("0x%08X - (0x%06X) tbdr_status\n", val, BCE_TBDR_STATUS);
9860
9861         val = REG_RD(sc, BCE_TDMA_STATUS);
9862         BCE_PRINTF("0x%08X - (0x%06X) tdma_status\n", val, BCE_TDMA_STATUS);
9863
9864         val = REG_RD(sc, BCE_HC_STATUS);
9865         BCE_PRINTF("0x%08X - (0x%06X) hc_status\n", val, BCE_HC_STATUS);
9866
9867         val = REG_RD_IND(sc, BCE_TXP_CPU_STATE);
9868         BCE_PRINTF("0x%08X - (0x%06X) txp_cpu_state\n", val, BCE_TXP_CPU_STATE);
9869
9870         val = REG_RD_IND(sc, BCE_TPAT_CPU_STATE);
9871         BCE_PRINTF("0x%08X - (0x%06X) tpat_cpu_state\n", val, BCE_TPAT_CPU_STATE);
9872
9873         val = REG_RD_IND(sc, BCE_RXP_CPU_STATE);
9874         BCE_PRINTF("0x%08X - (0x%06X) rxp_cpu_state\n", val, BCE_RXP_CPU_STATE);
9875
9876         val = REG_RD_IND(sc, BCE_COM_CPU_STATE);
9877         BCE_PRINTF("0x%08X - (0x%06X) com_cpu_state\n", val, BCE_COM_CPU_STATE);
9878
9879         val = REG_RD_IND(sc, BCE_MCP_CPU_STATE);
9880         BCE_PRINTF("0x%08X - (0x%06X) mcp_cpu_state\n", val, BCE_MCP_CPU_STATE);
9881
9882         val = REG_RD_IND(sc, BCE_CP_CPU_STATE);
9883         BCE_PRINTF("0x%08X - (0x%06X) cp_cpu_state\n", val, BCE_CP_CPU_STATE);
9884
9885         BCE_PRINTF(
9886                 "----------------------------"
9887                 "----------------"
9888                 "----------------------------\n");
9889
9890         BCE_PRINTF(
9891                 "----------------------------"
9892                 " Register  Dump "
9893                 "----------------------------\n");
9894
9895         for (int i = 0x400; i < 0x8000; i += 0x10) {
9896                 BCE_PRINTF("0x%04X: 0x%08X 0x%08X 0x%08X 0x%08X\n",
9897                         i, REG_RD(sc, i), REG_RD(sc, i + 0x4),
9898                         REG_RD(sc, i + 0x8), REG_RD(sc, i + 0xC));
9899         }
9900
9901         BCE_PRINTF(
9902                 "----------------------------"
9903                 "----------------"
9904                 "----------------------------\n");
9905 }
9906
9907
9908 /****************************************************************************/
9909 /* Prints out the mailbox queue registers.                                  */
9910 /*                                                                          */
9911 /* Returns:                                                                 */
9912 /*   Nothing.                                                               */
9913 /****************************************************************************/
9914 static __attribute__ ((noinline)) void
9915 bce_dump_mq_regs(struct bce_softc *sc)
9916 {
9917         BCE_PRINTF(
9918                 "----------------------------"
9919                 "    MQ Regs     "
9920                 "----------------------------\n");
9921
9922         BCE_PRINTF(
9923                 "----------------------------"
9924                 "----------------"
9925                 "----------------------------\n");
9926
9927         for (int i = 0x3c00; i < 0x4000; i += 0x10) {
9928                 BCE_PRINTF("0x%04X: 0x%08X 0x%08X 0x%08X 0x%08X\n",
9929                         i, REG_RD(sc, i), REG_RD(sc, i + 0x4),
9930                         REG_RD(sc, i + 0x8), REG_RD(sc, i + 0xC));
9931         }
9932
9933         BCE_PRINTF(
9934                 "----------------------------"
9935                 "----------------"
9936                 "----------------------------\n");
9937 }
9938
9939
9940 /****************************************************************************/
9941 /* Prints out the bootcode state.                                           */
9942 /*                                                                          */
9943 /* Returns:                                                                 */
9944 /*   Nothing.                                                               */
9945 /****************************************************************************/
9946 static __attribute__ ((noinline)) void
9947 bce_dump_bc_state(struct bce_softc *sc)
9948 {
9949         u32 val;
9950
9951         BCE_PRINTF(
9952                 "----------------------------"
9953                 " Bootcode State "
9954                 "----------------------------\n");
9955
9956         BCE_PRINTF("0x%08X - bootcode version\n", sc->bce_bc_ver);
9957
9958         val = REG_RD_IND(sc, sc->bce_shmem_base + BCE_BC_RESET_TYPE);
9959         BCE_PRINTF("0x%08X - (0x%06X) reset_type\n",
9960                 val, BCE_BC_RESET_TYPE);
9961
9962         val = REG_RD_IND(sc, sc->bce_shmem_base + BCE_BC_STATE);
9963         BCE_PRINTF("0x%08X - (0x%06X) state\n",
9964                 val, BCE_BC_STATE);
9965
9966         val = REG_RD_IND(sc, sc->bce_shmem_base + BCE_BC_CONDITION);
9967         BCE_PRINTF("0x%08X - (0x%06X) condition\n",
9968                 val, BCE_BC_CONDITION);
9969
9970         val = REG_RD_IND(sc, sc->bce_shmem_base + BCE_BC_STATE_DEBUG_CMD);
9971         BCE_PRINTF("0x%08X - (0x%06X) debug_cmd\n",
9972                 val, BCE_BC_STATE_DEBUG_CMD);
9973
9974         BCE_PRINTF(
9975                 "----------------------------"
9976                 "----------------"
9977                 "----------------------------\n");
9978 }
9979
9980
9981 /****************************************************************************/
9982 /* Prints out the TXP processor state.                                      */
9983 /*                                                                          */
9984 /* Returns:                                                                 */
9985 /*   Nothing.                                                               */
9986 /****************************************************************************/
9987 static __attribute__ ((noinline)) void
9988 bce_dump_txp_state(struct bce_softc *sc, int regs)
9989 {
9990         u32 val;
9991         u32 fw_version[3];
9992
9993         BCE_PRINTF(
9994                 "----------------------------"
9995                 "   TXP  State   "
9996                 "----------------------------\n");
9997
9998         for (int i = 0; i < 3; i++)
9999                 fw_version[i] = htonl(REG_RD_IND(sc,
10000                         (BCE_TXP_SCRATCH + 0x10 + i * 4)));
10001         BCE_PRINTF("Firmware version - %s\n", (char *) fw_version);
10002
10003         val = REG_RD_IND(sc, BCE_TXP_CPU_MODE);
10004         BCE_PRINTF("0x%08X - (0x%06X) txp_cpu_mode\n", val, BCE_TXP_CPU_MODE);
10005
10006         val = REG_RD_IND(sc, BCE_TXP_CPU_STATE);
10007         BCE_PRINTF("0x%08X - (0x%06X) txp_cpu_state\n", val, BCE_TXP_CPU_STATE);
10008
10009         val = REG_RD_IND(sc, BCE_TXP_CPU_EVENT_MASK);
10010         BCE_PRINTF("0x%08X - (0x%06X) txp_cpu_event_mask\n", val,
10011                 BCE_TXP_CPU_EVENT_MASK);
10012
10013         if (regs) {
10014                 BCE_PRINTF(
10015                         "----------------------------"
10016                         " Register  Dump "
10017                         "----------------------------\n");
10018
10019                 for (int i = BCE_TXP_CPU_MODE; i < 0x68000; i += 0x10) {
10020                         /* Skip the big blank spaces */
10021                         if (i < 0x454000 && i > 0x5ffff)
10022                                 BCE_PRINTF("0x%04X: 0x%08X 0x%08X 0x%08X 0x%08X\n",
10023                                         i, REG_RD_IND(sc, i), REG_RD_IND(sc, i + 0x4),
10024                                         REG_RD_IND(sc, i + 0x8), REG_RD_IND(sc, i + 0xC));
10025                 }
10026         }
10027
10028         BCE_PRINTF(
10029                 "----------------------------"
10030                 "----------------"
10031                 "----------------------------\n");
10032 }
10033
10034
10035 /****************************************************************************/
10036 /* Prints out the RXP processor state.                                      */
10037 /*                                                                          */
10038 /* Returns:                                                                 */
10039 /*   Nothing.                                                               */
10040 /****************************************************************************/
10041 static __attribute__ ((noinline)) void
10042 bce_dump_rxp_state(struct bce_softc *sc, int regs)
10043 {
10044         u32 val;
10045         u32 fw_version[3];
10046
10047         BCE_PRINTF(
10048                 "----------------------------"
10049                 "   RXP  State   "
10050                 "----------------------------\n");
10051
10052         for (int i = 0; i < 3; i++)
10053                 fw_version[i] = htonl(REG_RD_IND(sc,
10054                         (BCE_RXP_SCRATCH + 0x10 + i * 4)));
10055         BCE_PRINTF("Firmware version - %s\n", (char *) fw_version);
10056
10057         val = REG_RD_IND(sc, BCE_RXP_CPU_MODE);
10058         BCE_PRINTF("0x%08X - (0x%06X) rxp_cpu_mode\n", val, BCE_RXP_CPU_MODE);
10059
10060         val = REG_RD_IND(sc, BCE_RXP_CPU_STATE);
10061         BCE_PRINTF("0x%08X - (0x%06X) rxp_cpu_state\n", val, BCE_RXP_CPU_STATE);
10062
10063         val = REG_RD_IND(sc, BCE_RXP_CPU_EVENT_MASK);
10064         BCE_PRINTF("0x%08X - (0x%06X) rxp_cpu_event_mask\n", val,
10065                 BCE_RXP_CPU_EVENT_MASK);
10066
10067         if (regs) {
10068                 BCE_PRINTF(
10069                         "----------------------------"
10070                         " Register  Dump "
10071                         "----------------------------\n");
10072
10073                 for (int i = BCE_RXP_CPU_MODE; i < 0xe8fff; i += 0x10) {
10074                         /* Skip the big blank sapces */
10075                         if (i < 0xc5400 && i > 0xdffff)
10076                                 BCE_PRINTF("0x%04X: 0x%08X 0x%08X 0x%08X 0x%08X\n",
10077                                         i, REG_RD_IND(sc, i), REG_RD_IND(sc, i + 0x4),
10078                                         REG_RD_IND(sc, i + 0x8), REG_RD_IND(sc, i + 0xC));
10079                 }
10080         }
10081
10082         BCE_PRINTF(
10083                 "----------------------------"
10084                 "----------------"
10085                 "----------------------------\n");
10086 }
10087
10088
10089 /****************************************************************************/
10090 /* Prints out the TPAT processor state.                                     */
10091 /*                                                                          */
10092 /* Returns:                                                                 */
10093 /*   Nothing.                                                               */
10094 /****************************************************************************/
10095 static __attribute__ ((noinline)) void
10096 bce_dump_tpat_state(struct bce_softc *sc, int regs)
10097 {
10098         u32 val;
10099         u32 fw_version[3];
10100
10101         BCE_PRINTF(
10102                 "----------------------------"
10103                 "   TPAT State   "
10104                 "----------------------------\n");
10105
10106         for (int i = 0; i < 3; i++)
10107                 fw_version[i] = htonl(REG_RD_IND(sc,
10108                         (BCE_TPAT_SCRATCH + 0x410 + i * 4)));
10109         BCE_PRINTF("Firmware version - %s\n", (char *) fw_version);
10110
10111         val = REG_RD_IND(sc, BCE_TPAT_CPU_MODE);
10112         BCE_PRINTF("0x%08X - (0x%06X) tpat_cpu_mode\n", val, BCE_TPAT_CPU_MODE);
10113
10114         val = REG_RD_IND(sc, BCE_TPAT_CPU_STATE);
10115         BCE_PRINTF("0x%08X - (0x%06X) tpat_cpu_state\n", val, BCE_TPAT_CPU_STATE);
10116
10117         val = REG_RD_IND(sc, BCE_TPAT_CPU_EVENT_MASK);
10118         BCE_PRINTF("0x%08X - (0x%06X) tpat_cpu_event_mask\n", val,
10119                 BCE_TPAT_CPU_EVENT_MASK);
10120
10121         if (regs) {
10122                 BCE_PRINTF(
10123                         "----------------------------"
10124                         " Register  Dump "
10125                         "----------------------------\n");
10126
10127                 for (int i = BCE_TPAT_CPU_MODE; i < 0xa3fff; i += 0x10) {
10128                         /* Skip the big blank spaces */
10129                         if (i < 0x854000 && i > 0x9ffff)
10130                                 BCE_PRINTF("0x%04X: 0x%08X 0x%08X 0x%08X 0x%08X\n",
10131                                         i, REG_RD_IND(sc, i), REG_RD_IND(sc, i + 0x4),
10132                                         REG_RD_IND(sc, i + 0x8), REG_RD_IND(sc, i + 0xC));
10133                 }
10134         }
10135
10136         BCE_PRINTF(
10137                 "----------------------------"
10138                 "----------------"
10139                 "----------------------------\n");
10140 }
10141
10142
10143 /****************************************************************************/
10144 /* Prints out the Command Procesor (CP) state.                              */
10145 /*                                                                          */
10146 /* Returns:                                                                 */
10147 /*   Nothing.                                                               */
10148 /****************************************************************************/
10149 static __attribute__ ((noinline)) void
10150 bce_dump_cp_state(struct bce_softc *sc, int regs)
10151 {
10152         u32 val;
10153         u32 fw_version[3];
10154
10155         BCE_PRINTF(
10156                 "----------------------------"
10157                 "    CP State    "
10158                 "----------------------------\n");
10159
10160         for (int i = 0; i < 3; i++)
10161                 fw_version[i] = htonl(REG_RD_IND(sc,
10162                         (BCE_CP_SCRATCH + 0x10 + i * 4)));
10163         BCE_PRINTF("Firmware version - %s\n", (char *) fw_version);
10164
10165         val = REG_RD_IND(sc, BCE_CP_CPU_MODE);
10166         BCE_PRINTF("0x%08X - (0x%06X) cp_cpu_mode\n", val, BCE_CP_CPU_MODE);
10167
10168         val = REG_RD_IND(sc, BCE_CP_CPU_STATE);
10169         BCE_PRINTF("0x%08X - (0x%06X) cp_cpu_state\n", val, BCE_CP_CPU_STATE);
10170
10171         val = REG_RD_IND(sc, BCE_CP_CPU_EVENT_MASK);
10172         BCE_PRINTF("0x%08X - (0x%06X) cp_cpu_event_mask\n", val,
10173                 BCE_CP_CPU_EVENT_MASK);
10174
10175         if (regs) {
10176                 BCE_PRINTF(
10177                         "----------------------------"
10178                         " Register  Dump "
10179                         "----------------------------\n");
10180
10181                 for (int i = BCE_CP_CPU_MODE; i < 0x1aa000; i += 0x10) {
10182                         /* Skip the big blank spaces */
10183                         if (i < 0x185400 && i > 0x19ffff)
10184                                 BCE_PRINTF("0x%04X: 0x%08X 0x%08X 0x%08X 0x%08X\n",
10185                                         i, REG_RD_IND(sc, i), REG_RD_IND(sc, i + 0x4),
10186                                         REG_RD_IND(sc, i + 0x8), REG_RD_IND(sc, i + 0xC));
10187                 }
10188         }
10189
10190         BCE_PRINTF(
10191                 "----------------------------"
10192                 "----------------"
10193                 "----------------------------\n");
10194 }
10195
10196
10197 /****************************************************************************/
10198 /* Prints out the Completion Procesor (COM) state.                          */
10199 /*                                                                          */
10200 /* Returns:                                                                 */
10201 /*   Nothing.                                                               */
10202 /****************************************************************************/
10203 static __attribute__ ((noinline)) void
10204 bce_dump_com_state(struct bce_softc *sc, int regs)
10205 {
10206         u32 val;
10207         u32 fw_version[3];
10208
10209         BCE_PRINTF(
10210                 "----------------------------"
10211                 "   COM State    "
10212                 "----------------------------\n");
10213
10214         for (int i = 0; i < 3; i++)
10215                 fw_version[i] = htonl(REG_RD_IND(sc,
10216                         (BCE_COM_SCRATCH + 0x10 + i * 4)));
10217         BCE_PRINTF("Firmware version - %s\n", (char *) fw_version);
10218
10219         val = REG_RD_IND(sc, BCE_COM_CPU_MODE);
10220         BCE_PRINTF("0x%08X - (0x%06X) com_cpu_mode\n", val, BCE_COM_CPU_MODE);
10221
10222         val = REG_RD_IND(sc, BCE_COM_CPU_STATE);
10223         BCE_PRINTF("0x%08X - (0x%06X) com_cpu_state\n", val, BCE_COM_CPU_STATE);
10224
10225         val = REG_RD_IND(sc, BCE_COM_CPU_EVENT_MASK);
10226         BCE_PRINTF("0x%08X - (0x%06X) com_cpu_event_mask\n", val,
10227                 BCE_COM_CPU_EVENT_MASK);
10228
10229         if (regs) {
10230                 BCE_PRINTF(
10231                         "----------------------------"
10232                         " Register  Dump "
10233                         "----------------------------\n");
10234
10235                 for (int i = BCE_COM_CPU_MODE; i < 0x1053e8; i += 0x10) {
10236                         BCE_PRINTF("0x%04X: 0x%08X 0x%08X 0x%08X 0x%08X\n",
10237                                 i, REG_RD_IND(sc, i), REG_RD_IND(sc, i + 0x4),
10238                                 REG_RD_IND(sc, i + 0x8), REG_RD_IND(sc, i + 0xC));
10239                 }
10240         }
10241
10242         BCE_PRINTF(
10243                 "----------------------------"
10244                 "----------------"
10245                 "----------------------------\n");
10246 }
10247
10248
10249 /****************************************************************************/
10250 /* Prints out the driver state and then enters the debugger.                */
10251 /*                                                                          */
10252 /* Returns:                                                                 */
10253 /*   Nothing.                                                               */
10254 /****************************************************************************/
10255 static void
10256 bce_breakpoint(struct bce_softc *sc)
10257 {
10258
10259         /*
10260          * Unreachable code to silence compiler warnings
10261          * about unused functions.
10262          */
10263         if (0) {
10264                 bce_freeze_controller(sc);
10265                 bce_unfreeze_controller(sc);
10266                 bce_dump_enet(sc, NULL);
10267                 bce_dump_txbd(sc, 0, NULL);
10268                 bce_dump_rxbd(sc, 0, NULL);
10269                 bce_dump_tx_mbuf_chain(sc, 0, USABLE_TX_BD);
10270                 bce_dump_rx_mbuf_chain(sc, 0, USABLE_RX_BD);
10271                 bce_dump_l2fhdr(sc, 0, NULL);
10272                 bce_dump_ctx(sc, RX_CID);
10273                 bce_dump_ftqs(sc);
10274                 bce_dump_tx_chain(sc, 0, USABLE_TX_BD);
10275                 bce_dump_rx_chain(sc, 0, USABLE_RX_BD);
10276                 bce_dump_status_block(sc);
10277                 bce_dump_stats_block(sc);
10278                 bce_dump_driver_state(sc);
10279                 bce_dump_hw_state(sc);
10280                 bce_dump_bc_state(sc);
10281                 bce_dump_txp_state(sc, 0);
10282                 bce_dump_rxp_state(sc, 0);
10283                 bce_dump_tpat_state(sc, 0);
10284                 bce_dump_cp_state(sc, 0);
10285                 bce_dump_com_state(sc, 0);
10286 #ifdef ZERO_COPY_SOCKETS
10287                 bce_dump_pgbd(sc, 0, NULL);
10288                 bce_dump_pg_mbuf_chain(sc, 0, USABLE_PG_BD);
10289                 bce_dump_pg_chain(sc, 0, USABLE_PG_BD);
10290 #endif
10291         }
10292
10293         bce_dump_status_block(sc);
10294         bce_dump_driver_state(sc);
10295
10296         /* Call the debugger. */
10297         breakpoint();
10298
10299         return;
10300 }
10301 #endif
10302