sys/dev/bce/if_bce.c

   1 /*-
   2  * Copyright (c) 2006-2009 Broadcom Corporation
   3  *      David Christensen <davidch@broadcom.com>.  All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  *
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  * 3. Neither the name of Broadcom Corporation nor the name of its contributors
  15  *    may be used to endorse or promote products derived from this software
  16  *    without specific prior written consent.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS'
  19  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
  22  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  23  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  24  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  25  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  26  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  27  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  28  * THE POSSIBILITY OF SUCH DAMAGE.
  29  */
  30
  31 #include <sys/cdefs.h>
  32 __FBSDID("$FreeBSD$");
  33
  34 /*
  35  * The following controllers are supported by this driver:
  36  *   BCM5706C A2, A3
  37  *   BCM5706S A2, A3
  38  *   BCM5708C B1, B2
  39  *   BCM5708S B1, B2
  40  *   BCM5709C A1, C0
  41  *       BCM5716C C0
  42  *
  43  * The following controllers are not supported by this driver:
  44  *   BCM5706C A0, A1 (pre-production)
  45  *   BCM5706S A0, A1 (pre-production)
  46  *   BCM5708C A0, B0 (pre-production)
  47  *   BCM5708S A0, B0 (pre-production)
  48  *   BCM5709C A0  B0, B1, B2 (pre-production)
  49  *   BCM5709S A0, A1, B0, B1, B2, C0 (pre-production)
  50  */
  51
  52 #include "opt_bce.h"
  53
  54 #include <dev/bce/if_bcereg.h>
  55 #include <dev/bce/if_bcefw.h>
  56
  57 /****************************************************************************/
  58 /* BCE Debug Options                                                        */
  59 /****************************************************************************/
  60 #ifdef BCE_DEBUG
  61         u32 bce_debug = BCE_WARN;
  62
  63         /*          0 = Never              */
  64         /*          1 = 1 in 2,147,483,648 */
  65         /*        256 = 1 in     8,388,608 */
  66         /*       2048 = 1 in     1,048,576 */
  67         /*      65536 = 1 in        32,768 */
  68         /*    1048576 = 1 in         2,048 */
  69         /*  268435456 = 1 in             8 */
  70         /*  536870912 = 1 in             4 */
  71         /* 1073741824 = 1 in             2 */
  72
  73         /* Controls how often the l2_fhdr frame error check will fail. */
  74         int l2fhdr_error_sim_control = 0;
  75
  76         /* Controls how often the unexpected attention check will fail. */
  77         int unexpected_attention_sim_control = 0;
  78
  79         /* Controls how often to simulate an mbuf allocation failure. */
  80         int mbuf_alloc_failed_sim_control = 0;
  81
  82         /* Controls how often to simulate a DMA mapping failure. */
  83         int dma_map_addr_failed_sim_control = 0;
  84
  85         /* Controls how often to simulate a bootcode failure. */
  86         int bootcode_running_failure_sim_control = 0;
  87 #endif
  88
  89 /****************************************************************************/
  90 /* BCE Build Time Options                                                   */
  91 /****************************************************************************/
  92 /* #define BCE_NVRAM_WRITE_SUPPORT 1 */
  93
  94
  95 /****************************************************************************/
  96 /* PCI Device ID Table                                                      */
  97 /*                                                                          */
  98 /* Used by bce_probe() to identify the devices supported by this driver.    */
  99 /****************************************************************************/
 100 #define BCE_DEVDESC_MAX         64
 101
 102 static struct bce_type bce_devs[] = {
 103         /* BCM5706C Controllers and OEM boards. */
 104         { BRCM_VENDORID, BRCM_DEVICEID_BCM5706,  HP_VENDORID, 0x3101,
 105                 "HP NC370T Multifunction Gigabit Server Adapter" },
 106         { BRCM_VENDORID, BRCM_DEVICEID_BCM5706,  HP_VENDORID, 0x3106,
 107                 "HP NC370i Multifunction Gigabit Server Adapter" },
 108         { BRCM_VENDORID, BRCM_DEVICEID_BCM5706,  HP_VENDORID, 0x3070,
 109                 "HP NC380T PCIe DP Multifunc Gig Server Adapter" },
 110         { BRCM_VENDORID, BRCM_DEVICEID_BCM5706,  HP_VENDORID, 0x1709,
 111                 "HP NC371i Multifunction Gigabit Server Adapter" },
 112         { BRCM_VENDORID, BRCM_DEVICEID_BCM5706,  PCI_ANY_ID,  PCI_ANY_ID,
 113                 "Broadcom NetXtreme II BCM5706 1000Base-T" },
 114
 115         /* BCM5706S controllers and OEM boards. */
 116         { BRCM_VENDORID, BRCM_DEVICEID_BCM5706S, HP_VENDORID, 0x3102,
 117                 "HP NC370F Multifunction Gigabit Server Adapter" },
 118         { BRCM_VENDORID, BRCM_DEVICEID_BCM5706S, PCI_ANY_ID,  PCI_ANY_ID,
 119                 "Broadcom NetXtreme II BCM5706 1000Base-SX" },
 120
 121         /* BCM5708C controllers and OEM boards. */
 122         { BRCM_VENDORID, BRCM_DEVICEID_BCM5708,  HP_VENDORID, 0x7037,
 123                 "HP NC373T PCIe Multifunction Gig Server Adapter" },
 124         { BRCM_VENDORID, BRCM_DEVICEID_BCM5708,  HP_VENDORID, 0x7038,
 125                 "HP NC373i Multifunction Gigabit Server Adapter" },
 126         { BRCM_VENDORID, BRCM_DEVICEID_BCM5708,  HP_VENDORID, 0x7045,
 127                 "HP NC374m PCIe Multifunction Adapter" },
 128         { BRCM_VENDORID, BRCM_DEVICEID_BCM5708,  PCI_ANY_ID,  PCI_ANY_ID,
 129                 "Broadcom NetXtreme II BCM5708 1000Base-T" },
 130
 131         /* BCM5708S controllers and OEM boards. */
 132         { BRCM_VENDORID, BRCM_DEVICEID_BCM5708S,  HP_VENDORID, 0x1706,
 133                 "HP NC373m Multifunction Gigabit Server Adapter" },
 134         { BRCM_VENDORID, BRCM_DEVICEID_BCM5708S,  HP_VENDORID, 0x703b,
 135                 "HP NC373i Multifunction Gigabit Server Adapter" },
 136         { BRCM_VENDORID, BRCM_DEVICEID_BCM5708S,  HP_VENDORID, 0x703d,
 137                 "HP NC373F PCIe Multifunc Giga Server Adapter" },
 138         { BRCM_VENDORID, BRCM_DEVICEID_BCM5708S,  PCI_ANY_ID,  PCI_ANY_ID,
 139                 "Broadcom NetXtreme II BCM5708 1000Base-SX" },
 140
 141         /* BCM5709C controllers and OEM boards. */
 142         { BRCM_VENDORID, BRCM_DEVICEID_BCM5709,  HP_VENDORID, 0x7055,
 143                 "HP NC382i DP Multifunction Gigabit Server Adapter" },
 144         { BRCM_VENDORID, BRCM_DEVICEID_BCM5709,  HP_VENDORID, 0x7059,
 145                 "HP NC382T PCIe DP Multifunction Gigabit Server Adapter" },
 146         { BRCM_VENDORID, BRCM_DEVICEID_BCM5709,  PCI_ANY_ID,  PCI_ANY_ID,
 147                 "Broadcom NetXtreme II BCM5709 1000Base-T" },
 148
 149         /* BCM5709S controllers and OEM boards. */
 150         { BRCM_VENDORID, BRCM_DEVICEID_BCM5709S,  HP_VENDORID, 0x171d,
 151                 "HP NC382m DP 1GbE Multifunction BL-c Adapter" },
 152         { BRCM_VENDORID, BRCM_DEVICEID_BCM5709S,  HP_VENDORID, 0x7056,
 153                 "HP NC382i DP Multifunction Gigabit Server Adapter" },
 154         { BRCM_VENDORID, BRCM_DEVICEID_BCM5709S,  PCI_ANY_ID,  PCI_ANY_ID,
 155                 "Broadcom NetXtreme II BCM5709 1000Base-SX" },
 156
 157         /* BCM5716 controllers and OEM boards. */
 158         { BRCM_VENDORID, BRCM_DEVICEID_BCM5716,  PCI_ANY_ID,  PCI_ANY_ID,
 159                 "Broadcom NetXtreme II BCM5716 1000Base-T" },
 160
 161         { 0, 0, 0, 0, NULL }
 162 };
 163
 164
 165 /****************************************************************************/
 166 /* Supported Flash NVRAM device data.                                       */
 167 /****************************************************************************/
 168 static struct flash_spec flash_table[] =
 169 {
 170 #define BUFFERED_FLAGS          (BCE_NV_BUFFERED | BCE_NV_TRANSLATE)
 171 #define NONBUFFERED_FLAGS       (BCE_NV_WREN)
 172
 173         /* Slow EEPROM */
 174         {0x00000000, 0x40830380, 0x009f0081, 0xa184a053, 0xaf000400,
 175          BUFFERED_FLAGS, SEEPROM_PAGE_BITS, SEEPROM_PAGE_SIZE,
 176          SEEPROM_BYTE_ADDR_MASK, SEEPROM_TOTAL_SIZE,
 177          "EEPROM - slow"},
 178         /* Expansion entry 0001 */
 179         {0x08000002, 0x4b808201, 0x00050081, 0x03840253, 0xaf020406,
 180          NONBUFFERED_FLAGS, SAIFUN_FLASH_PAGE_BITS, SAIFUN_FLASH_PAGE_SIZE,
 181          SAIFUN_FLASH_BYTE_ADDR_MASK, 0,
 182          "Entry 0001"},
 183         /* Saifun SA25F010 (non-buffered flash) */
 184         /* strap, cfg1, & write1 need updates */
 185         {0x04000001, 0x47808201, 0x00050081, 0x03840253, 0xaf020406,
 186          NONBUFFERED_FLAGS, SAIFUN_FLASH_PAGE_BITS, SAIFUN_FLASH_PAGE_SIZE,
 187          SAIFUN_FLASH_BYTE_ADDR_MASK, SAIFUN_FLASH_BASE_TOTAL_SIZE*2,
 188          "Non-buffered flash (128kB)"},
 189         /* Saifun SA25F020 (non-buffered flash) */
 190         /* strap, cfg1, & write1 need updates */
 191         {0x0c000003, 0x4f808201, 0x00050081, 0x03840253, 0xaf020406,
 192          NONBUFFERED_FLAGS, SAIFUN_FLASH_PAGE_BITS, SAIFUN_FLASH_PAGE_SIZE,
 193          SAIFUN_FLASH_BYTE_ADDR_MASK, SAIFUN_FLASH_BASE_TOTAL_SIZE*4,
 194          "Non-buffered flash (256kB)"},
 195         /* Expansion entry 0100 */
 196         {0x11000000, 0x53808201, 0x00050081, 0x03840253, 0xaf020406,
 197          NONBUFFERED_FLAGS, SAIFUN_FLASH_PAGE_BITS, SAIFUN_FLASH_PAGE_SIZE,
 198          SAIFUN_FLASH_BYTE_ADDR_MASK, 0,
 199          "Entry 0100"},
 200         /* Entry 0101: ST M45PE10 (non-buffered flash, TetonII B0) */
 201         {0x19000002, 0x5b808201, 0x000500db, 0x03840253, 0xaf020406,
 202          NONBUFFERED_FLAGS, ST_MICRO_FLASH_PAGE_BITS, ST_MICRO_FLASH_PAGE_SIZE,
 203          ST_MICRO_FLASH_BYTE_ADDR_MASK, ST_MICRO_FLASH_BASE_TOTAL_SIZE*2,
 204          "Entry 0101: ST M45PE10 (128kB non-bufferred)"},
 205         /* Entry 0110: ST M45PE20 (non-buffered flash)*/
 206         {0x15000001, 0x57808201, 0x000500db, 0x03840253, 0xaf020406,
 207          NONBUFFERED_FLAGS, ST_MICRO_FLASH_PAGE_BITS, ST_MICRO_FLASH_PAGE_SIZE,
 208          ST_MICRO_FLASH_BYTE_ADDR_MASK, ST_MICRO_FLASH_BASE_TOTAL_SIZE*4,
 209          "Entry 0110: ST M45PE20 (256kB non-bufferred)"},
 210         /* Saifun SA25F005 (non-buffered flash) */
 211         /* strap, cfg1, & write1 need updates */
 212         {0x1d000003, 0x5f808201, 0x00050081, 0x03840253, 0xaf020406,
 213          NONBUFFERED_FLAGS, SAIFUN_FLASH_PAGE_BITS, SAIFUN_FLASH_PAGE_SIZE,
 214          SAIFUN_FLASH_BYTE_ADDR_MASK, SAIFUN_FLASH_BASE_TOTAL_SIZE,
 215          "Non-buffered flash (64kB)"},
 216         /* Fast EEPROM */
 217         {0x22000000, 0x62808380, 0x009f0081, 0xa184a053, 0xaf000400,
 218          BUFFERED_FLAGS, SEEPROM_PAGE_BITS, SEEPROM_PAGE_SIZE,
 219          SEEPROM_BYTE_ADDR_MASK, SEEPROM_TOTAL_SIZE,
 220          "EEPROM - fast"},
 221         /* Expansion entry 1001 */
 222         {0x2a000002, 0x6b808201, 0x00050081, 0x03840253, 0xaf020406,
 223          NONBUFFERED_FLAGS, SAIFUN_FLASH_PAGE_BITS, SAIFUN_FLASH_PAGE_SIZE,
 224          SAIFUN_FLASH_BYTE_ADDR_MASK, 0,
 225          "Entry 1001"},
 226         /* Expansion entry 1010 */
 227         {0x26000001, 0x67808201, 0x00050081, 0x03840253, 0xaf020406,
 228          NONBUFFERED_FLAGS, SAIFUN_FLASH_PAGE_BITS, SAIFUN_FLASH_PAGE_SIZE,
 229          SAIFUN_FLASH_BYTE_ADDR_MASK, 0,
 230          "Entry 1010"},
 231         /* ATMEL AT45DB011B (buffered flash) */
 232         {0x2e000003, 0x6e808273, 0x00570081, 0x68848353, 0xaf000400,
 233          BUFFERED_FLAGS, BUFFERED_FLASH_PAGE_BITS, BUFFERED_FLASH_PAGE_SIZE,
 234          BUFFERED_FLASH_BYTE_ADDR_MASK, BUFFERED_FLASH_TOTAL_SIZE,
 235          "Buffered flash (128kB)"},
 236         /* Expansion entry 1100 */
 237         {0x33000000, 0x73808201, 0x00050081, 0x03840253, 0xaf020406,
 238          NONBUFFERED_FLAGS, SAIFUN_FLASH_PAGE_BITS, SAIFUN_FLASH_PAGE_SIZE,
 239          SAIFUN_FLASH_BYTE_ADDR_MASK, 0,
 240          "Entry 1100"},
 241         /* Expansion entry 1101 */
 242         {0x3b000002, 0x7b808201, 0x00050081, 0x03840253, 0xaf020406,
 243          NONBUFFERED_FLAGS, SAIFUN_FLASH_PAGE_BITS, SAIFUN_FLASH_PAGE_SIZE,
 244          SAIFUN_FLASH_BYTE_ADDR_MASK, 0,
 245          "Entry 1101"},
 246         /* Ateml Expansion entry 1110 */
 247         {0x37000001, 0x76808273, 0x00570081, 0x68848353, 0xaf000400,
 248          BUFFERED_FLAGS, BUFFERED_FLASH_PAGE_BITS, BUFFERED_FLASH_PAGE_SIZE,
 249          BUFFERED_FLASH_BYTE_ADDR_MASK, 0,
 250          "Entry 1110 (Atmel)"},
 251         /* ATMEL AT45DB021B (buffered flash) */
 252         {0x3f000003, 0x7e808273, 0x00570081, 0x68848353, 0xaf000400,
 253          BUFFERED_FLAGS, BUFFERED_FLASH_PAGE_BITS, BUFFERED_FLASH_PAGE_SIZE,
 254          BUFFERED_FLASH_BYTE_ADDR_MASK, BUFFERED_FLASH_TOTAL_SIZE*2,
 255          "Buffered flash (256kB)"},
 256 };
 257
 258 /*
 259  * The BCM5709 controllers transparently handle the
 260  * differences between Atmel 264 byte pages and all
 261  * flash devices which use 256 byte pages, so no
 262  * logical-to-physical mapping is required in the
 263  * driver.
 264  */
 265 static struct flash_spec flash_5709 = {
 266         .flags          = BCE_NV_BUFFERED,
 267         .page_bits      = BCM5709_FLASH_PAGE_BITS,
 268         .page_size      = BCM5709_FLASH_PAGE_SIZE,
 269         .addr_mask      = BCM5709_FLASH_BYTE_ADDR_MASK,
 270         .total_size     = BUFFERED_FLASH_TOTAL_SIZE * 2,
 271         .name           = "5709/5716 buffered flash (256kB)",
 272 };
 273
 274
 275 /****************************************************************************/
 276 /* FreeBSD device entry points.                                             */
 277 /****************************************************************************/
 278 static int  bce_probe                           (device_t);
 279 static int  bce_attach                          (device_t);
 280 static int  bce_detach                          (device_t);
 281 static int  bce_shutdown                        (device_t);
 282
 283
 284 /****************************************************************************/
 285 /* BCE Debug Data Structure Dump Routines                                   */
 286 /****************************************************************************/
 287 #ifdef BCE_DEBUG
 288 static u32      bce_reg_rd                              (struct bce_softc *, u32);
 289 static void     bce_reg_wr                              (struct bce_softc *, u32, u32);
 290 static void     bce_reg_wr16                    (struct bce_softc *, u32, u16);
 291 static u32  bce_ctx_rd                          (struct bce_softc *, u32, u32);
 292 static void bce_dump_enet           (struct bce_softc *, struct mbuf *);
 293 static void bce_dump_mbuf                       (struct bce_softc *, struct mbuf *);
 294 static void bce_dump_tx_mbuf_chain      (struct bce_softc *, u16, int);
 295 static void bce_dump_rx_mbuf_chain      (struct bce_softc *, u16, int);
 296 #ifdef ZERO_COPY_SOCKETS
 297 static void bce_dump_pg_mbuf_chain      (struct bce_softc *, u16, int);
 298 #endif
 299 static void bce_dump_txbd                       (struct bce_softc *, int, struct tx_bd *);
 300 static void bce_dump_rxbd                       (struct bce_softc *, int, struct rx_bd *);
 301 #ifdef ZERO_COPY_SOCKETS
 302 static void bce_dump_pgbd                       (struct bce_softc *, int, struct rx_bd *);
 303 #endif
 304 static void bce_dump_l2fhdr                     (struct bce_softc *, int, struct l2_fhdr *);
 305 static void bce_dump_ctx                        (struct bce_softc *, u16);
 306 static void bce_dump_ftqs                       (struct bce_softc *);
 307 static void bce_dump_tx_chain           (struct bce_softc *, u16, int);
 308 static void bce_dump_rx_chain           (struct bce_softc *, u16, int);
 309 #ifdef ZERO_COPY_SOCKETS
 310 static void bce_dump_pg_chain           (struct bce_softc *, u16, int);
 311 #endif
 312 static void bce_dump_status_block       (struct bce_softc *);
 313 static void bce_dump_stats_block        (struct bce_softc *);
 314 static void bce_dump_driver_state       (struct bce_softc *);
 315 static void bce_dump_hw_state           (struct bce_softc *);
 316 static void bce_dump_mq_regs        (struct bce_softc *);
 317 static void bce_dump_bc_state           (struct bce_softc *);
 318 static void bce_dump_txp_state          (struct bce_softc *, int);
 319 static void bce_dump_rxp_state          (struct bce_softc *, int);
 320 static void bce_dump_tpat_state         (struct bce_softc *, int);
 321 static void bce_dump_cp_state           (struct bce_softc *, int);
 322 static void bce_dump_com_state          (struct bce_softc *, int);
 323 static void bce_breakpoint                      (struct bce_softc *);
 324 #endif
 325
 326
 327 /****************************************************************************/
 328 /* BCE Register/Memory Access Routines                                      */
 329 /****************************************************************************/
 330 static u32  bce_reg_rd_ind                      (struct bce_softc *, u32);
 331 static void bce_reg_wr_ind                      (struct bce_softc *, u32, u32);
 332 static void bce_ctx_wr                          (struct bce_softc *, u32, u32, u32);
 333 static int  bce_miibus_read_reg         (device_t, int, int);
 334 static int  bce_miibus_write_reg        (device_t, int, int, int);
 335 static void bce_miibus_statchg          (device_t);
 336
 337
 338 /****************************************************************************/
 339 /* BCE NVRAM Access Routines                                                */
 340 /****************************************************************************/
 341 static int  bce_acquire_nvram_lock      (struct bce_softc *);
 342 static int  bce_release_nvram_lock      (struct bce_softc *);
 343 static void bce_enable_nvram_access     (struct bce_softc *);
 344 static void     bce_disable_nvram_access(struct bce_softc *);
 345 static int  bce_nvram_read_dword        (struct bce_softc *, u32, u8 *, u32);
 346 static int  bce_init_nvram                      (struct bce_softc *);
 347 static int  bce_nvram_read                      (struct bce_softc *, u32, u8 *, int);
 348 static int  bce_nvram_test                      (struct bce_softc *);
 349 #ifdef BCE_NVRAM_WRITE_SUPPORT
 350 static int  bce_enable_nvram_write      (struct bce_softc *);
 351 static void bce_disable_nvram_write     (struct bce_softc *);
 352 static int  bce_nvram_erase_page        (struct bce_softc *, u32);
 353 static int  bce_nvram_write_dword       (struct bce_softc *, u32, u8 *, u32);
 354 static int  bce_nvram_write                     (struct bce_softc *, u32, u8 *, int);
 355 #endif
 356
 357 /****************************************************************************/
 358 /*                                                                          */
 359 /****************************************************************************/
 360 static void bce_get_media                       (struct bce_softc *);
 361 static void bce_dma_map_addr            (void *, bus_dma_segment_t *, int, int);
 362 static int  bce_dma_alloc                       (device_t);
 363 static void bce_dma_free                        (struct bce_softc *);
 364 static void bce_release_resources       (struct bce_softc *);
 365
 366 /****************************************************************************/
 367 /* BCE Firmware Synchronization and Load                                    */
 368 /****************************************************************************/
 369 static int  bce_fw_sync                         (struct bce_softc *, u32);
 370 static void bce_load_rv2p_fw            (struct bce_softc *, u32 *, u32, u32);
 371 static void bce_load_cpu_fw                     (struct bce_softc *, struct cpu_reg *, struct fw_info *);
 372 static void bce_init_rxp_cpu            (struct bce_softc *);
 373 static void bce_init_txp_cpu            (struct bce_softc *);
 374 static void bce_init_tpat_cpu           (struct bce_softc *);
 375 static void bce_init_cp_cpu                     (struct bce_softc *);
 376 static void bce_init_com_cpu            (struct bce_softc *);
 377 static void bce_init_cpus                       (struct bce_softc *);
 378
 379 static void     bce_print_adapter_info  (struct bce_softc *);
 380 static void bce_probe_pci_caps          (device_t, struct bce_softc *);
 381 static void bce_stop                            (struct bce_softc *);
 382 static int  bce_reset                           (struct bce_softc *, u32);
 383 static int  bce_chipinit                        (struct bce_softc *);
 384 static int  bce_blockinit                       (struct bce_softc *);
 385
 386 static int  bce_init_tx_chain           (struct bce_softc *);
 387 static void bce_free_tx_chain           (struct bce_softc *);
 388
 389 static int  bce_get_rx_buf                      (struct bce_softc *, struct mbuf *, u16 *, u16 *, u32 *);
 390 static int  bce_init_rx_chain           (struct bce_softc *);
 391 static void bce_fill_rx_chain           (struct bce_softc *);
 392 static void bce_free_rx_chain           (struct bce_softc *);
 393
 394 #ifdef ZERO_COPY_SOCKETS
 395 static int  bce_get_pg_buf                      (struct bce_softc *, struct mbuf *, u16 *, u16 *);
 396 static int  bce_init_pg_chain           (struct bce_softc *);
 397 static void bce_fill_pg_chain           (struct bce_softc *);
 398 static void bce_free_pg_chain           (struct bce_softc *);
 399 #endif
 400
 401 static int  bce_tx_encap                        (struct bce_softc *, struct mbuf **);
 402 static void bce_start_locked            (struct ifnet *);
 403 static void bce_start                           (struct ifnet *);
 404 static int  bce_ioctl                           (struct ifnet *, u_long, caddr_t);
 405 static void bce_watchdog                        (struct bce_softc *);
 406 static int  bce_ifmedia_upd                     (struct ifnet *);
 407 static void bce_ifmedia_upd_locked      (struct ifnet *);
 408 static void bce_ifmedia_sts                     (struct ifnet *, struct ifmediareq *);
 409 static void bce_init_locked                     (struct bce_softc *);
 410 static void bce_init                            (void *);
 411 static void bce_mgmt_init_locked        (struct bce_softc *sc);
 412
 413 static void bce_init_ctx                        (struct bce_softc *);
 414 static void bce_get_mac_addr            (struct bce_softc *);
 415 static void bce_set_mac_addr            (struct bce_softc *);
 416 static void bce_phy_intr                        (struct bce_softc *);
 417 static inline u16 bce_get_hw_rx_cons(struct bce_softc *);
 418 static void bce_rx_intr                         (struct bce_softc *);
 419 static void bce_tx_intr                         (struct bce_softc *);
 420 static void bce_disable_intr            (struct bce_softc *);
 421 static void bce_enable_intr                     (struct bce_softc *, int);
 422
 423 static void bce_intr                            (void *);
 424 static void bce_set_rx_mode                     (struct bce_softc *);
 425 static void bce_stats_update            (struct bce_softc *);
 426 static void bce_tick                            (void *);
 427 static void bce_pulse                           (void *);
 428 static void bce_add_sysctls                     (struct bce_softc *);
 429
 430
 431 /****************************************************************************/
 432 /* FreeBSD device dispatch table.                                           */
 433 /****************************************************************************/
 434 static device_method_t bce_methods[] = {
 435         /* Device interface (device_if.h) */
 436         DEVMETHOD(device_probe,         bce_probe),
 437         DEVMETHOD(device_attach,        bce_attach),
 438         DEVMETHOD(device_detach,        bce_detach),
 439         DEVMETHOD(device_shutdown,      bce_shutdown),
 440 /* Supported by device interface but not used here. */
 441 /*      DEVMETHOD(device_identify,      bce_identify),      */
 442 /*      DEVMETHOD(device_suspend,       bce_suspend),       */
 443 /*      DEVMETHOD(device_resume,        bce_resume),        */
 444 /*      DEVMETHOD(device_quiesce,       bce_quiesce),       */
 445
 446         /* Bus interface (bus_if.h) */
 447         DEVMETHOD(bus_print_child,      bus_generic_print_child),
 448         DEVMETHOD(bus_driver_added,     bus_generic_driver_added),
 449
 450         /* MII interface (miibus_if.h) */
 451         DEVMETHOD(miibus_readreg,       bce_miibus_read_reg),
 452         DEVMETHOD(miibus_writereg,      bce_miibus_write_reg),
 453         DEVMETHOD(miibus_statchg,       bce_miibus_statchg),
 454 /* Supported by MII interface but not used here.       */
 455 /*      DEVMETHOD(miibus_linkchg,       bce_miibus_linkchg),   */
 456 /*      DEVMETHOD(miibus_mediainit,     bce_miibus_mediainit), */
 457
 458         { 0, 0 }
 459 };
 460
 461 static driver_t bce_driver = {
 462         "bce",
 463         bce_methods,
 464         sizeof(struct bce_softc)
 465 };
 466
 467 static devclass_t bce_devclass;
 468
 469 MODULE_DEPEND(bce, pci, 1, 1, 1);
 470 MODULE_DEPEND(bce, ether, 1, 1, 1);
 471 MODULE_DEPEND(bce, miibus, 1, 1, 1);
 472
 473 DRIVER_MODULE(bce, pci, bce_driver, bce_devclass, 0, 0);
 474 DRIVER_MODULE(miibus, bce, miibus_driver, miibus_devclass, 0, 0);
 475
 476
 477 /****************************************************************************/
 478 /* Tunable device values                                                    */
 479 /****************************************************************************/
 480 SYSCTL_NODE(_hw, OID_AUTO, bce, CTLFLAG_RD, 0, "bce driver parameters");
 481
 482 /* Allowable values are TRUE or FALSE */
 483 static int bce_tso_enable = TRUE;
 484 TUNABLE_INT("hw.bce.tso_enable", &bce_tso_enable);
 485 SYSCTL_UINT(_hw_bce, OID_AUTO, tso_enable, CTLFLAG_RDTUN, &bce_tso_enable, 0,
 486 "TSO Enable/Disable");
 487
 488 /* Allowable values are 0 (IRQ), 1 (MSI/IRQ), and 2 (MSI-X/MSI/IRQ) */
 489 /* ToDo: Add MSI-X support. */
 490 static int bce_msi_enable = 1;
 491 TUNABLE_INT("hw.bce.msi_enable", &bce_msi_enable);
 492 SYSCTL_UINT(_hw_bce, OID_AUTO, msi_enable, CTLFLAG_RDTUN, &bce_msi_enable, 0,
 493 "MSI-X|MSI|INTx selector");
 494
 495 /* ToDo: Add tunable to enable/disable strict MTU handling. */
 496 /* Currently allows "loose" RX MTU checking (i.e. sets the  */
 497 /* H/W RX MTU to the size of the largest receive buffer, or */
 498 /* 2048 bytes). This will cause a UNH failure but is more   */
 499 /* desireable from a functional perspective.                */
 500
 501
 502 /****************************************************************************/
 503 /* Device probe function.                                                   */
 504 /*                                                                          */
 505 /* Compares the device to the driver's list of supported devices and        */
 506 /* reports back to the OS whether this is the right driver for the device.  */
 507 /*                                                                          */
 508 /* Returns:                                                                 */
 509 /*   BUS_PROBE_DEFAULT on success, positive value on failure.               */
 510 /****************************************************************************/
 511 static int
 512 bce_probe(device_t dev)
 513 {
 514         struct bce_type *t;
 515         struct bce_softc *sc;
 516         char *descbuf;
 517         u16 vid = 0, did = 0, svid = 0, sdid = 0;
 518
 519         t = bce_devs;
 520
 521         sc = device_get_softc(dev);
 522         bzero(sc, sizeof(struct bce_softc));
 523         sc->bce_unit = device_get_unit(dev);
 524         sc->bce_dev = dev;
 525
 526         /* Get the data for the device to be probed. */
 527         vid  = pci_get_vendor(dev);
 528         did  = pci_get_device(dev);
 529         svid = pci_get_subvendor(dev);
 530         sdid = pci_get_subdevice(dev);
 531
 532         DBPRINT(sc, BCE_EXTREME_LOAD,
 533                 "%s(); VID = 0x%04X, DID = 0x%04X, SVID = 0x%04X, "
 534                 "SDID = 0x%04X\n", __FUNCTION__, vid, did, svid, sdid);
 535
 536         /* Look through the list of known devices for a match. */
 537         while(t->bce_name != NULL) {
 538
 539                 if ((vid == t->bce_vid) && (did == t->bce_did) &&
 540                         ((svid == t->bce_svid) || (t->bce_svid == PCI_ANY_ID)) &&
 541                         ((sdid == t->bce_sdid) || (t->bce_sdid == PCI_ANY_ID))) {
 542
 543                         descbuf = malloc(BCE_DEVDESC_MAX, M_TEMP, M_NOWAIT);
 544
 545                         if (descbuf == NULL)
 546                                 return(ENOMEM);
 547
 548                         /* Print out the device identity. */
 549                         snprintf(descbuf, BCE_DEVDESC_MAX, "%s (%c%d)",
 550                                 t->bce_name,
 551                             (((pci_read_config(dev, PCIR_REVID, 4) & 0xf0) >> 4) + 'A'),
 552                             (pci_read_config(dev, PCIR_REVID, 4) & 0xf));
 553
 554                         device_set_desc_copy(dev, descbuf);
 555                         free(descbuf, M_TEMP);
 556                         return(BUS_PROBE_DEFAULT);
 557                 }
 558                 t++;
 559         }
 560
 561         return(ENXIO);
 562 }
 563
 564
 565 /****************************************************************************/
 566 /* PCI Capabilities Probe Function.                                         */
 567 /*                                                                          */
 568 /* Walks the PCI capabiites list for the device to find what features are   */
 569 /* supported.                                                               */
 570 /*                                                                          */
 571 /* Returns:                                                                 */
 572 /*   None.                                                                  */
 573 /****************************************************************************/
 574 static void
 575 bce_print_adapter_info(struct bce_softc *sc)
 576 {
 577         DBENTER(BCE_VERBOSE_LOAD);
 578
 579         BCE_PRINTF("ASIC (0x%08X); ", sc->bce_chipid);
 580         printf("Rev (%c%d); ", ((BCE_CHIP_ID(sc) & 0xf000) >> 12) + 'A',
 581                 ((BCE_CHIP_ID(sc) & 0x0ff0) >> 4));
 582
 583         /* Bus info. */
 584         if (sc->bce_flags & BCE_PCIE_FLAG) {
 585                 printf("Bus (PCIe x%d, ", sc->link_width);
 586                 switch (sc->link_speed) {
 587                         case 1: printf("2.5Gbps); "); break;
 588                         case 2: printf("5Gbps); "); break;
 589                         default: printf("Unknown link speed); ");
 590                 }
 591         } else {
 592                 printf("Bus (PCI%s, %s, %dMHz); ",
 593                         ((sc->bce_flags & BCE_PCIX_FLAG) ? "-X" : ""),
 594                         ((sc->bce_flags & BCE_PCI_32BIT_FLAG) ? "32-bit" : "64-bit"),
 595                         sc->bus_speed_mhz);
 596         }
 597
 598         /* Firmware version and device features. */
 599         printf("B/C (0x%08X); Flags( ", sc->bce_bc_ver);
 600 #ifdef ZERO_COPY_SOCKETS
 601         printf("SPLT ");
 602 #endif
 603         if (sc->bce_flags & BCE_MFW_ENABLE_FLAG)
 604                 printf("MFW ");
 605         if (sc->bce_flags & BCE_USING_MSI_FLAG)
 606                 printf("MSI ");
 607         if (sc->bce_flags & BCE_USING_MSIX_FLAG)
 608                 printf("MSI-X ");
 609         if (sc->bce_phy_flags & BCE_PHY_2_5G_CAPABLE_FLAG)
 610                 printf("2.5G ");
 611         printf(")\n");
 612
 613         DBEXIT(BCE_VERBOSE_LOAD);
 614 }
 615
 616
 617 /****************************************************************************/
 618 /* PCI Capabilities Probe Function.                                         */
 619 /*                                                                          */
 620 /* Walks the PCI capabiites list for the device to find what features are   */
 621 /* supported.                                                               */
 622 /*                                                                          */
 623 /* Returns:                                                                 */
 624 /*   None.                                                                  */
 625 /****************************************************************************/
 626 static void
 627 bce_probe_pci_caps(device_t dev, struct bce_softc *sc)
 628 {
 629         u32 reg;
 630
 631         DBENTER(BCE_VERBOSE_LOAD);
 632
 633         /* Check if PCI-X capability is enabled. */
 634         if (pci_find_extcap(dev, PCIY_PCIX, &reg) == 0) {
 635                 if (reg != 0)
 636                         sc->bce_cap_flags |= BCE_PCIX_CAPABLE_FLAG;
 637         }
 638
 639         /* Check if PCIe capability is enabled. */
 640         if (pci_find_extcap(dev, PCIY_EXPRESS, &reg) == 0) {
 641                 if (reg != 0) {
 642                         u16 link_status = pci_read_config(dev, reg + 0x12, 2);
 643                         DBPRINT(sc, BCE_INFO_LOAD, "PCIe link_status = 0x%08X\n",
 644                                 link_status);
 645                         sc->link_speed = link_status & 0xf;
 646                         sc->link_width = (link_status >> 4) & 0x3f;
 647                         sc->bce_cap_flags |= BCE_PCIE_CAPABLE_FLAG;
 648                         sc->bce_flags |= BCE_PCIE_FLAG;
 649                 }
 650         }
 651
 652         /* Check if MSI capability is enabled. */
 653         if (pci_find_extcap(dev, PCIY_MSI, &reg) == 0) {
 654                 if (reg != 0)
 655                         sc->bce_cap_flags |= BCE_MSI_CAPABLE_FLAG;
 656         }
 657
 658         /* Check if MSI-X capability is enabled. */
 659         if (pci_find_extcap(dev, PCIY_MSIX, &reg) == 0) {
 660                 if (reg != 0)
 661                         sc->bce_cap_flags |= BCE_MSIX_CAPABLE_FLAG;
 662         }
 663
 664         DBEXIT(BCE_VERBOSE_LOAD);
 665 }
 666
 667
 668 /****************************************************************************/
 669 /* Device attach function.                                                  */
 670 /*                                                                          */
 671 /* Allocates device resources, performs secondary chip identification,      */
 672 /* resets and initializes the hardware, and initializes driver instance     */
 673 /* variables.                                                               */
 674 /*                                                                          */
 675 /* Returns:                                                                 */
 676 /*   0 on success, positive value on failure.                               */
 677 /****************************************************************************/
 678 static int
 679 bce_attach(device_t dev)
 680 {
 681         struct bce_softc *sc;
 682         struct ifnet *ifp;
 683         u32 val;
 684         int error, rid, rc = 0;
 685
 686         sc = device_get_softc(dev);
 687         sc->bce_dev = dev;
 688
 689         DBENTER(BCE_VERBOSE_LOAD | BCE_VERBOSE_RESET);
 690
 691         sc->bce_unit = device_get_unit(dev);
 692
 693         /* Set initial device and PHY flags */
 694         sc->bce_flags = 0;
 695         sc->bce_phy_flags = 0;
 696
 697         pci_enable_busmaster(dev);
 698
 699         /* Allocate PCI memory resources. */
 700         rid = PCIR_BAR(0);
 701         sc->bce_res_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 702                 &rid, RF_ACTIVE | PCI_RF_DENSE);
 703
 704         if (sc->bce_res_mem == NULL) {
 705                 BCE_PRINTF("%s(%d): PCI memory allocation failed\n",
 706                         __FILE__, __LINE__);
 707                 rc = ENXIO;
 708                 goto bce_attach_fail;
 709         }
 710
 711         /* Get various resource handles. */
 712         sc->bce_btag    = rman_get_bustag(sc->bce_res_mem);
 713         sc->bce_bhandle = rman_get_bushandle(sc->bce_res_mem);
 714         sc->bce_vhandle = (vm_offset_t) rman_get_virtual(sc->bce_res_mem);
 715
 716         bce_probe_pci_caps(dev, sc);
 717
 718         rid = 1;
 719 #if 0
 720         /* Try allocating MSI-X interrupts. */
 721         if ((sc->bce_cap_flags & BCE_MSIX_CAPABLE_FLAG) &&
 722                 (bce_msi_enable >= 2) &&
 723                 ((sc->bce_res_irq = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 724                 &rid, RF_ACTIVE)) != NULL)) {
 725
 726                 msi_needed = sc->bce_msi_count = 1;
 727
 728                 if (((error = pci_alloc_msix(dev, &sc->bce_msi_count)) != 0) ||
 729                         (sc->bce_msi_count != msi_needed)) {
 730                         BCE_PRINTF("%s(%d): MSI-X allocation failed! Requested = %d,"
 731                                 "Received = %d, error = %d\n", __FILE__, __LINE__,
 732                                 msi_needed, sc->bce_msi_count, error);
 733                         sc->bce_msi_count = 0;
 734                         pci_release_msi(dev);
 735                         bus_release_resource(dev, SYS_RES_MEMORY, rid,
 736                                 sc->bce_res_irq);
 737                         sc->bce_res_irq = NULL;
 738                 } else {
 739                         DBPRINT(sc, BCE_INFO_LOAD, "%s(): Using MSI-X interrupt.\n",
 740                                 __FUNCTION__);
 741                         sc->bce_flags |= BCE_USING_MSIX_FLAG;
 742                         sc->bce_intr = bce_intr;
 743                 }
 744         }
 745 #endif
 746
 747         /* Try allocating a MSI interrupt. */
 748         if ((sc->bce_cap_flags & BCE_MSI_CAPABLE_FLAG) &&
 749                 (bce_msi_enable >= 1) && (sc->bce_msi_count == 0)) {
 750                 sc->bce_msi_count = 1;
 751                 if ((error = pci_alloc_msi(dev, &sc->bce_msi_count)) != 0) {
 752                         BCE_PRINTF("%s(%d): MSI allocation failed! error = %d\n",
 753                                 __FILE__, __LINE__, error);
 754                         sc->bce_msi_count = 0;
 755                         pci_release_msi(dev);
 756                 } else {
 757                         DBPRINT(sc, BCE_INFO_LOAD, "%s(): Using MSI interrupt.\n",
 758                                 __FUNCTION__);
 759                         sc->bce_flags |= BCE_USING_MSI_FLAG;
 760                         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
 761                                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716))
 762                                 sc->bce_flags |= BCE_ONE_SHOT_MSI_FLAG;
 763                         sc->bce_irq_rid = 1;
 764                         sc->bce_intr = bce_intr;
 765                 }
 766         }
 767
 768         /* Try allocating a legacy interrupt. */
 769         if (sc->bce_msi_count == 0) {
 770                 DBPRINT(sc, BCE_INFO_LOAD, "%s(): Using INTx interrupt.\n",
 771                         __FUNCTION__);
 772                 rid = 0;
 773                 sc->bce_intr = bce_intr;
 774         }
 775
 776         sc->bce_res_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ,
 777                 &rid, RF_SHAREABLE | RF_ACTIVE);
 778
 779         sc->bce_irq_rid = rid;
 780
 781         /* Report any IRQ allocation errors. */
 782         if (sc->bce_res_irq == NULL) {
 783                 BCE_PRINTF("%s(%d): PCI map interrupt failed!\n",
 784                         __FILE__, __LINE__);
 785                 rc = ENXIO;
 786                 goto bce_attach_fail;
 787         }
 788
 789         /* Initialize mutex for the current device instance. */
 790         BCE_LOCK_INIT(sc, device_get_nameunit(dev));
 791
 792         /*
 793          * Configure byte swap and enable indirect register access.
 794          * Rely on CPU to do target byte swapping on big endian systems.
 795          * Access to registers outside of PCI configurtion space are not
 796          * valid until this is done.
 797          */
 798         pci_write_config(dev, BCE_PCICFG_MISC_CONFIG,
 799                                BCE_PCICFG_MISC_CONFIG_REG_WINDOW_ENA |
 800                                BCE_PCICFG_MISC_CONFIG_TARGET_MB_WORD_SWAP, 4);
 801
 802         /* Save ASIC revsion info. */
 803         sc->bce_chipid =  REG_RD(sc, BCE_MISC_ID);
 804
 805         /* Weed out any non-production controller revisions. */
 806         switch(BCE_CHIP_ID(sc)) {
 807                 case BCE_CHIP_ID_5706_A0:
 808                 case BCE_CHIP_ID_5706_A1:
 809                 case BCE_CHIP_ID_5708_A0:
 810                 case BCE_CHIP_ID_5708_B0:
 811                 case BCE_CHIP_ID_5709_A0:
 812                 case BCE_CHIP_ID_5709_B0:
 813                 case BCE_CHIP_ID_5709_B1:
 814                 case BCE_CHIP_ID_5709_B2:
 815                         BCE_PRINTF("%s(%d): Unsupported controller revision (%c%d)!\n",
 816                                 __FILE__, __LINE__,
 817                                 (((pci_read_config(dev, PCIR_REVID, 4) & 0xf0) >> 4) + 'A'),
 818                             (pci_read_config(dev, PCIR_REVID, 4) & 0xf));
 819                         rc = ENODEV;
 820                         goto bce_attach_fail;
 821         }
 822
 823         /*
 824          * The embedded PCIe to PCI-X bridge (EPB)
 825          * in the 5708 cannot address memory above
 826          * 40 bits (E7_5708CB1_23043 & E6_5708SB1_23043).
 827          */
 828         if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5708)
 829                 sc->max_bus_addr = BCE_BUS_SPACE_MAXADDR;
 830         else
 831                 sc->max_bus_addr = BUS_SPACE_MAXADDR;
 832
 833         /*
 834          * Find the base address for shared memory access.
 835          * Newer versions of bootcode use a signature and offset
 836          * while older versions use a fixed address.
 837          */
 838         val = REG_RD_IND(sc, BCE_SHM_HDR_SIGNATURE);
 839         if ((val & BCE_SHM_HDR_SIGNATURE_SIG_MASK) == BCE_SHM_HDR_SIGNATURE_SIG)
 840                 /* Multi-port devices use different offsets in shared memory. */
 841                 sc->bce_shmem_base = REG_RD_IND(sc, BCE_SHM_HDR_ADDR_0 +
 842                         (pci_get_function(sc->bce_dev) << 2));
 843         else
 844                 sc->bce_shmem_base = HOST_VIEW_SHMEM_BASE;
 845
 846         DBPRINT(sc, BCE_VERBOSE_FIRMWARE, "%s(): bce_shmem_base = 0x%08X\n",
 847                 __FUNCTION__, sc->bce_shmem_base);
 848
 849         /* Fetch the bootcode revision. */
 850         sc->bce_bc_ver = REG_RD_IND(sc, sc->bce_shmem_base +
 851                 BCE_DEV_INFO_BC_REV);
 852
 853         /* Check if any management firmware is running. */
 854         val = REG_RD_IND(sc, sc->bce_shmem_base + BCE_PORT_FEATURE);
 855         if (val & (BCE_PORT_FEATURE_ASF_ENABLED | BCE_PORT_FEATURE_IMD_ENABLED))
 856                 sc->bce_flags |= BCE_MFW_ENABLE_FLAG;
 857
 858         /* Get PCI bus information (speed and type). */
 859         val = REG_RD(sc, BCE_PCICFG_MISC_STATUS);
 860         if (val & BCE_PCICFG_MISC_STATUS_PCIX_DET) {
 861                 u32 clkreg;
 862
 863                 sc->bce_flags |= BCE_PCIX_FLAG;
 864
 865                 clkreg = REG_RD(sc, BCE_PCICFG_PCI_CLOCK_CONTROL_BITS);
 866
 867                 clkreg &= BCE_PCICFG_PCI_CLOCK_CONTROL_BITS_PCI_CLK_SPD_DET;
 868                 switch (clkreg) {
 869                 case BCE_PCICFG_PCI_CLOCK_CONTROL_BITS_PCI_CLK_SPD_DET_133MHZ:
 870                         sc->bus_speed_mhz = 133;
 871                         break;
 872
 873                 case BCE_PCICFG_PCI_CLOCK_CONTROL_BITS_PCI_CLK_SPD_DET_95MHZ:
 874                         sc->bus_speed_mhz = 100;
 875                         break;
 876
 877                 case BCE_PCICFG_PCI_CLOCK_CONTROL_BITS_PCI_CLK_SPD_DET_66MHZ:
 878                 case BCE_PCICFG_PCI_CLOCK_CONTROL_BITS_PCI_CLK_SPD_DET_80MHZ:
 879                         sc->bus_speed_mhz = 66;
 880                         break;
 881
 882                 case BCE_PCICFG_PCI_CLOCK_CONTROL_BITS_PCI_CLK_SPD_DET_48MHZ:
 883                 case BCE_PCICFG_PCI_CLOCK_CONTROL_BITS_PCI_CLK_SPD_DET_55MHZ:
 884                         sc->bus_speed_mhz = 50;
 885                         break;
 886
 887                 case BCE_PCICFG_PCI_CLOCK_CONTROL_BITS_PCI_CLK_SPD_DET_LOW:
 888                 case BCE_PCICFG_PCI_CLOCK_CONTROL_BITS_PCI_CLK_SPD_DET_32MHZ:
 889                 case BCE_PCICFG_PCI_CLOCK_CONTROL_BITS_PCI_CLK_SPD_DET_38MHZ:
 890                         sc->bus_speed_mhz = 33;
 891                         break;
 892                 }
 893         } else {
 894                 if (val & BCE_PCICFG_MISC_STATUS_M66EN)
 895                         sc->bus_speed_mhz = 66;
 896                 else
 897                         sc->bus_speed_mhz = 33;
 898         }
 899
 900         if (val & BCE_PCICFG_MISC_STATUS_32BIT_DET)
 901                 sc->bce_flags |= BCE_PCI_32BIT_FLAG;
 902
 903         /* Reset the controller and announce to bootcode that driver is present. */
 904         if (bce_reset(sc, BCE_DRV_MSG_CODE_RESET)) {
 905                 BCE_PRINTF("%s(%d): Controller reset failed!\n",
 906                         __FILE__, __LINE__);
 907                 rc = ENXIO;
 908                 goto bce_attach_fail;
 909         }
 910
 911         /* Initialize the controller. */
 912         if (bce_chipinit(sc)) {
 913                 BCE_PRINTF("%s(%d): Controller initialization failed!\n",
 914                         __FILE__, __LINE__);
 915                 rc = ENXIO;
 916                 goto bce_attach_fail;
 917         }
 918
 919         /* Perform NVRAM test. */
 920         if (bce_nvram_test(sc)) {
 921                 BCE_PRINTF("%s(%d): NVRAM test failed!\n",
 922                         __FILE__, __LINE__);
 923                 rc = ENXIO;
 924                 goto bce_attach_fail;
 925         }
 926
 927         /* Fetch the permanent Ethernet MAC address. */
 928         bce_get_mac_addr(sc);
 929
 930         /*
 931          * Trip points control how many BDs
 932          * should be ready before generating an
 933          * interrupt while ticks control how long
 934          * a BD can sit in the chain before
 935          * generating an interrupt.  Set the default
 936          * values for the RX and TX chains.
 937          */
 938
 939 #ifdef BCE_DEBUG
 940         /* Force more frequent interrupts. */
 941         sc->bce_tx_quick_cons_trip_int = 1;
 942         sc->bce_tx_quick_cons_trip     = 1;
 943         sc->bce_tx_ticks_int           = 0;
 944         sc->bce_tx_ticks               = 0;
 945
 946         sc->bce_rx_quick_cons_trip_int = 1;
 947         sc->bce_rx_quick_cons_trip     = 1;
 948         sc->bce_rx_ticks_int           = 0;
 949         sc->bce_rx_ticks               = 0;
 950 #else
 951         /* Improve throughput at the expense of increased latency. */
 952         sc->bce_tx_quick_cons_trip_int = 20;
 953         sc->bce_tx_quick_cons_trip     = 20;
 954         sc->bce_tx_ticks_int           = 80;
 955         sc->bce_tx_ticks               = 80;
 956
 957         sc->bce_rx_quick_cons_trip_int = 6;
 958         sc->bce_rx_quick_cons_trip     = 6;
 959         sc->bce_rx_ticks_int           = 18;
 960         sc->bce_rx_ticks               = 18;
 961 #endif
 962
 963         /* Update statistics once every second. */
 964         sc->bce_stats_ticks = 1000000 & 0xffff00;
 965
 966         /* Find the media type for the adapter. */
 967         bce_get_media(sc);
 968
 969         /* Store data needed by PHY driver for backplane applications */
 970         sc->bce_shared_hw_cfg = REG_RD_IND(sc, sc->bce_shmem_base +
 971                 BCE_SHARED_HW_CFG_CONFIG);
 972         sc->bce_port_hw_cfg   = REG_RD_IND(sc, sc->bce_shmem_base +
 973                 BCE_PORT_HW_CFG_CONFIG);
 974
 975         /* Allocate DMA memory resources. */
 976         if (bce_dma_alloc(dev)) {
 977                 BCE_PRINTF("%s(%d): DMA resource allocation failed!\n",
 978                     __FILE__, __LINE__);
 979                 rc = ENXIO;
 980                 goto bce_attach_fail;
 981         }
 982
 983         /* Allocate an ifnet structure. */
 984         ifp = sc->bce_ifp = if_alloc(IFT_ETHER);
 985         if (ifp == NULL) {
 986                 BCE_PRINTF("%s(%d): Interface allocation failed!\n",
 987                         __FILE__, __LINE__);
 988                 rc = ENXIO;
 989                 goto bce_attach_fail;
 990         }
 991
 992         /* Initialize the ifnet interface. */
 993         ifp->if_softc        = sc;
 994         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 995         ifp->if_flags        = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 996         ifp->if_ioctl        = bce_ioctl;
 997         ifp->if_start        = bce_start;
 998         ifp->if_init         = bce_init;
 999         ifp->if_mtu          = ETHERMTU;
1000
1001         if (bce_tso_enable) {
1002                 ifp->if_hwassist = BCE_IF_HWASSIST | CSUM_TSO;
1003                 ifp->if_capabilities = BCE_IF_CAPABILITIES | IFCAP_TSO4;
1004         } else {
1005                 ifp->if_hwassist = BCE_IF_HWASSIST;
1006                 ifp->if_capabilities = BCE_IF_CAPABILITIES;
1007         }
1008
1009         ifp->if_capenable    = ifp->if_capabilities;
1010
1011         /*
1012          * Assume standard mbuf sizes for buffer allocation.
1013          * This may change later if the MTU size is set to
1014          * something other than 1500.
1015          */
1016 #ifdef ZERO_COPY_SOCKETS
1017         sc->rx_bd_mbuf_alloc_size = MHLEN;
1018         /* Make sure offset is 16 byte aligned for hardware. */
1019         sc->rx_bd_mbuf_align_pad  = roundup2((MSIZE - MHLEN), 16) -
1020                 (MSIZE - MHLEN);
1021         sc->rx_bd_mbuf_data_len   = sc->rx_bd_mbuf_alloc_size -
1022                 sc->rx_bd_mbuf_align_pad;
1023         sc->pg_bd_mbuf_alloc_size = MCLBYTES;
1024 #else
1025         sc->rx_bd_mbuf_alloc_size = MCLBYTES;
1026         sc->rx_bd_mbuf_align_pad  = roundup2(MCLBYTES, 16) - MCLBYTES;
1027         sc->rx_bd_mbuf_data_len   = sc->rx_bd_mbuf_alloc_size -
1028                 sc->rx_bd_mbuf_align_pad;
1029 #endif
1030
1031         ifp->if_snd.ifq_drv_maxlen = USABLE_TX_BD;
1032         IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen);
1033         IFQ_SET_READY(&ifp->if_snd);
1034
1035         if (sc->bce_phy_flags & BCE_PHY_2_5G_CAPABLE_FLAG)
1036                 ifp->if_baudrate = IF_Mbps(2500ULL);
1037         else
1038                 ifp->if_baudrate = IF_Mbps(1000);
1039
1040         /* Check for an MII child bus by probing the PHY. */
1041         if (mii_phy_probe(dev, &sc->bce_miibus, bce_ifmedia_upd,
1042                 bce_ifmedia_sts)) {
1043                 BCE_PRINTF("%s(%d): No PHY found on child MII bus!\n",
1044                         __FILE__, __LINE__);
1045                 rc = ENXIO;
1046                 goto bce_attach_fail;
1047         }
1048
1049         /* Attach to the Ethernet interface list. */
1050         ether_ifattach(ifp, sc->eaddr);
1051
1052 #if __FreeBSD_version < 500000
1053         callout_init(&sc->bce_tick_callout);
1054         callout_init(&sc->bce_pulse_callout);
1055 #else
1056         callout_init_mtx(&sc->bce_tick_callout, &sc->bce_mtx, 0);
1057         callout_init_mtx(&sc->bce_pulse_callout, &sc->bce_mtx, 0);
1058 #endif
1059
1060         /* Hookup IRQ last. */
1061         rc = bus_setup_intr(dev, sc->bce_res_irq, INTR_TYPE_NET | INTR_MPSAFE,
1062                 NULL, bce_intr, sc, &sc->bce_intrhand);
1063
1064         if (rc) {
1065                 BCE_PRINTF("%s(%d): Failed to setup IRQ!\n",
1066                         __FILE__, __LINE__);
1067                 bce_detach(dev);
1068                 goto bce_attach_exit;
1069         }
1070
1071         /*
1072          * At this point we've acquired all the resources
1073          * we need to run so there's no turning back, we're
1074          * cleared for launch.
1075          */
1076
1077         /* Print some important debugging info. */
1078         DBRUNMSG(BCE_INFO, bce_dump_driver_state(sc));
1079
1080         /* Add the supported sysctls to the kernel. */
1081         bce_add_sysctls(sc);
1082
1083         BCE_LOCK(sc);
1084
1085         /*
1086          * The chip reset earlier notified the bootcode that
1087          * a driver is present.  We now need to start our pulse
1088          * routine so that the bootcode is reminded that we're
1089          * still running.
1090          */
1091         bce_pulse(sc);
1092
1093         bce_mgmt_init_locked(sc);
1094         BCE_UNLOCK(sc);
1095
1096         /* Finally, print some useful adapter info */
1097         bce_print_adapter_info(sc);
1098         DBPRINT(sc, BCE_FATAL, "%s(): sc = %p\n",
1099                 __FUNCTION__, sc);
1100
1101         goto bce_attach_exit;
1102
1103 bce_attach_fail:
1104         bce_release_resources(sc);
1105
1106 bce_attach_exit:
1107
1108         DBEXIT(BCE_VERBOSE_LOAD | BCE_VERBOSE_RESET);
1109
1110         return(rc);
1111 }
1112
1113
1114 /****************************************************************************/
1115 /* Device detach function.                                                  */
1116 /*                                                                          */
1117 /* Stops the controller, resets the controller, and releases resources.     */
1118 /*                                                                          */
1119 /* Returns:                                                                 */
1120 /*   0 on success, positive value on failure.                               */
1121 /****************************************************************************/
1122 static int
1123 bce_detach(device_t dev)
1124 {
1125         struct bce_softc *sc = device_get_softc(dev);
1126         struct ifnet *ifp;
1127         u32 msg;
1128
1129         DBENTER(BCE_VERBOSE_UNLOAD | BCE_VERBOSE_RESET);
1130
1131         ifp = sc->bce_ifp;
1132
1133         /* Stop and reset the controller. */
1134         BCE_LOCK(sc);
1135
1136         /* Stop the pulse so the bootcode can go to driver absent state. */
1137         callout_stop(&sc->bce_pulse_callout);
1138
1139         bce_stop(sc);
1140         if (sc->bce_flags & BCE_NO_WOL_FLAG)
1141                 msg = BCE_DRV_MSG_CODE_UNLOAD_LNK_DN;
1142         else
1143                 msg = BCE_DRV_MSG_CODE_UNLOAD;
1144         bce_reset(sc, msg);
1145
1146         BCE_UNLOCK(sc);
1147
1148         ether_ifdetach(ifp);
1149
1150         /* If we have a child device on the MII bus remove it too. */
1151         bus_generic_detach(dev);
1152         device_delete_child(dev, sc->bce_miibus);
1153
1154         /* Release all remaining resources. */
1155         bce_release_resources(sc);
1156
1157         DBEXIT(BCE_VERBOSE_UNLOAD | BCE_VERBOSE_RESET);
1158
1159         return(0);
1160 }
1161
1162
1163 /****************************************************************************/
1164 /* Device shutdown function.                                                */
1165 /*                                                                          */
1166 /* Stops and resets the controller.                                         */
1167 /*                                                                          */
1168 /* Returns:                                                                 */
1169 /*   0 on success, positive value on failure.                               */
1170 /****************************************************************************/
1171 static int
1172 bce_shutdown(device_t dev)
1173 {
1174         struct bce_softc *sc = device_get_softc(dev);
1175         u32 msg;
1176
1177         DBENTER(BCE_VERBOSE);
1178
1179         BCE_LOCK(sc);
1180         bce_stop(sc);
1181         if (sc->bce_flags & BCE_NO_WOL_FLAG)
1182                 msg = BCE_DRV_MSG_CODE_UNLOAD_LNK_DN;
1183         else
1184                 msg = BCE_DRV_MSG_CODE_UNLOAD;
1185         bce_reset(sc, msg);
1186         BCE_UNLOCK(sc);
1187
1188         DBEXIT(BCE_VERBOSE);
1189
1190         return (0);
1191 }
1192
1193
1194 #ifdef BCE_DEBUG
1195 /****************************************************************************/
1196 /* Register read.                                                           */
1197 /*                                                                          */
1198 /* Returns:                                                                 */
1199 /*   The value of the register.                                             */
1200 /****************************************************************************/
1201 static u32
1202 bce_reg_rd(struct bce_softc *sc, u32 offset)
1203 {
1204         u32 val = bus_space_read_4(sc->bce_btag, sc->bce_bhandle, offset);
1205         DBPRINT(sc, BCE_INSANE_REG, "%s(); offset = 0x%08X, val = 0x%08X\n",
1206                 __FUNCTION__, offset, val);
1207         return val;
1208 }
1209
1210
1211 /****************************************************************************/
1212 /* Register write (16 bit).                                                 */
1213 /*                                                                          */
1214 /* Returns:                                                                 */
1215 /*   Nothing.                                                               */
1216 /****************************************************************************/
1217 static void
1218 bce_reg_wr16(struct bce_softc *sc, u32 offset, u16 val)
1219 {
1220         DBPRINT(sc, BCE_INSANE_REG, "%s(); offset = 0x%08X, val = 0x%04X\n",
1221                 __FUNCTION__, offset, val);
1222         bus_space_write_2(sc->bce_btag, sc->bce_bhandle, offset, val);
1223 }
1224
1225
1226 /****************************************************************************/
1227 /* Register write.                                                          */
1228 /*                                                                          */
1229 /* Returns:                                                                 */
1230 /*   Nothing.                                                               */
1231 /****************************************************************************/
1232 static void
1233 bce_reg_wr(struct bce_softc *sc, u32 offset, u32 val)
1234 {
1235         DBPRINT(sc, BCE_INSANE_REG, "%s(); offset = 0x%08X, val = 0x%08X\n",
1236                 __FUNCTION__, offset, val);
1237         bus_space_write_4(sc->bce_btag, sc->bce_bhandle, offset, val);
1238 }
1239 #endif
1240
1241 /****************************************************************************/
1242 /* Indirect register read.                                                  */
1243 /*                                                                          */
1244 /* Reads NetXtreme II registers using an index/data register pair in PCI    */
1245 /* configuration space.  Using this mechanism avoids issues with posted     */
1246 /* reads but is much slower than memory-mapped I/O.                         */
1247 /*                                                                          */
1248 /* Returns:                                                                 */
1249 /*   The value of the register.                                             */
1250 /****************************************************************************/
1251 static u32
1252 bce_reg_rd_ind(struct bce_softc *sc, u32 offset)
1253 {
1254         device_t dev;
1255         dev = sc->bce_dev;
1256
1257         pci_write_config(dev, BCE_PCICFG_REG_WINDOW_ADDRESS, offset, 4);
1258 #ifdef BCE_DEBUG
1259         {
1260                 u32 val;
1261                 val = pci_read_config(dev, BCE_PCICFG_REG_WINDOW, 4);
1262                 DBPRINT(sc, BCE_INSANE_REG, "%s(); offset = 0x%08X, val = 0x%08X\n",
1263                         __FUNCTION__, offset, val);
1264                 return val;
1265         }
1266 #else
1267         return pci_read_config(dev, BCE_PCICFG_REG_WINDOW, 4);
1268 #endif
1269 }
1270
1271
1272 /****************************************************************************/
1273 /* Indirect register write.                                                 */
1274 /*                                                                          */
1275 /* Writes NetXtreme II registers using an index/data register pair in PCI   */
1276 /* configuration space.  Using this mechanism avoids issues with posted     */
1277 /* writes but is muchh slower than memory-mapped I/O.                       */
1278 /*                                                                          */
1279 /* Returns:                                                                 */
1280 /*   Nothing.                                                               */
1281 /****************************************************************************/
1282 static void
1283 bce_reg_wr_ind(struct bce_softc *sc, u32 offset, u32 val)
1284 {
1285         device_t dev;
1286         dev = sc->bce_dev;
1287
1288         DBPRINT(sc, BCE_INSANE_REG, "%s(); offset = 0x%08X, val = 0x%08X\n",
1289                 __FUNCTION__, offset, val);
1290
1291         pci_write_config(dev, BCE_PCICFG_REG_WINDOW_ADDRESS, offset, 4);
1292         pci_write_config(dev, BCE_PCICFG_REG_WINDOW, val, 4);
1293 }
1294
1295
1296 #ifdef BCE_DEBUG
1297 /****************************************************************************/
1298 /* Context memory read.                                                     */
1299 /*                                                                          */
1300 /* The NetXtreme II controller uses context memory to track connection      */
1301 /* information for L2 and higher network protocols.                         */
1302 /*                                                                          */
1303 /* Returns:                                                                 */
1304 /*   The requested 32 bit value of context memory.                          */
1305 /****************************************************************************/
1306 static u32
1307 bce_ctx_rd(struct bce_softc *sc, u32 cid_addr, u32 ctx_offset)
1308 {
1309         u32 idx, offset, retry_cnt = 5, val;
1310
1311         DBRUNIF((cid_addr > MAX_CID_ADDR || ctx_offset & 0x3 || cid_addr & CTX_MASK),
1312                 BCE_PRINTF("%s(): Invalid CID address: 0x%08X.\n",
1313                         __FUNCTION__, cid_addr));
1314
1315         offset = ctx_offset + cid_addr;
1316
1317         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
1318                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
1319
1320                 REG_WR(sc, BCE_CTX_CTX_CTRL, (offset | BCE_CTX_CTX_CTRL_READ_REQ));
1321
1322                 for (idx = 0; idx < retry_cnt; idx++) {
1323                         val = REG_RD(sc, BCE_CTX_CTX_CTRL);
1324                         if ((val & BCE_CTX_CTX_CTRL_READ_REQ) == 0)
1325                                 break;
1326                         DELAY(5);
1327                 }
1328
1329                 if (val & BCE_CTX_CTX_CTRL_READ_REQ)
1330                         BCE_PRINTF("%s(%d); Unable to read CTX memory: "
1331                                 "cid_addr = 0x%08X, offset = 0x%08X!\n",
1332                                 __FILE__, __LINE__, cid_addr, ctx_offset);
1333
1334                 val = REG_RD(sc, BCE_CTX_CTX_DATA);
1335         } else {
1336                 REG_WR(sc, BCE_CTX_DATA_ADR, offset);
1337                 val = REG_RD(sc, BCE_CTX_DATA);
1338         }
1339
1340         DBPRINT(sc, BCE_EXTREME_CTX, "%s(); cid_addr = 0x%08X, offset = 0x%08X, "
1341                 "val = 0x%08X\n", __FUNCTION__, cid_addr, ctx_offset, val);
1342
1343         return(val);
1344 }
1345 #endif
1346
1347
1348 /****************************************************************************/
1349 /* Context memory write.                                                    */
1350 /*                                                                          */
1351 /* The NetXtreme II controller uses context memory to track connection      */
1352 /* information for L2 and higher network protocols.                         */
1353 /*                                                                          */
1354 /* Returns:                                                                 */
1355 /*   Nothing.                                                               */
1356 /****************************************************************************/
1357 static void
1358 bce_ctx_wr(struct bce_softc *sc, u32 cid_addr, u32 ctx_offset, u32 ctx_val)
1359 {
1360         u32 idx, offset = ctx_offset + cid_addr;
1361         u32 val, retry_cnt = 5;
1362
1363         DBPRINT(sc, BCE_EXTREME_CTX, "%s(); cid_addr = 0x%08X, offset = 0x%08X, "
1364                 "val = 0x%08X\n", __FUNCTION__, cid_addr, ctx_offset, ctx_val);
1365
1366         DBRUNIF((cid_addr > MAX_CID_ADDR || ctx_offset & 0x3 || cid_addr & CTX_MASK),
1367                 BCE_PRINTF("%s(): Invalid CID address: 0x%08X.\n",
1368                         __FUNCTION__, cid_addr));
1369
1370         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
1371                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
1372
1373                 REG_WR(sc, BCE_CTX_CTX_DATA, ctx_val);
1374                 REG_WR(sc, BCE_CTX_CTX_CTRL, (offset | BCE_CTX_CTX_CTRL_WRITE_REQ));
1375
1376                 for (idx = 0; idx < retry_cnt; idx++) {
1377                         val = REG_RD(sc, BCE_CTX_CTX_CTRL);
1378                         if ((val & BCE_CTX_CTX_CTRL_WRITE_REQ) == 0)
1379                                 break;
1380                         DELAY(5);
1381                 }
1382
1383                 if (val & BCE_CTX_CTX_CTRL_WRITE_REQ)
1384                         BCE_PRINTF("%s(%d); Unable to write CTX memory: "
1385                                 "cid_addr = 0x%08X, offset = 0x%08X!\n",
1386                                 __FILE__, __LINE__, cid_addr, ctx_offset);
1387
1388         } else {
1389                 REG_WR(sc, BCE_CTX_DATA_ADR, offset);
1390                 REG_WR(sc, BCE_CTX_DATA, ctx_val);
1391         }
1392 }
1393
1394
1395 /****************************************************************************/
1396 /* PHY register read.                                                       */
1397 /*                                                                          */
1398 /* Implements register reads on the MII bus.                                */
1399 /*                                                                          */
1400 /* Returns:                                                                 */
1401 /*   The value of the register.                                             */
1402 /****************************************************************************/
1403 static int
1404 bce_miibus_read_reg(device_t dev, int phy, int reg)
1405 {
1406         struct bce_softc *sc;
1407         u32 val;
1408         int i;
1409
1410         sc = device_get_softc(dev);
1411
1412         /* Make sure we are accessing the correct PHY address. */
1413         if (phy != sc->bce_phy_addr) {
1414                 DBPRINT(sc, BCE_INSANE_PHY, "Invalid PHY address %d for PHY read!\n", phy);
1415                 return(0);
1416         }
1417
1418         if (sc->bce_phy_flags & BCE_PHY_INT_MODE_AUTO_POLLING_FLAG) {
1419                 val = REG_RD(sc, BCE_EMAC_MDIO_MODE);
1420                 val &= ~BCE_EMAC_MDIO_MODE_AUTO_POLL;
1421
1422                 REG_WR(sc, BCE_EMAC_MDIO_MODE, val);
1423                 REG_RD(sc, BCE_EMAC_MDIO_MODE);
1424
1425                 DELAY(40);
1426         }
1427
1428
1429         val = BCE_MIPHY(phy) | BCE_MIREG(reg) |
1430                 BCE_EMAC_MDIO_COMM_COMMAND_READ | BCE_EMAC_MDIO_COMM_DISEXT |
1431                 BCE_EMAC_MDIO_COMM_START_BUSY;
1432         REG_WR(sc, BCE_EMAC_MDIO_COMM, val);
1433
1434         for (i = 0; i < BCE_PHY_TIMEOUT; i++) {
1435                 DELAY(10);
1436
1437                 val = REG_RD(sc, BCE_EMAC_MDIO_COMM);
1438                 if (!(val & BCE_EMAC_MDIO_COMM_START_BUSY)) {
1439                         DELAY(5);
1440
1441                         val = REG_RD(sc, BCE_EMAC_MDIO_COMM);
1442                         val &= BCE_EMAC_MDIO_COMM_DATA;
1443
1444                         break;
1445                 }
1446         }
1447
1448         if (val & BCE_EMAC_MDIO_COMM_START_BUSY) {
1449                 BCE_PRINTF("%s(%d): Error: PHY read timeout! phy = %d, reg = 0x%04X\n",
1450                         __FILE__, __LINE__, phy, reg);
1451                 val = 0x0;
1452         } else {
1453                 val = REG_RD(sc, BCE_EMAC_MDIO_COMM);
1454         }
1455
1456
1457         if (sc->bce_phy_flags & BCE_PHY_INT_MODE_AUTO_POLLING_FLAG) {
1458                 val = REG_RD(sc, BCE_EMAC_MDIO_MODE);
1459                 val |= BCE_EMAC_MDIO_MODE_AUTO_POLL;
1460
1461                 REG_WR(sc, BCE_EMAC_MDIO_MODE, val);
1462                 REG_RD(sc, BCE_EMAC_MDIO_MODE);
1463
1464                 DELAY(40);
1465         }
1466
1467         DB_PRINT_PHY_REG(reg, val);
1468         return (val & 0xffff);
1469
1470 }
1471
1472
1473 /****************************************************************************/
1474 /* PHY register write.                                                      */
1475 /*                                                                          */
1476 /* Implements register writes on the MII bus.                               */
1477 /*                                                                          */
1478 /* Returns:                                                                 */
1479 /*   The value of the register.                                             */
1480 /****************************************************************************/
1481 static int
1482 bce_miibus_write_reg(device_t dev, int phy, int reg, int val)
1483 {
1484         struct bce_softc *sc;
1485         u32 val1;
1486         int i;
1487
1488         sc = device_get_softc(dev);
1489
1490         /* Make sure we are accessing the correct PHY address. */
1491         if (phy != sc->bce_phy_addr) {
1492                 DBPRINT(sc, BCE_INSANE_PHY, "Invalid PHY address %d for PHY write!\n", phy);
1493                 return(0);
1494         }
1495
1496         DB_PRINT_PHY_REG(reg, val);
1497
1498         if (sc->bce_phy_flags & BCE_PHY_INT_MODE_AUTO_POLLING_FLAG) {
1499                 val1 = REG_RD(sc, BCE_EMAC_MDIO_MODE);
1500                 val1 &= ~BCE_EMAC_MDIO_MODE_AUTO_POLL;
1501
1502                 REG_WR(sc, BCE_EMAC_MDIO_MODE, val1);
1503                 REG_RD(sc, BCE_EMAC_MDIO_MODE);
1504
1505                 DELAY(40);
1506         }
1507
1508         val1 = BCE_MIPHY(phy) | BCE_MIREG(reg) | val |
1509                 BCE_EMAC_MDIO_COMM_COMMAND_WRITE |
1510                 BCE_EMAC_MDIO_COMM_START_BUSY | BCE_EMAC_MDIO_COMM_DISEXT;
1511         REG_WR(sc, BCE_EMAC_MDIO_COMM, val1);
1512
1513         for (i = 0; i < BCE_PHY_TIMEOUT; i++) {
1514                 DELAY(10);
1515
1516                 val1 = REG_RD(sc, BCE_EMAC_MDIO_COMM);
1517                 if (!(val1 & BCE_EMAC_MDIO_COMM_START_BUSY)) {
1518                         DELAY(5);
1519                         break;
1520                 }
1521         }
1522
1523         if (val1 & BCE_EMAC_MDIO_COMM_START_BUSY)
1524                 BCE_PRINTF("%s(%d): PHY write timeout!\n",
1525                         __FILE__, __LINE__);
1526
1527         if (sc->bce_phy_flags & BCE_PHY_INT_MODE_AUTO_POLLING_FLAG) {
1528                 val1 = REG_RD(sc, BCE_EMAC_MDIO_MODE);
1529                 val1 |= BCE_EMAC_MDIO_MODE_AUTO_POLL;
1530
1531                 REG_WR(sc, BCE_EMAC_MDIO_MODE, val1);
1532                 REG_RD(sc, BCE_EMAC_MDIO_MODE);
1533
1534                 DELAY(40);
1535         }
1536
1537         return 0;
1538 }
1539
1540
1541 /****************************************************************************/
1542 /* MII bus status change.                                                   */
1543 /*                                                                          */
1544 /* Called by the MII bus driver when the PHY establishes link to set the    */
1545 /* MAC interface registers.                                                 */
1546 /*                                                                          */
1547 /* Returns:                                                                 */
1548 /*   Nothing.                                                               */
1549 /****************************************************************************/
1550 static void
1551 bce_miibus_statchg(device_t dev)
1552 {
1553         struct bce_softc *sc;
1554         struct mii_data *mii;
1555         int val;
1556
1557         sc = device_get_softc(dev);
1558
1559         DBENTER(BCE_VERBOSE_PHY);
1560
1561         mii = device_get_softc(sc->bce_miibus);
1562
1563         val = REG_RD(sc, BCE_EMAC_MODE);
1564         val &= ~(BCE_EMAC_MODE_PORT | BCE_EMAC_MODE_HALF_DUPLEX |
1565                 BCE_EMAC_MODE_MAC_LOOP | BCE_EMAC_MODE_FORCE_LINK |
1566                 BCE_EMAC_MODE_25G);
1567
1568         /* Set MII or GMII interface based on the speed negotiated by the PHY. */
1569         switch (IFM_SUBTYPE(mii->mii_media_active)) {
1570         case IFM_10_T:
1571                 if (BCE_CHIP_NUM(sc) != BCE_CHIP_NUM_5706) {
1572                         DBPRINT(sc, BCE_INFO, "Enabling 10Mb interface.\n");
1573                         val |= BCE_EMAC_MODE_PORT_MII_10;
1574                         break;
1575                 }
1576                 /* fall-through */
1577         case IFM_100_TX:
1578                 DBPRINT(sc, BCE_INFO, "Enabling MII interface.\n");
1579                 val |= BCE_EMAC_MODE_PORT_MII;
1580                 break;
1581         case IFM_2500_SX:
1582                 DBPRINT(sc, BCE_INFO, "Enabling 2.5G MAC mode.\n");
1583                 val |= BCE_EMAC_MODE_25G;
1584                 /* fall-through */
1585         case IFM_1000_T:
1586         case IFM_1000_SX:
1587                 DBPRINT(sc, BCE_INFO, "Enabling GMII interface.\n");
1588                 val |= BCE_EMAC_MODE_PORT_GMII;
1589                 break;
1590         default:
1591                 DBPRINT(sc, BCE_INFO, "Unknown speed, enabling default GMII "
1592                         "interface.\n");
1593                 val |= BCE_EMAC_MODE_PORT_GMII;
1594         }
1595
1596         /* Set half or full duplex based on the duplicity negotiated by the PHY. */
1597         if ((mii->mii_media_active & IFM_GMASK) == IFM_HDX) {
1598                 DBPRINT(sc, BCE_INFO, "Setting Half-Duplex interface.\n");
1599                 val |= BCE_EMAC_MODE_HALF_DUPLEX;
1600         } else
1601                 DBPRINT(sc, BCE_INFO, "Setting Full-Duplex interface.\n");
1602
1603         REG_WR(sc, BCE_EMAC_MODE, val);
1604
1605 #if 0
1606         /* ToDo: Enable flow control support in brgphy and bge. */
1607         /* FLAG0 is set if RX is enabled and FLAG1 if TX is enabled */
1608         if (mii->mii_media_active & IFM_FLAG0)
1609                 BCE_SETBIT(sc, BCE_EMAC_RX_MODE, BCE_EMAC_RX_MODE_FLOW_EN);
1610         if (mii->mii_media_active & IFM_FLAG1)
1611                 BCE_SETBIT(sc, BCE_EMAC_RX_MODE, BCE_EMAC_TX_MODE_FLOW_EN);
1612 #endif
1613
1614         DBEXIT(BCE_VERBOSE_PHY);
1615 }
1616
1617
1618 /****************************************************************************/
1619 /* Acquire NVRAM lock.                                                      */
1620 /*                                                                          */
1621 /* Before the NVRAM can be accessed the caller must acquire an NVRAM lock.  */
1622 /* Locks 0 and 2 are reserved, lock 1 is used by firmware and lock 2 is     */
1623 /* for use by the driver.                                                   */
1624 /*                                                                          */
1625 /* Returns:                                                                 */
1626 /*   0 on success, positive value on failure.                               */
1627 /****************************************************************************/
1628 static int
1629 bce_acquire_nvram_lock(struct bce_softc *sc)
1630 {
1631         u32 val;
1632         int j, rc = 0;
1633
1634         DBENTER(BCE_VERBOSE_NVRAM);
1635
1636         /* Request access to the flash interface. */
1637         REG_WR(sc, BCE_NVM_SW_ARB, BCE_NVM_SW_ARB_ARB_REQ_SET2);
1638         for (j = 0; j < NVRAM_TIMEOUT_COUNT; j++) {
1639                 val = REG_RD(sc, BCE_NVM_SW_ARB);
1640                 if (val & BCE_NVM_SW_ARB_ARB_ARB2)
1641                         break;
1642
1643                 DELAY(5);
1644         }
1645
1646         if (j >= NVRAM_TIMEOUT_COUNT) {
1647                 DBPRINT(sc, BCE_WARN, "Timeout acquiring NVRAM lock!\n");
1648                 rc = EBUSY;
1649         }
1650
1651         DBEXIT(BCE_VERBOSE_NVRAM);
1652         return (rc);
1653 }
1654
1655
1656 /****************************************************************************/
1657 /* Release NVRAM lock.                                                      */
1658 /*                                                                          */
1659 /* When the caller is finished accessing NVRAM the lock must be released.   */
1660 /* Locks 0 and 2 are reserved, lock 1 is used by firmware and lock 2 is     */
1661 /* for use by the driver.                                                   */
1662 /*                                                                          */
1663 /* Returns:                                                                 */
1664 /*   0 on success, positive value on failure.                               */
1665 /****************************************************************************/
1666 static int
1667 bce_release_nvram_lock(struct bce_softc *sc)
1668 {
1669         u32 val;
1670         int j, rc = 0;
1671
1672         DBENTER(BCE_VERBOSE_NVRAM);
1673
1674         /*
1675          * Relinquish nvram interface.
1676          */
1677         REG_WR(sc, BCE_NVM_SW_ARB, BCE_NVM_SW_ARB_ARB_REQ_CLR2);
1678
1679         for (j = 0; j < NVRAM_TIMEOUT_COUNT; j++) {
1680                 val = REG_RD(sc, BCE_NVM_SW_ARB);
1681                 if (!(val & BCE_NVM_SW_ARB_ARB_ARB2))
1682                         break;
1683
1684                 DELAY(5);
1685         }
1686
1687         if (j >= NVRAM_TIMEOUT_COUNT) {
1688                 DBPRINT(sc, BCE_WARN, "Timeout releasing NVRAM lock!\n");
1689                 rc = EBUSY;
1690         }
1691
1692         DBEXIT(BCE_VERBOSE_NVRAM);
1693         return (rc);
1694 }
1695
1696
1697 #ifdef BCE_NVRAM_WRITE_SUPPORT
1698 /****************************************************************************/
1699 /* Enable NVRAM write access.                                               */
1700 /*                                                                          */
1701 /* Before writing to NVRAM the caller must enable NVRAM writes.             */
1702 /*                                                                          */
1703 /* Returns:                                                                 */
1704 /*   0 on success, positive value on failure.                               */
1705 /****************************************************************************/
1706 static int
1707 bce_enable_nvram_write(struct bce_softc *sc)
1708 {
1709         u32 val;
1710         int rc = 0;
1711
1712         DBENTER(BCE_VERBOSE_NVRAM);
1713
1714         val = REG_RD(sc, BCE_MISC_CFG);
1715         REG_WR(sc, BCE_MISC_CFG, val | BCE_MISC_CFG_NVM_WR_EN_PCI);
1716
1717         if (!(sc->bce_flash_info->flags & BCE_NV_BUFFERED)) {
1718                 int j;
1719
1720                 REG_WR(sc, BCE_NVM_COMMAND, BCE_NVM_COMMAND_DONE);
1721                 REG_WR(sc, BCE_NVM_COMMAND,     BCE_NVM_COMMAND_WREN | BCE_NVM_COMMAND_DOIT);
1722
1723                 for (j = 0; j < NVRAM_TIMEOUT_COUNT; j++) {
1724                         DELAY(5);
1725
1726                         val = REG_RD(sc, BCE_NVM_COMMAND);
1727                         if (val & BCE_NVM_COMMAND_DONE)
1728                                 break;
1729                 }
1730
1731                 if (j >= NVRAM_TIMEOUT_COUNT) {
1732                         DBPRINT(sc, BCE_WARN, "Timeout writing NVRAM!\n");
1733                         rc = EBUSY;
1734                 }
1735         }
1736
1737         DBENTER(BCE_VERBOSE_NVRAM);
1738         return (rc);
1739 }
1740
1741
1742 /****************************************************************************/
1743 /* Disable NVRAM write access.                                              */
1744 /*                                                                          */
1745 /* When the caller is finished writing to NVRAM write access must be        */
1746 /* disabled.                                                                */
1747 /*                                                                          */
1748 /* Returns:                                                                 */
1749 /*   Nothing.                                                               */
1750 /****************************************************************************/
1751 static void
1752 bce_disable_nvram_write(struct bce_softc *sc)
1753 {
1754         u32 val;
1755
1756         DBENTER(BCE_VERBOSE_NVRAM);
1757
1758         val = REG_RD(sc, BCE_MISC_CFG);
1759         REG_WR(sc, BCE_MISC_CFG, val & ~BCE_MISC_CFG_NVM_WR_EN);
1760
1761         DBEXIT(BCE_VERBOSE_NVRAM);
1762
1763 }
1764 #endif
1765
1766
1767 /****************************************************************************/
1768 /* Enable NVRAM access.                                                     */
1769 /*                                                                          */
1770 /* Before accessing NVRAM for read or write operations the caller must      */
1771 /* enabled NVRAM access.                                                    */
1772 /*                                                                          */
1773 /* Returns:                                                                 */
1774 /*   Nothing.                                                               */
1775 /****************************************************************************/
1776 static void
1777 bce_enable_nvram_access(struct bce_softc *sc)
1778 {
1779         u32 val;
1780
1781         DBENTER(BCE_VERBOSE_NVRAM);
1782
1783         val = REG_RD(sc, BCE_NVM_ACCESS_ENABLE);
1784         /* Enable both bits, even on read. */
1785         REG_WR(sc, BCE_NVM_ACCESS_ENABLE,
1786                val | BCE_NVM_ACCESS_ENABLE_EN | BCE_NVM_ACCESS_ENABLE_WR_EN);
1787
1788         DBEXIT(BCE_VERBOSE_NVRAM);
1789 }
1790
1791
1792 /****************************************************************************/
1793 /* Disable NVRAM access.                                                    */
1794 /*                                                                          */
1795 /* When the caller is finished accessing NVRAM access must be disabled.     */
1796 /*                                                                          */
1797 /* Returns:                                                                 */
1798 /*   Nothing.                                                               */
1799 /****************************************************************************/
1800 static void
1801 bce_disable_nvram_access(struct bce_softc *sc)
1802 {
1803         u32 val;
1804
1805         DBENTER(BCE_VERBOSE_NVRAM);
1806
1807         val = REG_RD(sc, BCE_NVM_ACCESS_ENABLE);
1808
1809         /* Disable both bits, even after read. */
1810         REG_WR(sc, BCE_NVM_ACCESS_ENABLE,
1811                 val & ~(BCE_NVM_ACCESS_ENABLE_EN |
1812                         BCE_NVM_ACCESS_ENABLE_WR_EN));
1813
1814         DBEXIT(BCE_VERBOSE_NVRAM);
1815 }
1816
1817
1818 #ifdef BCE_NVRAM_WRITE_SUPPORT
1819 /****************************************************************************/
1820 /* Erase NVRAM page before writing.                                         */
1821 /*                                                                          */
1822 /* Non-buffered flash parts require that a page be erased before it is      */
1823 /* written.                                                                 */
1824 /*                                                                          */
1825 /* Returns:                                                                 */
1826 /*   0 on success, positive value on failure.                               */
1827 /****************************************************************************/
1828 static int
1829 bce_nvram_erase_page(struct bce_softc *sc, u32 offset)
1830 {
1831         u32 cmd;
1832         int j, rc = 0;
1833
1834         DBENTER(BCE_VERBOSE_NVRAM);
1835
1836         /* Buffered flash doesn't require an erase. */
1837         if (sc->bce_flash_info->flags & BCE_NV_BUFFERED)
1838                 goto bce_nvram_erase_page_exit;
1839
1840         /* Build an erase command. */
1841         cmd = BCE_NVM_COMMAND_ERASE | BCE_NVM_COMMAND_WR |
1842               BCE_NVM_COMMAND_DOIT;
1843
1844         /*
1845          * Clear the DONE bit separately, set the NVRAM adress to erase,
1846          * and issue the erase command.
1847          */
1848         REG_WR(sc, BCE_NVM_COMMAND, BCE_NVM_COMMAND_DONE);
1849         REG_WR(sc, BCE_NVM_ADDR, offset & BCE_NVM_ADDR_NVM_ADDR_VALUE);
1850         REG_WR(sc, BCE_NVM_COMMAND, cmd);
1851
1852         /* Wait for completion. */
1853         for (j = 0; j < NVRAM_TIMEOUT_COUNT; j++) {
1854                 u32 val;
1855
1856                 DELAY(5);
1857
1858                 val = REG_RD(sc, BCE_NVM_COMMAND);
1859                 if (val & BCE_NVM_COMMAND_DONE)
1860                         break;
1861         }
1862
1863         if (j >= NVRAM_TIMEOUT_COUNT) {
1864                 DBPRINT(sc, BCE_WARN, "Timeout erasing NVRAM.\n");
1865                 rc = EBUSY;
1866         }
1867
1868 bce_nvram_erase_page_exit:
1869         DBEXIT(BCE_VERBOSE_NVRAM);
1870         return (rc);
1871 }
1872 #endif /* BCE_NVRAM_WRITE_SUPPORT */
1873
1874
1875 /****************************************************************************/
1876 /* Read a dword (32 bits) from NVRAM.                                       */
1877 /*                                                                          */
1878 /* Read a 32 bit word from NVRAM.  The caller is assumed to have already    */
1879 /* obtained the NVRAM lock and enabled the controller for NVRAM access.     */
1880 /*                                                                          */
1881 /* Returns:                                                                 */
1882 /*   0 on success and the 32 bit value read, positive value on failure.     */
1883 /****************************************************************************/
1884 static int
1885 bce_nvram_read_dword(struct bce_softc *sc, u32 offset, u8 *ret_val,
1886                                                         u32 cmd_flags)
1887 {
1888         u32 cmd;
1889         int i, rc = 0;
1890
1891         DBENTER(BCE_EXTREME_NVRAM);
1892
1893         /* Build the command word. */
1894         cmd = BCE_NVM_COMMAND_DOIT | cmd_flags;
1895
1896         /* Calculate the offset for buffered flash if translation is used. */
1897         if (sc->bce_flash_info->flags & BCE_NV_TRANSLATE) {
1898                 offset = ((offset / sc->bce_flash_info->page_size) <<
1899                            sc->bce_flash_info->page_bits) +
1900                           (offset % sc->bce_flash_info->page_size);
1901         }
1902
1903         /*
1904          * Clear the DONE bit separately, set the address to read,
1905          * and issue the read.
1906          */
1907         REG_WR(sc, BCE_NVM_COMMAND, BCE_NVM_COMMAND_DONE);
1908         REG_WR(sc, BCE_NVM_ADDR, offset & BCE_NVM_ADDR_NVM_ADDR_VALUE);
1909         REG_WR(sc, BCE_NVM_COMMAND, cmd);
1910
1911         /* Wait for completion. */
1912         for (i = 0; i < NVRAM_TIMEOUT_COUNT; i++) {
1913                 u32 val;
1914
1915                 DELAY(5);
1916
1917                 val = REG_RD(sc, BCE_NVM_COMMAND);
1918                 if (val & BCE_NVM_COMMAND_DONE) {
1919                         val = REG_RD(sc, BCE_NVM_READ);
1920
1921                         val = bce_be32toh(val);
1922                         memcpy(ret_val, &val, 4);
1923                         break;
1924                 }
1925         }
1926
1927         /* Check for errors. */
1928         if (i >= NVRAM_TIMEOUT_COUNT) {
1929                 BCE_PRINTF("%s(%d): Timeout error reading NVRAM at offset 0x%08X!\n",
1930                         __FILE__, __LINE__, offset);
1931                 rc = EBUSY;
1932         }
1933
1934         DBEXIT(BCE_EXTREME_NVRAM);
1935         return(rc);
1936 }
1937
1938
1939 #ifdef BCE_NVRAM_WRITE_SUPPORT
1940 /****************************************************************************/
1941 /* Write a dword (32 bits) to NVRAM.                                        */
1942 /*                                                                          */
1943 /* Write a 32 bit word to NVRAM.  The caller is assumed to have already     */
1944 /* obtained the NVRAM lock, enabled the controller for NVRAM access, and    */
1945 /* enabled NVRAM write access.                                              */
1946 /*                                                                          */
1947 /* Returns:                                                                 */
1948 /*   0 on success, positive value on failure.                               */
1949 /****************************************************************************/
1950 static int
1951 bce_nvram_write_dword(struct bce_softc *sc, u32 offset, u8 *val,
1952         u32 cmd_flags)
1953 {
1954         u32 cmd, val32;
1955         int j, rc = 0;
1956
1957         DBENTER(BCE_VERBOSE_NVRAM);
1958
1959         /* Build the command word. */
1960         cmd = BCE_NVM_COMMAND_DOIT | BCE_NVM_COMMAND_WR | cmd_flags;
1961
1962         /* Calculate the offset for buffered flash if translation is used. */
1963         if (sc->bce_flash_info->flags & BCE_NV_TRANSLATE) {
1964                 offset = ((offset / sc->bce_flash_info->page_size) <<
1965                           sc->bce_flash_info->page_bits) +
1966                          (offset % sc->bce_flash_info->page_size);
1967         }
1968
1969         /*
1970          * Clear the DONE bit separately, convert NVRAM data to big-endian,
1971          * set the NVRAM address to write, and issue the write command
1972          */
1973         REG_WR(sc, BCE_NVM_COMMAND, BCE_NVM_COMMAND_DONE);
1974         memcpy(&val32, val, 4);
1975         val32 = htobe32(val32);
1976         REG_WR(sc, BCE_NVM_WRITE, val32);
1977         REG_WR(sc, BCE_NVM_ADDR, offset & BCE_NVM_ADDR_NVM_ADDR_VALUE);
1978         REG_WR(sc, BCE_NVM_COMMAND, cmd);
1979
1980         /* Wait for completion. */
1981         for (j = 0; j < NVRAM_TIMEOUT_COUNT; j++) {
1982                 DELAY(5);
1983
1984                 if (REG_RD(sc, BCE_NVM_COMMAND) & BCE_NVM_COMMAND_DONE)
1985                         break;
1986         }
1987         if (j >= NVRAM_TIMEOUT_COUNT) {
1988                 BCE_PRINTF("%s(%d): Timeout error writing NVRAM at offset 0x%08X\n",
1989                         __FILE__, __LINE__, offset);
1990                 rc = EBUSY;
1991         }
1992
1993         DBEXIT(BCE_VERBOSE_NVRAM);
1994         return (rc);
1995 }
1996 #endif /* BCE_NVRAM_WRITE_SUPPORT */
1997
1998
1999 /****************************************************************************/
2000 /* Initialize NVRAM access.                                                 */
2001 /*                                                                          */
2002 /* Identify the NVRAM device in use and prepare the NVRAM interface to      */
2003 /* access that device.                                                      */
2004 /*                                                                          */
2005 /* Returns:                                                                 */
2006 /*   0 on success, positive value on failure.                               */
2007 /****************************************************************************/
2008 static int
2009 bce_init_nvram(struct bce_softc *sc)
2010 {
2011         u32 val;
2012         int j, entry_count, rc = 0;
2013         struct flash_spec *flash;
2014
2015         DBENTER(BCE_VERBOSE_NVRAM);
2016
2017         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
2018                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
2019                 sc->bce_flash_info = &flash_5709;
2020                 goto bce_init_nvram_get_flash_size;
2021         }
2022
2023         /* Determine the selected interface. */
2024         val = REG_RD(sc, BCE_NVM_CFG1);
2025
2026         entry_count = sizeof(flash_table) / sizeof(struct flash_spec);
2027
2028         /*
2029          * Flash reconfiguration is required to support additional
2030          * NVRAM devices not directly supported in hardware.
2031          * Check if the flash interface was reconfigured
2032          * by the bootcode.
2033          */
2034
2035         if (val & 0x40000000) {
2036                 /* Flash interface reconfigured by bootcode. */
2037
2038                 DBPRINT(sc,BCE_INFO_LOAD,
2039                         "bce_init_nvram(): Flash WAS reconfigured.\n");
2040
2041                 for (j = 0, flash = &flash_table[0]; j < entry_count;
2042                      j++, flash++) {
2043                         if ((val & FLASH_BACKUP_STRAP_MASK) ==
2044                             (flash->config1 & FLASH_BACKUP_STRAP_MASK)) {
2045                                 sc->bce_flash_info = flash;
2046                                 break;
2047                         }
2048                 }
2049         } else {
2050                 /* Flash interface not yet reconfigured. */
2051                 u32 mask;
2052
2053                 DBPRINT(sc, BCE_INFO_LOAD, "%s(): Flash was NOT reconfigured.\n",
2054                         __FUNCTION__);
2055
2056                 if (val & (1 << 23))
2057                         mask = FLASH_BACKUP_STRAP_MASK;
2058                 else
2059                         mask = FLASH_STRAP_MASK;
2060
2061                 /* Look for the matching NVRAM device configuration data. */
2062                 for (j = 0, flash = &flash_table[0]; j < entry_count; j++, flash++) {
2063
2064                         /* Check if the device matches any of the known devices. */
2065                         if ((val & mask) == (flash->strapping & mask)) {
2066                                 /* Found a device match. */
2067                                 sc->bce_flash_info = flash;
2068
2069                                 /* Request access to the flash interface. */
2070                                 if ((rc = bce_acquire_nvram_lock(sc)) != 0)
2071                                         return rc;
2072
2073                                 /* Reconfigure the flash interface. */
2074                                 bce_enable_nvram_access(sc);
2075                                 REG_WR(sc, BCE_NVM_CFG1, flash->config1);
2076                                 REG_WR(sc, BCE_NVM_CFG2, flash->config2);
2077                                 REG_WR(sc, BCE_NVM_CFG3, flash->config3);
2078                                 REG_WR(sc, BCE_NVM_WRITE1, flash->write1);
2079                                 bce_disable_nvram_access(sc);
2080                                 bce_release_nvram_lock(sc);
2081
2082                                 break;
2083                         }
2084                 }
2085         }
2086
2087         /* Check if a matching device was found. */
2088         if (j == entry_count) {
2089                 sc->bce_flash_info = NULL;
2090                 BCE_PRINTF("%s(%d): Unknown Flash NVRAM found!\n",
2091                         __FILE__, __LINE__);
2092                 rc = ENODEV;
2093         }
2094
2095 bce_init_nvram_get_flash_size:
2096         /* Write the flash config data to the shared memory interface. */
2097         val = REG_RD_IND(sc, sc->bce_shmem_base + BCE_SHARED_HW_CFG_CONFIG2);
2098         val &= BCE_SHARED_HW_CFG2_NVM_SIZE_MASK;
2099         if (val)
2100                 sc->bce_flash_size = val;
2101         else
2102                 sc->bce_flash_size = sc->bce_flash_info->total_size;
2103
2104         DBPRINT(sc, BCE_INFO_LOAD, "%s(): Found %s, size = 0x%08X\n",
2105                 __FUNCTION__, sc->bce_flash_info->name,
2106                 sc->bce_flash_info->total_size);
2107
2108         DBEXIT(BCE_VERBOSE_NVRAM);
2109         return rc;
2110 }
2111
2112
2113 /****************************************************************************/
2114 /* Read an arbitrary range of data from NVRAM.                              */
2115 /*                                                                          */
2116 /* Prepares the NVRAM interface for access and reads the requested data     */
2117 /* into the supplied buffer.                                                */
2118 /*                                                                          */
2119 /* Returns:                                                                 */
2120 /*   0 on success and the data read, positive value on failure.             */
2121 /****************************************************************************/
2122 static int
2123 bce_nvram_read(struct bce_softc *sc, u32 offset, u8 *ret_buf,
2124         int buf_size)
2125 {
2126         int rc = 0;
2127         u32 cmd_flags, offset32, len32, extra;
2128
2129         DBENTER(BCE_VERBOSE_NVRAM);
2130
2131         if (buf_size == 0)
2132                 goto bce_nvram_read_exit;
2133
2134         /* Request access to the flash interface. */
2135         if ((rc = bce_acquire_nvram_lock(sc)) != 0)
2136                 goto bce_nvram_read_exit;
2137
2138         /* Enable access to flash interface */
2139         bce_enable_nvram_access(sc);
2140
2141         len32 = buf_size;
2142         offset32 = offset;
2143         extra = 0;
2144
2145         cmd_flags = 0;
2146
2147         if (offset32 & 3) {
2148                 u8 buf[4];
2149                 u32 pre_len;
2150
2151                 offset32 &= ~3;
2152                 pre_len = 4 - (offset & 3);
2153
2154                 if (pre_len >= len32) {
2155                         pre_len = len32;
2156                         cmd_flags = BCE_NVM_COMMAND_FIRST | BCE_NVM_COMMAND_LAST;
2157                 }
2158                 else {
2159                         cmd_flags = BCE_NVM_COMMAND_FIRST;
2160                 }
2161
2162                 rc = bce_nvram_read_dword(sc, offset32, buf, cmd_flags);
2163
2164                 if (rc)
2165                         return rc;
2166
2167                 memcpy(ret_buf, buf + (offset & 3), pre_len);
2168
2169                 offset32 += 4;
2170                 ret_buf += pre_len;
2171                 len32 -= pre_len;
2172         }
2173
2174         if (len32 & 3) {
2175                 extra = 4 - (len32 & 3);
2176                 len32 = (len32 + 4) & ~3;
2177         }
2178
2179         if (len32 == 4) {
2180                 u8 buf[4];
2181
2182                 if (cmd_flags)
2183                         cmd_flags = BCE_NVM_COMMAND_LAST;
2184                 else
2185                         cmd_flags = BCE_NVM_COMMAND_FIRST |
2186                                     BCE_NVM_COMMAND_LAST;
2187
2188                 rc = bce_nvram_read_dword(sc, offset32, buf, cmd_flags);
2189
2190                 memcpy(ret_buf, buf, 4 - extra);
2191         }
2192         else if (len32 > 0) {
2193                 u8 buf[4];
2194
2195                 /* Read the first word. */
2196                 if (cmd_flags)
2197                         cmd_flags = 0;
2198                 else
2199                         cmd_flags = BCE_NVM_COMMAND_FIRST;
2200
2201                 rc = bce_nvram_read_dword(sc, offset32, ret_buf, cmd_flags);
2202
2203                 /* Advance to the next dword. */
2204                 offset32 += 4;
2205                 ret_buf += 4;
2206                 len32 -= 4;
2207
2208                 while (len32 > 4 && rc == 0) {
2209                         rc = bce_nvram_read_dword(sc, offset32, ret_buf, 0);
2210
2211                         /* Advance to the next dword. */
2212                         offset32 += 4;
2213                         ret_buf += 4;
2214                         len32 -= 4;
2215                 }
2216
2217                 if (rc)
2218                         goto bce_nvram_read_locked_exit;
2219
2220                 cmd_flags = BCE_NVM_COMMAND_LAST;
2221                 rc = bce_nvram_read_dword(sc, offset32, buf, cmd_flags);
2222
2223                 memcpy(ret_buf, buf, 4 - extra);
2224         }
2225
2226 bce_nvram_read_locked_exit:
2227         /* Disable access to flash interface and release the lock. */
2228         bce_disable_nvram_access(sc);
2229         bce_release_nvram_lock(sc);
2230
2231 bce_nvram_read_exit:
2232         DBEXIT(BCE_VERBOSE_NVRAM);
2233         return rc;
2234 }
2235
2236
2237 #ifdef BCE_NVRAM_WRITE_SUPPORT
2238 /****************************************************************************/
2239 /* Write an arbitrary range of data from NVRAM.                             */
2240 /*                                                                          */
2241 /* Prepares the NVRAM interface for write access and writes the requested   */
2242 /* data from the supplied buffer.  The caller is responsible for            */
2243 /* calculating any appropriate CRCs.                                        */
2244 /*                                                                          */
2245 /* Returns:                                                                 */
2246 /*   0 on success, positive value on failure.                               */
2247 /****************************************************************************/
2248 static int
2249 bce_nvram_write(struct bce_softc *sc, u32 offset, u8 *data_buf,
2250         int buf_size)
2251 {
2252         u32 written, offset32, len32;
2253         u8 *buf, start[4], end[4];
2254         int rc = 0;
2255         int align_start, align_end;
2256
2257         DBENTER(BCE_VERBOSE_NVRAM);
2258
2259         buf = data_buf;
2260         offset32 = offset;
2261         len32 = buf_size;
2262         align_start = align_end = 0;
2263
2264         if ((align_start = (offset32 & 3))) {
2265                 offset32 &= ~3;
2266                 len32 += align_start;
2267                 if ((rc = bce_nvram_read(sc, offset32, start, 4)))
2268                         goto bce_nvram_write_exit;
2269         }
2270
2271         if (len32 & 3) {
2272                 if ((len32 > 4) || !align_start) {
2273                         align_end = 4 - (len32 & 3);
2274                         len32 += align_end;
2275                         if ((rc = bce_nvram_read(sc, offset32 + len32 - 4,
2276                                 end, 4))) {
2277                                 goto bce_nvram_write_exit;
2278                         }
2279                 }
2280         }
2281
2282         if (align_start || align_end) {
2283                 buf = malloc(len32, M_DEVBUF, M_NOWAIT);
2284                 if (buf == 0) {
2285                         rc = ENOMEM;
2286                         goto bce_nvram_write_exit;
2287                 }
2288
2289                 if (align_start) {
2290                         memcpy(buf, start, 4);
2291                 }
2292
2293                 if (align_end) {
2294                         memcpy(buf + len32 - 4, end, 4);
2295                 }
2296                 memcpy(buf + align_start, data_buf, buf_size);
2297         }
2298
2299         written = 0;
2300         while ((written < len32) && (rc == 0)) {
2301                 u32 page_start, page_end, data_start, data_end;
2302                 u32 addr, cmd_flags;
2303                 int i;
2304                 u8 flash_buffer[264];
2305
2306             /* Find the page_start addr */
2307                 page_start = offset32 + written;
2308                 page_start -= (page_start % sc->bce_flash_info->page_size);
2309                 /* Find the page_end addr */
2310                 page_end = page_start + sc->bce_flash_info->page_size;
2311                 /* Find the data_start addr */
2312                 data_start = (written == 0) ? offset32 : page_start;
2313                 /* Find the data_end addr */
2314                 data_end = (page_end > offset32 + len32) ?
2315                         (offset32 + len32) : page_end;
2316
2317                 /* Request access to the flash interface. */
2318                 if ((rc = bce_acquire_nvram_lock(sc)) != 0)
2319                         goto bce_nvram_write_exit;
2320
2321                 /* Enable access to flash interface */
2322                 bce_enable_nvram_access(sc);
2323
2324                 cmd_flags = BCE_NVM_COMMAND_FIRST;
2325                 if (!(sc->bce_flash_info->flags & BCE_NV_BUFFERED)) {
2326                         int j;
2327
2328                         /* Read the whole page into the buffer
2329                          * (non-buffer flash only) */
2330                         for (j = 0; j < sc->bce_flash_info->page_size; j += 4) {
2331                                 if (j == (sc->bce_flash_info->page_size - 4)) {
2332                                         cmd_flags |= BCE_NVM_COMMAND_LAST;
2333                                 }
2334                                 rc = bce_nvram_read_dword(sc,
2335                                         page_start + j,
2336                                         &flash_buffer[j],
2337                                         cmd_flags);
2338
2339                                 if (rc)
2340                                         goto bce_nvram_write_locked_exit;
2341
2342                                 cmd_flags = 0;
2343                         }
2344                 }
2345
2346                 /* Enable writes to flash interface (unlock write-protect) */
2347                 if ((rc = bce_enable_nvram_write(sc)) != 0)
2348                         goto bce_nvram_write_locked_exit;
2349
2350                 /* Erase the page */
2351                 if ((rc = bce_nvram_erase_page(sc, page_start)) != 0)
2352                         goto bce_nvram_write_locked_exit;
2353
2354                 /* Re-enable the write again for the actual write */
2355                 bce_enable_nvram_write(sc);
2356
2357                 /* Loop to write back the buffer data from page_start to
2358                  * data_start */
2359                 i = 0;
2360                 if (!(sc->bce_flash_info->flags & BCE_NV_BUFFERED)) {
2361                         for (addr = page_start; addr < data_start;
2362                                 addr += 4, i += 4) {
2363
2364                                 rc = bce_nvram_write_dword(sc, addr,
2365                                         &flash_buffer[i], cmd_flags);
2366
2367                                 if (rc != 0)
2368                                         goto bce_nvram_write_locked_exit;
2369
2370                                 cmd_flags = 0;
2371                         }
2372                 }
2373
2374                 /* Loop to write the new data from data_start to data_end */
2375                 for (addr = data_start; addr < data_end; addr += 4, i++) {
2376                         if ((addr == page_end - 4) ||
2377                                 ((sc->bce_flash_info->flags & BCE_NV_BUFFERED) &&
2378                                 (addr == data_end - 4))) {
2379
2380                                 cmd_flags |= BCE_NVM_COMMAND_LAST;
2381                         }
2382                         rc = bce_nvram_write_dword(sc, addr, buf,
2383                                 cmd_flags);
2384
2385                         if (rc != 0)
2386                                 goto bce_nvram_write_locked_exit;
2387
2388                         cmd_flags = 0;
2389                         buf += 4;
2390                 }
2391
2392                 /* Loop to write back the buffer data from data_end
2393                  * to page_end */
2394                 if (!(sc->bce_flash_info->flags & BCE_NV_BUFFERED)) {
2395                         for (addr = data_end; addr < page_end;
2396                                 addr += 4, i += 4) {
2397
2398                                 if (addr == page_end-4) {
2399                                         cmd_flags = BCE_NVM_COMMAND_LAST;
2400                                 }
2401                                 rc = bce_nvram_write_dword(sc, addr,
2402                                         &flash_buffer[i], cmd_flags);
2403
2404                                 if (rc != 0)
2405                                         goto bce_nvram_write_locked_exit;
2406
2407                                 cmd_flags = 0;
2408                         }
2409                 }
2410
2411                 /* Disable writes to flash interface (lock write-protect) */
2412                 bce_disable_nvram_write(sc);
2413
2414                 /* Disable access to flash interface */
2415                 bce_disable_nvram_access(sc);
2416                 bce_release_nvram_lock(sc);
2417
2418                 /* Increment written */
2419                 written += data_end - data_start;
2420         }
2421
2422         goto bce_nvram_write_exit;
2423
2424 bce_nvram_write_locked_exit:
2425                 bce_disable_nvram_write(sc);
2426                 bce_disable_nvram_access(sc);
2427                 bce_release_nvram_lock(sc);
2428
2429 bce_nvram_write_exit:
2430         if (align_start || align_end)
2431                 free(buf, M_DEVBUF);
2432
2433         DBEXIT(BCE_VERBOSE_NVRAM);
2434         return (rc);
2435 }
2436 #endif /* BCE_NVRAM_WRITE_SUPPORT */
2437
2438
2439 /****************************************************************************/
2440 /* Verifies that NVRAM is accessible and contains valid data.               */
2441 /*                                                                          */
2442 /* Reads the configuration data from NVRAM and verifies that the CRC is     */
2443 /* correct.                                                                 */
2444 /*                                                                          */
2445 /* Returns:                                                                 */
2446 /*   0 on success, positive value on failure.                               */
2447 /****************************************************************************/
2448 static int
2449 bce_nvram_test(struct bce_softc *sc)
2450 {
2451         u32 buf[BCE_NVRAM_SIZE / 4];
2452         u8 *data = (u8 *) buf;
2453         int rc = 0;
2454         u32 magic, csum;
2455
2456         DBENTER(BCE_VERBOSE_NVRAM | BCE_VERBOSE_LOAD | BCE_VERBOSE_RESET);
2457
2458         /*
2459          * Check that the device NVRAM is valid by reading
2460          * the magic value at offset 0.
2461          */
2462         if ((rc = bce_nvram_read(sc, 0, data, 4)) != 0) {
2463                 BCE_PRINTF("%s(%d): Unable to read NVRAM!\n", __FILE__, __LINE__);
2464                 goto bce_nvram_test_exit;
2465         }
2466
2467         /*
2468          * Verify that offset 0 of the NVRAM contains
2469          * a valid magic number.
2470          */
2471     magic = bce_be32toh(buf[0]);
2472         if (magic != BCE_NVRAM_MAGIC) {
2473                 rc = ENODEV;
2474                 BCE_PRINTF("%s(%d): Invalid NVRAM magic value! Expected: 0x%08X, "
2475                         "Found: 0x%08X\n",
2476                         __FILE__, __LINE__, BCE_NVRAM_MAGIC, magic);
2477                 goto bce_nvram_test_exit;
2478         }
2479
2480         /*
2481          * Verify that the device NVRAM includes valid
2482          * configuration data.
2483          */
2484         if ((rc = bce_nvram_read(sc, 0x100, data, BCE_NVRAM_SIZE)) != 0) {
2485                 BCE_PRINTF("%s(%d): Unable to read Manufacturing Information from "
2486                         "NVRAM!\n", __FILE__, __LINE__);
2487                 goto bce_nvram_test_exit;
2488         }
2489
2490         csum = ether_crc32_le(data, 0x100);
2491         if (csum != BCE_CRC32_RESIDUAL) {
2492                 rc = ENODEV;
2493                 BCE_PRINTF("%s(%d): Invalid Manufacturing Information NVRAM CRC! "
2494                         "Expected: 0x%08X, Found: 0x%08X\n",
2495                         __FILE__, __LINE__, BCE_CRC32_RESIDUAL, csum);
2496                 goto bce_nvram_test_exit;
2497         }
2498
2499         csum = ether_crc32_le(data + 0x100, 0x100);
2500         if (csum != BCE_CRC32_RESIDUAL) {
2501                 rc = ENODEV;
2502                 BCE_PRINTF("%s(%d): Invalid Feature Configuration Information "
2503                         "NVRAM CRC! Expected: 0x%08X, Found: 08%08X\n",
2504                         __FILE__, __LINE__, BCE_CRC32_RESIDUAL, csum);
2505         }
2506
2507 bce_nvram_test_exit:
2508         DBEXIT(BCE_VERBOSE_NVRAM | BCE_VERBOSE_LOAD | BCE_VERBOSE_RESET);
2509         return rc;
2510 }
2511
2512
2513 /****************************************************************************/
2514 /* Identifies the current media type of the controller and sets the PHY     */
2515 /* address.                                                                 */
2516 /*                                                                          */
2517 /* Returns:                                                                 */
2518 /*   Nothing.                                                               */
2519 /****************************************************************************/
2520 static void
2521 bce_get_media(struct bce_softc *sc)
2522 {
2523         u32 val;
2524
2525         DBENTER(BCE_VERBOSE);
2526
2527         /* Assume PHY address for copper controllers. */
2528         sc->bce_phy_addr = 1;
2529
2530         if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) {
2531                 u32 val = REG_RD(sc, BCE_MISC_DUAL_MEDIA_CTRL);
2532                 u32 bond_id = val & BCE_MISC_DUAL_MEDIA_CTRL_BOND_ID;
2533                 u32 strap;
2534
2535                 /*
2536                  * The BCM5709S is software configurable
2537                  * for Copper or SerDes operation.
2538                  */
2539                 if (bond_id == BCE_MISC_DUAL_MEDIA_CTRL_BOND_ID_C) {
2540                         DBPRINT(sc, BCE_INFO_LOAD, "5709 bonded for copper.\n");
2541                         goto bce_get_media_exit;
2542                 } else if (bond_id == BCE_MISC_DUAL_MEDIA_CTRL_BOND_ID_S) {
2543                         DBPRINT(sc, BCE_INFO_LOAD, "5709 bonded for dual media.\n");
2544                         sc->bce_phy_flags |= BCE_PHY_SERDES_FLAG;
2545                         goto bce_get_media_exit;
2546                 }
2547
2548                 if (val & BCE_MISC_DUAL_MEDIA_CTRL_STRAP_OVERRIDE)
2549                         strap = (val & BCE_MISC_DUAL_MEDIA_CTRL_PHY_CTRL) >> 21;
2550                 else
2551                         strap = (val & BCE_MISC_DUAL_MEDIA_CTRL_PHY_CTRL_STRAP) >> 8;
2552
2553                 if (pci_get_function(sc->bce_dev) == 0) {
2554                         switch (strap) {
2555                         case 0x4:
2556                         case 0x5:
2557                         case 0x6:
2558                                 DBPRINT(sc, BCE_INFO_LOAD,
2559                                         "BCM5709 s/w configured for SerDes.\n");
2560                                 sc->bce_phy_flags |= BCE_PHY_SERDES_FLAG;
2561                         default:
2562                                 DBPRINT(sc, BCE_INFO_LOAD,
2563                                         "BCM5709 s/w configured for Copper.\n");
2564                         }
2565                 } else {
2566                         switch (strap) {
2567                         case 0x1:
2568                         case 0x2:
2569                         case 0x4:
2570                                 DBPRINT(sc, BCE_INFO_LOAD,
2571                                         "BCM5709 s/w configured for SerDes.\n");
2572                                 sc->bce_phy_flags |= BCE_PHY_SERDES_FLAG;
2573                         default:
2574                                 DBPRINT(sc, BCE_INFO_LOAD,
2575                                         "BCM5709 s/w configured for Copper.\n");
2576                         }
2577                 }
2578
2579         } else if (BCE_CHIP_BOND_ID(sc) & BCE_CHIP_BOND_ID_SERDES_BIT)
2580                 sc->bce_phy_flags |= BCE_PHY_SERDES_FLAG;
2581
2582         if (sc->bce_phy_flags & BCE_PHY_SERDES_FLAG) {
2583                 sc->bce_flags |= BCE_NO_WOL_FLAG;
2584                 if (BCE_CHIP_NUM(sc) != BCE_CHIP_NUM_5706) {
2585                         sc->bce_phy_addr = 2;
2586                         val = REG_RD_IND(sc, sc->bce_shmem_base +
2587                                  BCE_SHARED_HW_CFG_CONFIG);
2588                         if (val & BCE_SHARED_HW_CFG_PHY_2_5G) {
2589                                 sc->bce_phy_flags |= BCE_PHY_2_5G_CAPABLE_FLAG;
2590                                 DBPRINT(sc, BCE_INFO_LOAD, "Found 2.5Gb capable adapter\n");
2591                         }
2592                 }
2593         } else if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5706) ||
2594                    (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5708))
2595                 sc->bce_phy_flags |= BCE_PHY_CRC_FIX_FLAG;
2596
2597 bce_get_media_exit:
2598         DBPRINT(sc, (BCE_INFO_LOAD | BCE_INFO_PHY),
2599                 "Using PHY address %d.\n", sc->bce_phy_addr);
2600
2601         DBEXIT(BCE_VERBOSE);
2602 }
2603
2604
2605 /****************************************************************************/
2606 /* Free any DMA memory owned by the driver.                                 */
2607 /*                                                                          */
2608 /* Scans through each data structre that requires DMA memory and frees      */
2609 /* the memory if allocated.                                                 */
2610 /*                                                                          */
2611 /* Returns:                                                                 */
2612 /*   Nothing.                                                               */
2613 /****************************************************************************/
2614 static void
2615 bce_dma_free(struct bce_softc *sc)
2616 {
2617         int i;
2618
2619         DBENTER(BCE_VERBOSE_RESET | BCE_VERBOSE_UNLOAD | BCE_VERBOSE_CTX);
2620
2621         /* Free, unmap, and destroy the status block. */
2622         if (sc->status_block != NULL) {
2623                 bus_dmamem_free(
2624                         sc->status_tag,
2625                     sc->status_block,
2626                     sc->status_map);
2627                 sc->status_block = NULL;
2628         }
2629
2630         if (sc->status_map != NULL) {
2631                 bus_dmamap_unload(
2632                         sc->status_tag,
2633                     sc->status_map);
2634                 bus_dmamap_destroy(sc->status_tag,
2635                     sc->status_map);
2636                 sc->status_map = NULL;
2637         }
2638
2639         if (sc->status_tag != NULL) {
2640                 bus_dma_tag_destroy(sc->status_tag);
2641                 sc->status_tag = NULL;
2642         }
2643
2644
2645         /* Free, unmap, and destroy the statistics block. */
2646         if (sc->stats_block != NULL) {
2647                 bus_dmamem_free(
2648                         sc->stats_tag,
2649                     sc->stats_block,
2650                     sc->stats_map);
2651                 sc->stats_block = NULL;
2652         }
2653
2654         if (sc->stats_map != NULL) {
2655                 bus_dmamap_unload(
2656                         sc->stats_tag,
2657                     sc->stats_map);
2658                 bus_dmamap_destroy(sc->stats_tag,
2659                     sc->stats_map);
2660                 sc->stats_map = NULL;
2661         }
2662
2663         if (sc->stats_tag != NULL) {
2664                 bus_dma_tag_destroy(sc->stats_tag);
2665                 sc->stats_tag = NULL;
2666         }
2667
2668
2669         /* Free, unmap and destroy all context memory pages. */
2670         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
2671                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
2672                 for (i = 0; i < sc->ctx_pages; i++ ) {
2673                         if (sc->ctx_block[i] != NULL) {
2674                                 bus_dmamem_free(
2675                                         sc->ctx_tag,
2676                                     sc->ctx_block[i],
2677                                     sc->ctx_map[i]);
2678                                 sc->ctx_block[i] = NULL;
2679                         }
2680
2681                         if (sc->ctx_map[i] != NULL) {
2682                                 bus_dmamap_unload(
2683                                         sc->ctx_tag,
2684                                 sc->ctx_map[i]);
2685                                 bus_dmamap_destroy(
2686                                         sc->ctx_tag,
2687                                     sc->ctx_map[i]);
2688                                 sc->ctx_map[i] = NULL;
2689                         }
2690                 }
2691
2692                 /* Destroy the context memory tag. */
2693                 if (sc->ctx_tag != NULL) {
2694                         bus_dma_tag_destroy(sc->ctx_tag);
2695                         sc->ctx_tag = NULL;
2696                 }
2697         }
2698
2699
2700         /* Free, unmap and destroy all TX buffer descriptor chain pages. */
2701         for (i = 0; i < TX_PAGES; i++ ) {
2702                 if (sc->tx_bd_chain[i] != NULL) {
2703                         bus_dmamem_free(
2704                                 sc->tx_bd_chain_tag,
2705                             sc->tx_bd_chain[i],
2706                             sc->tx_bd_chain_map[i]);
2707                         sc->tx_bd_chain[i] = NULL;
2708                 }
2709
2710                 if (sc->tx_bd_chain_map[i] != NULL) {
2711                         bus_dmamap_unload(
2712                                 sc->tx_bd_chain_tag,
2713                         sc->tx_bd_chain_map[i]);
2714                         bus_dmamap_destroy(
2715                                 sc->tx_bd_chain_tag,
2716                             sc->tx_bd_chain_map[i]);
2717                         sc->tx_bd_chain_map[i] = NULL;
2718                 }
2719         }
2720
2721         /* Destroy the TX buffer descriptor tag. */
2722         if (sc->tx_bd_chain_tag != NULL) {
2723                 bus_dma_tag_destroy(sc->tx_bd_chain_tag);
2724                 sc->tx_bd_chain_tag = NULL;
2725         }
2726
2727
2728         /* Free, unmap and destroy all RX buffer descriptor chain pages. */
2729         for (i = 0; i < RX_PAGES; i++ ) {
2730                 if (sc->rx_bd_chain[i] != NULL) {
2731                         bus_dmamem_free(
2732                                 sc->rx_bd_chain_tag,
2733                             sc->rx_bd_chain[i],
2734                             sc->rx_bd_chain_map[i]);
2735                         sc->rx_bd_chain[i] = NULL;
2736                 }
2737
2738                 if (sc->rx_bd_chain_map[i] != NULL) {
2739                         bus_dmamap_unload(
2740                                 sc->rx_bd_chain_tag,
2741                         sc->rx_bd_chain_map[i]);
2742                         bus_dmamap_destroy(
2743                                 sc->rx_bd_chain_tag,
2744                             sc->rx_bd_chain_map[i]);
2745                         sc->rx_bd_chain_map[i] = NULL;
2746                 }
2747         }
2748
2749         /* Destroy the RX buffer descriptor tag. */
2750         if (sc->rx_bd_chain_tag != NULL) {
2751                 bus_dma_tag_destroy(sc->rx_bd_chain_tag);
2752                 sc->rx_bd_chain_tag = NULL;
2753         }
2754
2755
2756 #ifdef ZERO_COPY_SOCKETS
2757         /* Free, unmap and destroy all page buffer descriptor chain pages. */
2758         for (i = 0; i < PG_PAGES; i++ ) {
2759                 if (sc->pg_bd_chain[i] != NULL) {
2760                         bus_dmamem_free(
2761                                 sc->pg_bd_chain_tag,
2762                             sc->pg_bd_chain[i],
2763                             sc->pg_bd_chain_map[i]);
2764                         sc->pg_bd_chain[i] = NULL;
2765                 }
2766
2767                 if (sc->pg_bd_chain_map[i] != NULL) {
2768                         bus_dmamap_unload(
2769                                 sc->pg_bd_chain_tag,
2770                         sc->pg_bd_chain_map[i]);
2771                         bus_dmamap_destroy(
2772                                 sc->pg_bd_chain_tag,
2773                             sc->pg_bd_chain_map[i]);
2774                         sc->pg_bd_chain_map[i] = NULL;
2775                 }
2776         }
2777
2778         /* Destroy the page buffer descriptor tag. */
2779         if (sc->pg_bd_chain_tag != NULL) {
2780                 bus_dma_tag_destroy(sc->pg_bd_chain_tag);
2781                 sc->pg_bd_chain_tag = NULL;
2782         }
2783 #endif
2784
2785
2786         /* Unload and destroy the TX mbuf maps. */
2787         for (i = 0; i < TOTAL_TX_BD; i++) {
2788                 if (sc->tx_mbuf_map[i] != NULL) {
2789                         bus_dmamap_unload(sc->tx_mbuf_tag,
2790                                 sc->tx_mbuf_map[i]);
2791                         bus_dmamap_destroy(sc->tx_mbuf_tag,
2792                                 sc->tx_mbuf_map[i]);
2793                         sc->tx_mbuf_map[i] = NULL;
2794                 }
2795         }
2796
2797         /* Destroy the TX mbuf tag. */
2798         if (sc->tx_mbuf_tag != NULL) {
2799                 bus_dma_tag_destroy(sc->tx_mbuf_tag);
2800                 sc->tx_mbuf_tag = NULL;
2801         }
2802
2803         /* Unload and destroy the RX mbuf maps. */
2804         for (i = 0; i < TOTAL_RX_BD; i++) {
2805                 if (sc->rx_mbuf_map[i] != NULL) {
2806                         bus_dmamap_unload(sc->rx_mbuf_tag,
2807                                 sc->rx_mbuf_map[i]);
2808                         bus_dmamap_destroy(sc->rx_mbuf_tag,
2809                                 sc->rx_mbuf_map[i]);
2810                         sc->rx_mbuf_map[i] = NULL;
2811                 }
2812         }
2813
2814         /* Destroy the RX mbuf tag. */
2815         if (sc->rx_mbuf_tag != NULL) {
2816                 bus_dma_tag_destroy(sc->rx_mbuf_tag);
2817                 sc->rx_mbuf_tag = NULL;
2818         }
2819
2820 #ifdef ZERO_COPY_SOCKETS
2821         /* Unload and destroy the page mbuf maps. */
2822         for (i = 0; i < TOTAL_PG_BD; i++) {
2823                 if (sc->pg_mbuf_map[i] != NULL) {
2824                         bus_dmamap_unload(sc->pg_mbuf_tag,
2825                                 sc->pg_mbuf_map[i]);
2826                         bus_dmamap_destroy(sc->pg_mbuf_tag,
2827                                 sc->pg_mbuf_map[i]);
2828                         sc->pg_mbuf_map[i] = NULL;
2829                 }
2830         }
2831
2832         /* Destroy the page mbuf tag. */
2833         if (sc->pg_mbuf_tag != NULL) {
2834                 bus_dma_tag_destroy(sc->pg_mbuf_tag);
2835                 sc->pg_mbuf_tag = NULL;
2836         }
2837 #endif
2838
2839         /* Destroy the parent tag */
2840         if (sc->parent_tag != NULL) {
2841                 bus_dma_tag_destroy(sc->parent_tag);
2842                 sc->parent_tag = NULL;
2843         }
2844
2845         DBEXIT(BCE_VERBOSE_RESET | BCE_VERBOSE_UNLOAD | BCE_VERBOSE_CTX);
2846 }
2847
2848
2849 /****************************************************************************/
2850 /* Get DMA memory from the OS.                                              */
2851 /*                                                                          */
2852 /* Validates that the OS has provided DMA buffers in response to a          */
2853 /* bus_dmamap_load() call and saves the physical address of those buffers.  */
2854 /* When the callback is used the OS will return 0 for the mapping function  */
2855 /* (bus_dmamap_load()) so we use the value of map_arg->maxsegs to pass any  */
2856 /* failures back to the caller.                                             */
2857 /*                                                                          */
2858 /* Returns:                                                                 */
2859 /*   Nothing.                                                               */
2860 /****************************************************************************/
2861 static void
2862 bce_dma_map_addr(void *arg, bus_dma_segment_t *segs, int nseg, int error)
2863 {
2864         bus_addr_t *busaddr = arg;
2865
2866         /* Simulate a mapping failure. */
2867         DBRUNIF(DB_RANDOMTRUE(dma_map_addr_failed_sim_control),
2868                 error = ENOMEM);
2869
2870         /* Check for an error and signal the caller that an error occurred. */
2871         if (error) {
2872                 *busaddr = 0;
2873         } else {
2874                 *busaddr = segs->ds_addr;
2875         }
2876
2877         return;
2878 }
2879
2880
2881 /****************************************************************************/
2882 /* Allocate any DMA memory needed by the driver.                            */
2883 /*                                                                          */
2884 /* Allocates DMA memory needed for the various global structures needed by  */
2885 /* hardware.                                                                */
2886 /*                                                                          */
2887 /* Memory alignment requirements:                                           */
2888 /* +-----------------+----------+----------+----------+----------+          */
2889 /* |                 |   5706   |   5708   |   5709   |   5716   |          */
2890 /* +-----------------+----------+----------+----------+----------+          */
2891 /* |Status Block     | 8 bytes  | 8 bytes  | 16 bytes | 16 bytes |          */
2892 /* |Statistics Block | 8 bytes  | 8 bytes  | 16 bytes | 16 bytes |          */
2893 /* |RX Buffers       | 16 bytes | 16 bytes | 16 bytes | 16 bytes |          */
2894 /* |PG Buffers       |   none   |   none   |   none   |   none   |          */
2895 /* |TX Buffers       |   none   |   none   |   none   |   none   |          */
2896 /* |Chain Pages(1)   |   4KiB   |   4KiB   |   4KiB   |   4KiB   |          */
2897 /* +-----------------+----------+----------+----------+----------+          */
2898 /*                                                                          */
2899 /* (1) Must align with CPU page size (BCM_PAGE_SZIE).                       */
2900 /*                                                                          */
2901 /* Returns:                                                                 */
2902 /*   0 for success, positive value for failure.                             */
2903 /****************************************************************************/
2904 static int
2905 bce_dma_alloc(device_t dev)
2906 {
2907         struct bce_softc *sc;
2908         int i, error, rc = 0;
2909         bus_size_t max_size, max_seg_size;
2910         int max_segments;
2911
2912         sc = device_get_softc(dev);
2913
2914         DBENTER(BCE_VERBOSE_RESET | BCE_VERBOSE_CTX);
2915
2916         /*
2917          * Allocate the parent bus DMA tag appropriate for PCI.
2918          */
2919         if (bus_dma_tag_create(NULL,
2920                         1,
2921                         BCE_DMA_BOUNDARY,
2922                         sc->max_bus_addr,
2923                         BUS_SPACE_MAXADDR,
2924                         NULL, NULL,
2925                         MAXBSIZE,
2926                         BUS_SPACE_UNRESTRICTED,
2927                         BUS_SPACE_MAXSIZE_32BIT,
2928                         0,
2929                         NULL, NULL,
2930                         &sc->parent_tag)) {
2931                 BCE_PRINTF("%s(%d): Could not allocate parent DMA tag!\n",
2932                         __FILE__, __LINE__);
2933                 rc = ENOMEM;
2934                 goto bce_dma_alloc_exit;
2935         }
2936
2937         /*
2938          * Create a DMA tag for the status block, allocate and clear the
2939          * memory, map the memory into DMA space, and fetch the physical
2940          * address of the block.
2941          */
2942         if (bus_dma_tag_create(sc->parent_tag,
2943                 BCE_DMA_ALIGN,
2944                 BCE_DMA_BOUNDARY,
2945                 sc->max_bus_addr,
2946                 BUS_SPACE_MAXADDR,
2947                 NULL, NULL,
2948                 BCE_STATUS_BLK_SZ,
2949                 1,
2950                 BCE_STATUS_BLK_SZ,
2951                 0,
2952                 NULL, NULL,
2953                 &sc->status_tag)) {
2954                 BCE_PRINTF("%s(%d): Could not allocate status block DMA tag!\n",
2955                         __FILE__, __LINE__);
2956                 rc = ENOMEM;
2957                 goto bce_dma_alloc_exit;
2958         }
2959
2960         if(bus_dmamem_alloc(sc->status_tag,
2961                 (void **)&sc->status_block,
2962                 BUS_DMA_NOWAIT,
2963                 &sc->status_map)) {
2964                 BCE_PRINTF("%s(%d): Could not allocate status block DMA memory!\n",
2965                         __FILE__, __LINE__);
2966                 rc = ENOMEM;
2967                 goto bce_dma_alloc_exit;
2968         }
2969
2970         bzero((char *)sc->status_block, BCE_STATUS_BLK_SZ);
2971
2972         error = bus_dmamap_load(sc->status_tag,
2973                 sc->status_map,
2974                 sc->status_block,
2975                 BCE_STATUS_BLK_SZ,
2976                 bce_dma_map_addr,
2977                 &sc->status_block_paddr,
2978                 BUS_DMA_NOWAIT);
2979
2980         if (error) {
2981                 BCE_PRINTF("%s(%d): Could not map status block DMA memory!\n",
2982                         __FILE__, __LINE__);
2983                 rc = ENOMEM;
2984                 goto bce_dma_alloc_exit;
2985         }
2986
2987         DBPRINT(sc, BCE_INFO, "%s(): status_block_paddr = 0x%jX\n",
2988                 __FUNCTION__, (uintmax_t) sc->status_block_paddr);
2989
2990         /*
2991          * Create a DMA tag for the statistics block, allocate and clear the
2992          * memory, map the memory into DMA space, and fetch the physical
2993          * address of the block.
2994          */
2995         if (bus_dma_tag_create(sc->parent_tag,
2996                 BCE_DMA_ALIGN,
2997                 BCE_DMA_BOUNDARY,
2998                 sc->max_bus_addr,
2999                 BUS_SPACE_MAXADDR,
3000                 NULL, NULL,
3001                 BCE_STATS_BLK_SZ,
3002                 1,
3003                 BCE_STATS_BLK_SZ,
3004                 0,
3005                 NULL, NULL,
3006                 &sc->stats_tag)) {
3007                 BCE_PRINTF("%s(%d): Could not allocate statistics block DMA tag!\n",
3008                         __FILE__, __LINE__);
3009                 rc = ENOMEM;
3010                 goto bce_dma_alloc_exit;
3011         }
3012
3013         if (bus_dmamem_alloc(sc->stats_tag,
3014                 (void **)&sc->stats_block,
3015                 BUS_DMA_NOWAIT,
3016                 &sc->stats_map)) {
3017                 BCE_PRINTF("%s(%d): Could not allocate statistics block DMA memory!\n",
3018                         __FILE__, __LINE__);
3019                 rc = ENOMEM;
3020                 goto bce_dma_alloc_exit;
3021         }
3022
3023         bzero((char *)sc->stats_block, BCE_STATS_BLK_SZ);
3024
3025         error = bus_dmamap_load(sc->stats_tag,
3026                 sc->stats_map,
3027                 sc->stats_block,
3028                 BCE_STATS_BLK_SZ,
3029                 bce_dma_map_addr,
3030                 &sc->stats_block_paddr,
3031                 BUS_DMA_NOWAIT);
3032
3033         if(error) {
3034                 BCE_PRINTF("%s(%d): Could not map statistics block DMA memory!\n",
3035                         __FILE__, __LINE__);
3036                 rc = ENOMEM;
3037                 goto bce_dma_alloc_exit;
3038         }
3039
3040         DBPRINT(sc, BCE_INFO, "%s(): stats_block_paddr = 0x%jX\n",
3041                 __FUNCTION__, (uintmax_t) sc->stats_block_paddr);
3042
3043         /* BCM5709 uses host memory as cache for context memory. */
3044         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
3045                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
3046                 sc->ctx_pages = 0x2000 / BCM_PAGE_SIZE;
3047                 if (sc->ctx_pages == 0)
3048                         sc->ctx_pages = 1;
3049
3050                 DBRUNIF((sc->ctx_pages > 512),
3051                         BCE_PRINTF("%s(%d): Too many CTX pages! %d > 512\n",
3052                                 __FILE__, __LINE__, sc->ctx_pages));
3053
3054                 /*
3055                  * Create a DMA tag for the context pages,
3056                  * allocate and clear the memory, map the
3057                  * memory into DMA space, and fetch the
3058                  * physical address of the block.
3059                  */
3060                 if(bus_dma_tag_create(sc->parent_tag,
3061                         BCM_PAGE_SIZE,
3062                     BCE_DMA_BOUNDARY,
3063                         sc->max_bus_addr,
3064                         BUS_SPACE_MAXADDR,
3065                         NULL, NULL,
3066                         BCM_PAGE_SIZE,
3067                         1,
3068                         BCM_PAGE_SIZE,
3069                         0,
3070                         NULL, NULL,
3071                         &sc->ctx_tag)) {
3072                         BCE_PRINTF("%s(%d): Could not allocate CTX DMA tag!\n",
3073                                 __FILE__, __LINE__);
3074                         rc = ENOMEM;
3075                         goto bce_dma_alloc_exit;
3076                 }
3077
3078                 for (i = 0; i < sc->ctx_pages; i++) {
3079
3080                         if(bus_dmamem_alloc(sc->ctx_tag,
3081                                 (void **)&sc->ctx_block[i],
3082                         BUS_DMA_NOWAIT,
3083                         &sc->ctx_map[i])) {
3084                                 BCE_PRINTF("%s(%d): Could not allocate CTX "
3085                                         "DMA memory!\n", __FILE__, __LINE__);
3086                                 rc = ENOMEM;
3087                                 goto bce_dma_alloc_exit;
3088                         }
3089
3090                         bzero((char *)sc->ctx_block[i], BCM_PAGE_SIZE);
3091
3092                         error = bus_dmamap_load(sc->ctx_tag,
3093                         sc->ctx_map[i],
3094                         sc->ctx_block[i],
3095                         BCM_PAGE_SIZE,
3096                         bce_dma_map_addr,
3097                         &sc->ctx_paddr[i],
3098                         BUS_DMA_NOWAIT);
3099
3100                         if (error) {
3101                                 BCE_PRINTF("%s(%d): Could not map CTX DMA memory!\n",
3102                                         __FILE__, __LINE__);
3103                                 rc = ENOMEM;
3104                                 goto bce_dma_alloc_exit;
3105                         }
3106
3107                         DBPRINT(sc, BCE_INFO, "%s(): ctx_paddr[%d] = 0x%jX\n",
3108                                 __FUNCTION__, i, (uintmax_t) sc->ctx_paddr[i]);
3109                 }
3110         }
3111
3112         /*
3113          * Create a DMA tag for the TX buffer descriptor chain,
3114          * allocate and clear the  memory, and fetch the
3115          * physical address of the block.
3116          */
3117         if(bus_dma_tag_create(sc->parent_tag,
3118                         BCM_PAGE_SIZE,
3119                     BCE_DMA_BOUNDARY,
3120                         sc->max_bus_addr,
3121                         BUS_SPACE_MAXADDR,
3122                         NULL, NULL,
3123                         BCE_TX_CHAIN_PAGE_SZ,
3124                         1,
3125                         BCE_TX_CHAIN_PAGE_SZ,
3126                         0,
3127                         NULL, NULL,
3128                         &sc->tx_bd_chain_tag)) {
3129                 BCE_PRINTF("%s(%d): Could not allocate TX descriptor chain DMA tag!\n",
3130                         __FILE__, __LINE__);
3131                 rc = ENOMEM;
3132                 goto bce_dma_alloc_exit;
3133         }
3134
3135         for (i = 0; i < TX_PAGES; i++) {
3136
3137                 if(bus_dmamem_alloc(sc->tx_bd_chain_tag,
3138                         (void **)&sc->tx_bd_chain[i],
3139                         BUS_DMA_NOWAIT,
3140                         &sc->tx_bd_chain_map[i])) {
3141                         BCE_PRINTF("%s(%d): Could not allocate TX descriptor "
3142                                 "chain DMA memory!\n", __FILE__, __LINE__);
3143                         rc = ENOMEM;
3144                         goto bce_dma_alloc_exit;
3145                 }
3146
3147                 error = bus_dmamap_load(sc->tx_bd_chain_tag,
3148                         sc->tx_bd_chain_map[i],
3149                         sc->tx_bd_chain[i],
3150                         BCE_TX_CHAIN_PAGE_SZ,
3151                         bce_dma_map_addr,
3152                         &sc->tx_bd_chain_paddr[i],
3153                         BUS_DMA_NOWAIT);
3154
3155                 if (error) {
3156                         BCE_PRINTF("%s(%d): Could not map TX descriptor chain DMA memory!\n",
3157                                 __FILE__, __LINE__);
3158                         rc = ENOMEM;
3159                         goto bce_dma_alloc_exit;
3160                 }
3161
3162                 DBPRINT(sc, BCE_INFO, "%s(): tx_bd_chain_paddr[%d] = 0x%jX\n",
3163                         __FUNCTION__, i, (uintmax_t) sc->tx_bd_chain_paddr[i]);
3164         }
3165
3166         /* Check the required size before mapping to conserve resources. */
3167         if (bce_tso_enable) {
3168                 max_size     = BCE_TSO_MAX_SIZE;
3169                 max_segments = BCE_MAX_SEGMENTS;
3170                 max_seg_size = BCE_TSO_MAX_SEG_SIZE;
3171         } else {
3172                 max_size     = MCLBYTES * BCE_MAX_SEGMENTS;
3173                 max_segments = BCE_MAX_SEGMENTS;
3174                 max_seg_size = MCLBYTES;
3175         }
3176
3177         /* Create a DMA tag for TX mbufs. */
3178         if (bus_dma_tag_create(sc->parent_tag,
3179                         1,
3180                         BCE_DMA_BOUNDARY,
3181                         sc->max_bus_addr,
3182                         BUS_SPACE_MAXADDR,
3183                         NULL, NULL,
3184                         max_size,
3185                         max_segments,
3186                         max_seg_size,
3187                         0,
3188                         NULL, NULL,
3189                         &sc->tx_mbuf_tag)) {
3190                 BCE_PRINTF("%s(%d): Could not allocate TX mbuf DMA tag!\n",
3191                         __FILE__, __LINE__);
3192                 rc = ENOMEM;
3193                 goto bce_dma_alloc_exit;
3194         }
3195
3196         /* Create DMA maps for the TX mbufs clusters. */
3197         for (i = 0; i < TOTAL_TX_BD; i++) {
3198                 if (bus_dmamap_create(sc->tx_mbuf_tag, BUS_DMA_NOWAIT,
3199                         &sc->tx_mbuf_map[i])) {
3200                         BCE_PRINTF("%s(%d): Unable to create TX mbuf DMA map!\n",
3201                                 __FILE__, __LINE__);
3202                         rc = ENOMEM;
3203                         goto bce_dma_alloc_exit;
3204                 }
3205         }
3206
3207         /*
3208          * Create a DMA tag for the RX buffer descriptor chain,
3209          * allocate and clear the memory, and fetch the physical
3210          * address of the blocks.
3211          */
3212         if (bus_dma_tag_create(sc->parent_tag,
3213                         BCM_PAGE_SIZE,
3214                         BCE_DMA_BOUNDARY,
3215                         BUS_SPACE_MAXADDR,
3216                         sc->max_bus_addr,
3217                         NULL, NULL,
3218                         BCE_RX_CHAIN_PAGE_SZ,
3219                         1,
3220                         BCE_RX_CHAIN_PAGE_SZ,
3221                         0,
3222                         NULL, NULL,
3223                         &sc->rx_bd_chain_tag)) {
3224                 BCE_PRINTF("%s(%d): Could not allocate RX descriptor chain DMA tag!\n",
3225                         __FILE__, __LINE__);
3226                 rc = ENOMEM;
3227                 goto bce_dma_alloc_exit;
3228         }
3229
3230         for (i = 0; i < RX_PAGES; i++) {
3231
3232                 if (bus_dmamem_alloc(sc->rx_bd_chain_tag,
3233                         (void **)&sc->rx_bd_chain[i],
3234                         BUS_DMA_NOWAIT,
3235                         &sc->rx_bd_chain_map[i])) {
3236                         BCE_PRINTF("%s(%d): Could not allocate RX descriptor chain "
3237                                 "DMA memory!\n", __FILE__, __LINE__);
3238                         rc = ENOMEM;
3239                         goto bce_dma_alloc_exit;
3240                 }
3241
3242                 bzero((char *)sc->rx_bd_chain[i], BCE_RX_CHAIN_PAGE_SZ);
3243
3244                 error = bus_dmamap_load(sc->rx_bd_chain_tag,
3245                         sc->rx_bd_chain_map[i],
3246                         sc->rx_bd_chain[i],
3247                         BCE_RX_CHAIN_PAGE_SZ,
3248                         bce_dma_map_addr,
3249                         &sc->rx_bd_chain_paddr[i],
3250                         BUS_DMA_NOWAIT);
3251
3252                 if (error) {
3253                         BCE_PRINTF("%s(%d): Could not map RX descriptor chain DMA memory!\n",
3254                                 __FILE__, __LINE__);
3255                         rc = ENOMEM;
3256                         goto bce_dma_alloc_exit;
3257                 }
3258
3259                 DBPRINT(sc, BCE_INFO, "%s(): rx_bd_chain_paddr[%d] = 0x%jX\n",
3260                         __FUNCTION__, i, (uintmax_t) sc->rx_bd_chain_paddr[i]);
3261         }
3262
3263         /*
3264          * Create a DMA tag for RX mbufs.
3265          */
3266 #ifdef ZERO_COPY_SOCKETS
3267         max_size = max_seg_size = ((sc->rx_bd_mbuf_alloc_size < MCLBYTES) ?
3268                 MCLBYTES : sc->rx_bd_mbuf_alloc_size);
3269 #else
3270         max_size = max_seg_size = MJUM9BYTES;
3271 #endif
3272         max_segments = 1;
3273
3274         DBPRINT(sc, BCE_INFO, "%s(): Creating rx_mbuf_tag (max size = 0x%jX "
3275                 "max segments = %d, max segment size = 0x%jX)\n", __FUNCTION__,
3276                 (uintmax_t) max_size, max_segments, (uintmax_t) max_seg_size);
3277
3278         if (bus_dma_tag_create(sc->parent_tag,
3279                         1,
3280                         BCE_DMA_BOUNDARY,
3281                         sc->max_bus_addr,
3282                         BUS_SPACE_MAXADDR,
3283                         NULL, NULL,
3284                         max_size,
3285                         max_segments,
3286                         max_seg_size,
3287                         0,
3288                         NULL, NULL,
3289                 &sc->rx_mbuf_tag)) {
3290                 BCE_PRINTF("%s(%d): Could not allocate RX mbuf DMA tag!\n",
3291                         __FILE__, __LINE__);
3292                 rc = ENOMEM;
3293                 goto bce_dma_alloc_exit;
3294         }
3295
3296         /* Create DMA maps for the RX mbuf clusters. */
3297         for (i = 0; i < TOTAL_RX_BD; i++) {
3298                 if (bus_dmamap_create(sc->rx_mbuf_tag, BUS_DMA_NOWAIT,
3299                                 &sc->rx_mbuf_map[i])) {
3300                         BCE_PRINTF("%s(%d): Unable to create RX mbuf DMA map!\n",
3301                                 __FILE__, __LINE__);
3302                         rc = ENOMEM;
3303                         goto bce_dma_alloc_exit;
3304                 }
3305         }
3306
3307 #ifdef ZERO_COPY_SOCKETS
3308         /*
3309          * Create a DMA tag for the page buffer descriptor chain,
3310          * allocate and clear the memory, and fetch the physical
3311          * address of the blocks.
3312          */
3313         if (bus_dma_tag_create(sc->parent_tag,
3314                         BCM_PAGE_SIZE,
3315                         BCE_DMA_BOUNDARY,
3316                         BUS_SPACE_MAXADDR,
3317                         sc->max_bus_addr,
3318                         NULL, NULL,
3319                         BCE_PG_CHAIN_PAGE_SZ,
3320                         1,
3321                         BCE_PG_CHAIN_PAGE_SZ,
3322                         0,
3323                         NULL, NULL,
3324                         &sc->pg_bd_chain_tag)) {
3325                 BCE_PRINTF("%s(%d): Could not allocate page descriptor chain DMA tag!\n",
3326                         __FILE__, __LINE__);
3327                 rc = ENOMEM;
3328                 goto bce_dma_alloc_exit;
3329         }
3330
3331         for (i = 0; i < PG_PAGES; i++) {
3332
3333                 if (bus_dmamem_alloc(sc->pg_bd_chain_tag,
3334                         (void **)&sc->pg_bd_chain[i],
3335                         BUS_DMA_NOWAIT,
3336                         &sc->pg_bd_chain_map[i])) {
3337                         BCE_PRINTF("%s(%d): Could not allocate page descriptor chain "
3338                                 "DMA memory!\n", __FILE__, __LINE__);
3339                         rc = ENOMEM;
3340                         goto bce_dma_alloc_exit;
3341                 }
3342
3343                 bzero((char *)sc->pg_bd_chain[i], BCE_PG_CHAIN_PAGE_SZ);
3344
3345                 error = bus_dmamap_load(sc->pg_bd_chain_tag,
3346                         sc->pg_bd_chain_map[i],
3347                         sc->pg_bd_chain[i],
3348                         BCE_PG_CHAIN_PAGE_SZ,
3349                         bce_dma_map_addr,
3350                         &sc->pg_bd_chain_paddr[i],
3351                         BUS_DMA_NOWAIT);
3352
3353                 if (error) {
3354                         BCE_PRINTF("%s(%d): Could not map page descriptor chain DMA memory!\n",
3355                                 __FILE__, __LINE__);
3356                         rc = ENOMEM;
3357                         goto bce_dma_alloc_exit;
3358                 }
3359
3360                 DBPRINT(sc, BCE_INFO, "%s(): pg_bd_chain_paddr[%d] = 0x%jX\n",
3361                         __FUNCTION__, i, (uintmax_t) sc->pg_bd_chain_paddr[i]);
3362         }
3363
3364         /*
3365          * Create a DMA tag for page mbufs.
3366          */
3367         max_size = max_seg_size = ((sc->pg_bd_mbuf_alloc_size < MCLBYTES) ?
3368                 MCLBYTES : sc->pg_bd_mbuf_alloc_size);
3369
3370         if (bus_dma_tag_create(sc->parent_tag,
3371                         1,
3372                         BCE_DMA_BOUNDARY,
3373                         sc->max_bus_addr,
3374                         BUS_SPACE_MAXADDR,
3375                         NULL, NULL,
3376                         max_size,
3377                         1,
3378                         max_seg_size,
3379                         0,
3380                         NULL, NULL,
3381                 &sc->pg_mbuf_tag)) {
3382                 BCE_PRINTF("%s(%d): Could not allocate page mbuf DMA tag!\n",
3383                         __FILE__, __LINE__);
3384                 rc = ENOMEM;
3385                 goto bce_dma_alloc_exit;
3386         }
3387
3388         /* Create DMA maps for the page mbuf clusters. */
3389         for (i = 0; i < TOTAL_PG_BD; i++) {
3390                 if (bus_dmamap_create(sc->pg_mbuf_tag, BUS_DMA_NOWAIT,
3391                                 &sc->pg_mbuf_map[i])) {
3392                         BCE_PRINTF("%s(%d): Unable to create page mbuf DMA map!\n",
3393                                 __FILE__, __LINE__);
3394                         rc = ENOMEM;
3395                         goto bce_dma_alloc_exit;
3396                 }
3397         }
3398 #endif
3399
3400 bce_dma_alloc_exit:
3401         DBEXIT(BCE_VERBOSE_RESET | BCE_VERBOSE_CTX);
3402         return(rc);
3403 }
3404
3405
3406 /****************************************************************************/
3407 /* Release all resources used by the driver.                                */
3408 /*                                                                          */
3409 /* Releases all resources acquired by the driver including interrupts,      */
3410 /* interrupt handler, interfaces, mutexes, and DMA memory.                  */
3411 /*                                                                          */
3412 /* Returns:                                                                 */
3413 /*   Nothing.                                                               */
3414 /****************************************************************************/
3415 static void
3416 bce_release_resources(struct bce_softc *sc)
3417 {
3418         device_t dev;
3419
3420         DBENTER(BCE_VERBOSE_RESET);
3421
3422         dev = sc->bce_dev;
3423
3424         bce_dma_free(sc);
3425
3426         if (sc->bce_intrhand != NULL) {
3427                 DBPRINT(sc, BCE_INFO_RESET, "Removing interrupt handler.\n");
3428                 bus_teardown_intr(dev, sc->bce_res_irq, sc->bce_intrhand);
3429         }
3430
3431         if (sc->bce_res_irq != NULL) {
3432                 DBPRINT(sc, BCE_INFO_RESET, "Releasing IRQ.\n");
3433                 bus_release_resource(dev, SYS_RES_IRQ, sc->bce_irq_rid,
3434                         sc->bce_res_irq);
3435         }
3436
3437         if (sc->bce_flags & (BCE_USING_MSI_FLAG | BCE_USING_MSIX_FLAG)) {
3438                 DBPRINT(sc, BCE_INFO_RESET, "Releasing MSI/MSI-X vector.\n");
3439                 pci_release_msi(dev);
3440         }
3441
3442         if (sc->bce_res_mem != NULL) {
3443                 DBPRINT(sc, BCE_INFO_RESET, "Releasing PCI memory.\n");
3444                 bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(0), sc->bce_res_mem);
3445         }
3446
3447         if (sc->bce_ifp != NULL) {
3448                 DBPRINT(sc, BCE_INFO_RESET, "Releasing IF.\n");
3449                 if_free(sc->bce_ifp);
3450         }
3451
3452         if (mtx_initialized(&sc->bce_mtx))
3453                 BCE_LOCK_DESTROY(sc);
3454
3455         DBEXIT(BCE_VERBOSE_RESET);
3456 }
3457
3458
3459 /****************************************************************************/
3460 /* Firmware synchronization.                                                */
3461 /*                                                                          */
3462 /* Before performing certain events such as a chip reset, synchronize with  */
3463 /* the firmware first.                                                      */
3464 /*                                                                          */
3465 /* Returns:                                                                 */
3466 /*   0 for success, positive value for failure.                             */
3467 /****************************************************************************/
3468 static int
3469 bce_fw_sync(struct bce_softc *sc, u32 msg_data)
3470 {
3471         int i, rc = 0;
3472         u32 val;
3473
3474         DBENTER(BCE_VERBOSE_RESET);
3475
3476         /* Don't waste any time if we've timed out before. */
3477         if (sc->bce_fw_timed_out) {
3478                 rc = EBUSY;
3479                 goto bce_fw_sync_exit;
3480         }
3481
3482         /* Increment the message sequence number. */
3483         sc->bce_fw_wr_seq++;
3484         msg_data |= sc->bce_fw_wr_seq;
3485
3486         DBPRINT(sc, BCE_VERBOSE_FIRMWARE, "bce_fw_sync(): msg_data = 0x%08X\n",
3487                 msg_data);
3488
3489         /* Send the message to the bootcode driver mailbox. */
3490         REG_WR_IND(sc, sc->bce_shmem_base + BCE_DRV_MB, msg_data);
3491
3492         /* Wait for the bootcode to acknowledge the message. */
3493         for (i = 0; i < FW_ACK_TIME_OUT_MS; i++) {
3494                 /* Check for a response in the bootcode firmware mailbox. */
3495                 val = REG_RD_IND(sc, sc->bce_shmem_base + BCE_FW_MB);
3496                 if ((val & BCE_FW_MSG_ACK) == (msg_data & BCE_DRV_MSG_SEQ))
3497                         break;
3498                 DELAY(1000);
3499         }
3500
3501         /* If we've timed out, tell the bootcode that we've stopped waiting. */
3502         if (((val & BCE_FW_MSG_ACK) != (msg_data & BCE_DRV_MSG_SEQ)) &&
3503                 ((msg_data & BCE_DRV_MSG_DATA) != BCE_DRV_MSG_DATA_WAIT0)) {
3504
3505                 BCE_PRINTF("%s(%d): Firmware synchronization timeout! "
3506                         "msg_data = 0x%08X\n",
3507                         __FILE__, __LINE__, msg_data);
3508
3509                 msg_data &= ~BCE_DRV_MSG_CODE;
3510                 msg_data |= BCE_DRV_MSG_CODE_FW_TIMEOUT;
3511
3512                 REG_WR_IND(sc, sc->bce_shmem_base + BCE_DRV_MB, msg_data);
3513
3514                 sc->bce_fw_timed_out = 1;
3515                 rc = EBUSY;
3516         }
3517
3518 bce_fw_sync_exit:
3519         DBEXIT(BCE_VERBOSE_RESET);
3520         return (rc);
3521 }
3522
3523
3524 /****************************************************************************/
3525 /* Load Receive Virtual 2 Physical (RV2P) processor firmware.               */
3526 /*                                                                          */
3527 /* Returns:                                                                 */
3528 /*   Nothing.                                                               */
3529 /****************************************************************************/
3530 static void
3531 bce_load_rv2p_fw(struct bce_softc *sc, u32 *rv2p_code,
3532         u32 rv2p_code_len, u32 rv2p_proc)
3533 {
3534         int i;
3535         u32 val;
3536
3537         DBENTER(BCE_VERBOSE_RESET);
3538
3539         /* Set the page size used by RV2P. */
3540         if (rv2p_proc == RV2P_PROC2) {
3541                 BCE_RV2P_PROC2_CHG_MAX_BD_PAGE(USABLE_RX_BD_PER_PAGE);
3542         }
3543
3544         for (i = 0; i < rv2p_code_len; i += 8) {
3545                 REG_WR(sc, BCE_RV2P_INSTR_HIGH, *rv2p_code);
3546                 rv2p_code++;
3547                 REG_WR(sc, BCE_RV2P_INSTR_LOW, *rv2p_code);
3548                 rv2p_code++;
3549
3550                 if (rv2p_proc == RV2P_PROC1) {
3551                         val = (i / 8) | BCE_RV2P_PROC1_ADDR_CMD_RDWR;
3552                         REG_WR(sc, BCE_RV2P_PROC1_ADDR_CMD, val);
3553                 }
3554                 else {
3555                         val = (i / 8) | BCE_RV2P_PROC2_ADDR_CMD_RDWR;
3556                         REG_WR(sc, BCE_RV2P_PROC2_ADDR_CMD, val);
3557                 }
3558         }
3559
3560         /* Reset the processor, un-stall is done later. */
3561         if (rv2p_proc == RV2P_PROC1) {
3562                 REG_WR(sc, BCE_RV2P_COMMAND, BCE_RV2P_COMMAND_PROC1_RESET);
3563         }
3564         else {
3565                 REG_WR(sc, BCE_RV2P_COMMAND, BCE_RV2P_COMMAND_PROC2_RESET);
3566         }
3567
3568         DBEXIT(BCE_VERBOSE_RESET);
3569 }
3570
3571
3572 /****************************************************************************/
3573 /* Load RISC processor firmware.                                            */
3574 /*                                                                          */
3575 /* Loads firmware from the file if_bcefw.h into the scratchpad memory       */
3576 /* associated with a particular processor.                                  */
3577 /*                                                                          */
3578 /* Returns:                                                                 */
3579 /*   Nothing.                                                               */
3580 /****************************************************************************/
3581 static void
3582 bce_load_cpu_fw(struct bce_softc *sc, struct cpu_reg *cpu_reg,
3583         struct fw_info *fw)
3584 {
3585         u32 offset;
3586         u32 val;
3587
3588         DBENTER(BCE_VERBOSE_RESET);
3589
3590         /* Halt the CPU. */
3591         val = REG_RD_IND(sc, cpu_reg->mode);
3592         val |= cpu_reg->mode_value_halt;
3593         REG_WR_IND(sc, cpu_reg->mode, val);
3594         REG_WR_IND(sc, cpu_reg->state, cpu_reg->state_value_clear);
3595
3596         /* Load the Text area. */
3597         offset = cpu_reg->spad_base + (fw->text_addr - cpu_reg->mips_view_base);
3598         if (fw->text) {
3599                 int j;
3600
3601                 for (j = 0; j < (fw->text_len / 4); j++, offset += 4) {
3602                         REG_WR_IND(sc, offset, fw->text[j]);
3603                 }
3604         }
3605
3606         /* Load the Data area. */
3607         offset = cpu_reg->spad_base + (fw->data_addr - cpu_reg->mips_view_base);
3608         if (fw->data) {
3609                 int j;
3610
3611                 for (j = 0; j < (fw->data_len / 4); j++, offset += 4) {
3612                         REG_WR_IND(sc, offset, fw->data[j]);
3613                 }
3614         }
3615
3616         /* Load the SBSS area. */
3617         offset = cpu_reg->spad_base + (fw->sbss_addr - cpu_reg->mips_view_base);
3618         if (fw->sbss) {
3619                 int j;
3620
3621                 for (j = 0; j < (fw->sbss_len / 4); j++, offset += 4) {
3622                         REG_WR_IND(sc, offset, fw->sbss[j]);
3623                 }
3624         }
3625
3626         /* Load the BSS area. */
3627         offset = cpu_reg->spad_base + (fw->bss_addr - cpu_reg->mips_view_base);
3628         if (fw->bss) {
3629                 int j;
3630
3631                 for (j = 0; j < (fw->bss_len/4); j++, offset += 4) {
3632                         REG_WR_IND(sc, offset, fw->bss[j]);
3633                 }
3634         }
3635
3636         /* Load the Read-Only area. */
3637         offset = cpu_reg->spad_base +
3638                 (fw->rodata_addr - cpu_reg->mips_view_base);
3639         if (fw->rodata) {
3640                 int j;
3641
3642                 for (j = 0; j < (fw->rodata_len / 4); j++, offset += 4) {
3643                         REG_WR_IND(sc, offset, fw->rodata[j]);
3644                 }
3645         }
3646
3647         /* Clear the pre-fetch instruction. */
3648         REG_WR_IND(sc, cpu_reg->inst, 0);
3649         REG_WR_IND(sc, cpu_reg->pc, fw->start_addr);
3650
3651         /* Start the CPU. */
3652         val = REG_RD_IND(sc, cpu_reg->mode);
3653         val &= ~cpu_reg->mode_value_halt;
3654         REG_WR_IND(sc, cpu_reg->state, cpu_reg->state_value_clear);
3655         REG_WR_IND(sc, cpu_reg->mode, val);
3656
3657         DBEXIT(BCE_VERBOSE_RESET);
3658 }
3659
3660
3661 /****************************************************************************/
3662 /* Initialize the RX CPU.                                                   */
3663 /*                                                                          */
3664 /* Returns:                                                                 */
3665 /*   Nothing.                                                               */
3666 /****************************************************************************/
3667 static void
3668 bce_init_rxp_cpu(struct bce_softc *sc)
3669 {
3670         struct cpu_reg cpu_reg;
3671         struct fw_info fw;
3672
3673         DBENTER(BCE_VERBOSE_RESET);
3674
3675         cpu_reg.mode = BCE_RXP_CPU_MODE;
3676         cpu_reg.mode_value_halt = BCE_RXP_CPU_MODE_SOFT_HALT;
3677         cpu_reg.mode_value_sstep = BCE_RXP_CPU_MODE_STEP_ENA;
3678         cpu_reg.state = BCE_RXP_CPU_STATE;
3679         cpu_reg.state_value_clear = 0xffffff;
3680         cpu_reg.gpr0 = BCE_RXP_CPU_REG_FILE;
3681         cpu_reg.evmask = BCE_RXP_CPU_EVENT_MASK;
3682         cpu_reg.pc = BCE_RXP_CPU_PROGRAM_COUNTER;
3683         cpu_reg.inst = BCE_RXP_CPU_INSTRUCTION;
3684         cpu_reg.bp = BCE_RXP_CPU_HW_BREAKPOINT;
3685         cpu_reg.spad_base = BCE_RXP_SCRATCH;
3686         cpu_reg.mips_view_base = 0x8000000;
3687
3688         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
3689                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
3690                 fw.ver_major = bce_RXP_b09FwReleaseMajor;
3691                 fw.ver_minor = bce_RXP_b09FwReleaseMinor;
3692                 fw.ver_fix = bce_RXP_b09FwReleaseFix;
3693                 fw.start_addr = bce_RXP_b09FwStartAddr;
3694
3695                 fw.text_addr = bce_RXP_b09FwTextAddr;
3696                 fw.text_len = bce_RXP_b09FwTextLen;
3697                 fw.text_index = 0;
3698                 fw.text = bce_RXP_b09FwText;
3699
3700                 fw.data_addr = bce_RXP_b09FwDataAddr;
3701                 fw.data_len = bce_RXP_b09FwDataLen;
3702                 fw.data_index = 0;
3703                 fw.data = bce_RXP_b09FwData;
3704
3705                 fw.sbss_addr = bce_RXP_b09FwSbssAddr;
3706                 fw.sbss_len = bce_RXP_b09FwSbssLen;
3707                 fw.sbss_index = 0;
3708                 fw.sbss = bce_RXP_b09FwSbss;
3709
3710                 fw.bss_addr = bce_RXP_b09FwBssAddr;
3711                 fw.bss_len = bce_RXP_b09FwBssLen;
3712                 fw.bss_index = 0;
3713                 fw.bss = bce_RXP_b09FwBss;
3714
3715                 fw.rodata_addr = bce_RXP_b09FwRodataAddr;
3716                 fw.rodata_len = bce_RXP_b09FwRodataLen;
3717                 fw.rodata_index = 0;
3718                 fw.rodata = bce_RXP_b09FwRodata;
3719         } else {
3720                 fw.ver_major = bce_RXP_b06FwReleaseMajor;
3721                 fw.ver_minor = bce_RXP_b06FwReleaseMinor;
3722                 fw.ver_fix = bce_RXP_b06FwReleaseFix;
3723                 fw.start_addr = bce_RXP_b06FwStartAddr;
3724
3725                 fw.text_addr = bce_RXP_b06FwTextAddr;
3726                 fw.text_len = bce_RXP_b06FwTextLen;
3727                 fw.text_index = 0;
3728                 fw.text = bce_RXP_b06FwText;
3729
3730                 fw.data_addr = bce_RXP_b06FwDataAddr;
3731                 fw.data_len = bce_RXP_b06FwDataLen;
3732                 fw.data_index = 0;
3733                 fw.data = bce_RXP_b06FwData;
3734
3735                 fw.sbss_addr = bce_RXP_b06FwSbssAddr;
3736                 fw.sbss_len = bce_RXP_b06FwSbssLen;
3737                 fw.sbss_index = 0;
3738                 fw.sbss = bce_RXP_b06FwSbss;
3739
3740                 fw.bss_addr = bce_RXP_b06FwBssAddr;
3741                 fw.bss_len = bce_RXP_b06FwBssLen;
3742                 fw.bss_index = 0;
3743                 fw.bss = bce_RXP_b06FwBss;
3744
3745                 fw.rodata_addr = bce_RXP_b06FwRodataAddr;
3746                 fw.rodata_len = bce_RXP_b06FwRodataLen;
3747                 fw.rodata_index = 0;
3748                 fw.rodata = bce_RXP_b06FwRodata;
3749         }
3750
3751         DBPRINT(sc, BCE_INFO_RESET, "Loading RX firmware.\n");
3752         bce_load_cpu_fw(sc, &cpu_reg, &fw);
3753
3754         DBEXIT(BCE_VERBOSE_RESET);
3755 }
3756
3757
3758 /****************************************************************************/
3759 /* Initialize the TX CPU.                                                   */
3760 /*                                                                          */
3761 /* Returns:                                                                 */
3762 /*   Nothing.                                                               */
3763 /****************************************************************************/
3764 static void
3765 bce_init_txp_cpu(struct bce_softc *sc)
3766 {
3767         struct cpu_reg cpu_reg;
3768         struct fw_info fw;
3769
3770         DBENTER(BCE_VERBOSE_RESET);
3771
3772         cpu_reg.mode = BCE_TXP_CPU_MODE;
3773         cpu_reg.mode_value_halt = BCE_TXP_CPU_MODE_SOFT_HALT;
3774         cpu_reg.mode_value_sstep = BCE_TXP_CPU_MODE_STEP_ENA;
3775         cpu_reg.state = BCE_TXP_CPU_STATE;
3776         cpu_reg.state_value_clear = 0xffffff;
3777         cpu_reg.gpr0 = BCE_TXP_CPU_REG_FILE;
3778         cpu_reg.evmask = BCE_TXP_CPU_EVENT_MASK;
3779         cpu_reg.pc = BCE_TXP_CPU_PROGRAM_COUNTER;
3780         cpu_reg.inst = BCE_TXP_CPU_INSTRUCTION;
3781         cpu_reg.bp = BCE_TXP_CPU_HW_BREAKPOINT;
3782         cpu_reg.spad_base = BCE_TXP_SCRATCH;
3783         cpu_reg.mips_view_base = 0x8000000;
3784
3785         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
3786                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
3787                 fw.ver_major = bce_TXP_b09FwReleaseMajor;
3788                 fw.ver_minor = bce_TXP_b09FwReleaseMinor;
3789                 fw.ver_fix = bce_TXP_b09FwReleaseFix;
3790                 fw.start_addr = bce_TXP_b09FwStartAddr;
3791
3792                 fw.text_addr = bce_TXP_b09FwTextAddr;
3793                 fw.text_len = bce_TXP_b09FwTextLen;
3794                 fw.text_index = 0;
3795                 fw.text = bce_TXP_b09FwText;
3796
3797                 fw.data_addr = bce_TXP_b09FwDataAddr;
3798                 fw.data_len = bce_TXP_b09FwDataLen;
3799                 fw.data_index = 0;
3800                 fw.data = bce_TXP_b09FwData;
3801
3802                 fw.sbss_addr = bce_TXP_b09FwSbssAddr;
3803                 fw.sbss_len = bce_TXP_b09FwSbssLen;
3804                 fw.sbss_index = 0;
3805                 fw.sbss = bce_TXP_b09FwSbss;
3806
3807                 fw.bss_addr = bce_TXP_b09FwBssAddr;
3808                 fw.bss_len = bce_TXP_b09FwBssLen;
3809                 fw.bss_index = 0;
3810                 fw.bss = bce_TXP_b09FwBss;
3811
3812                 fw.rodata_addr = bce_TXP_b09FwRodataAddr;
3813                 fw.rodata_len = bce_TXP_b09FwRodataLen;
3814                 fw.rodata_index = 0;
3815                 fw.rodata = bce_TXP_b09FwRodata;
3816         } else {
3817                 fw.ver_major = bce_TXP_b06FwReleaseMajor;
3818                 fw.ver_minor = bce_TXP_b06FwReleaseMinor;
3819                 fw.ver_fix = bce_TXP_b06FwReleaseFix;
3820                 fw.start_addr = bce_TXP_b06FwStartAddr;
3821
3822                 fw.text_addr = bce_TXP_b06FwTextAddr;
3823                 fw.text_len = bce_TXP_b06FwTextLen;
3824                 fw.text_index = 0;
3825                 fw.text = bce_TXP_b06FwText;
3826
3827                 fw.data_addr = bce_TXP_b06FwDataAddr;
3828                 fw.data_len = bce_TXP_b06FwDataLen;
3829                 fw.data_index = 0;
3830                 fw.data = bce_TXP_b06FwData;
3831
3832                 fw.sbss_addr = bce_TXP_b06FwSbssAddr;
3833                 fw.sbss_len = bce_TXP_b06FwSbssLen;
3834                 fw.sbss_index = 0;
3835                 fw.sbss = bce_TXP_b06FwSbss;
3836
3837                 fw.bss_addr = bce_TXP_b06FwBssAddr;
3838                 fw.bss_len = bce_TXP_b06FwBssLen;
3839                 fw.bss_index = 0;
3840                 fw.bss = bce_TXP_b06FwBss;
3841
3842                 fw.rodata_addr = bce_TXP_b06FwRodataAddr;
3843                 fw.rodata_len = bce_TXP_b06FwRodataLen;
3844                 fw.rodata_index = 0;
3845                 fw.rodata = bce_TXP_b06FwRodata;
3846         }
3847
3848         DBPRINT(sc, BCE_INFO_RESET, "Loading TX firmware.\n");
3849         bce_load_cpu_fw(sc, &cpu_reg, &fw);
3850
3851         DBEXIT(BCE_VERBOSE_RESET);
3852 }
3853
3854
3855 /****************************************************************************/
3856 /* Initialize the TPAT CPU.                                                 */
3857 /*                                                                          */
3858 /* Returns:                                                                 */
3859 /*   Nothing.                                                               */
3860 /****************************************************************************/
3861 static void
3862 bce_init_tpat_cpu(struct bce_softc *sc)
3863 {
3864         struct cpu_reg cpu_reg;
3865         struct fw_info fw;
3866
3867         DBENTER(BCE_VERBOSE_RESET);
3868
3869         cpu_reg.mode = BCE_TPAT_CPU_MODE;
3870         cpu_reg.mode_value_halt = BCE_TPAT_CPU_MODE_SOFT_HALT;
3871         cpu_reg.mode_value_sstep = BCE_TPAT_CPU_MODE_STEP_ENA;
3872         cpu_reg.state = BCE_TPAT_CPU_STATE;
3873         cpu_reg.state_value_clear = 0xffffff;
3874         cpu_reg.gpr0 = BCE_TPAT_CPU_REG_FILE;
3875         cpu_reg.evmask = BCE_TPAT_CPU_EVENT_MASK;
3876         cpu_reg.pc = BCE_TPAT_CPU_PROGRAM_COUNTER;
3877         cpu_reg.inst = BCE_TPAT_CPU_INSTRUCTION;
3878         cpu_reg.bp = BCE_TPAT_CPU_HW_BREAKPOINT;
3879         cpu_reg.spad_base = BCE_TPAT_SCRATCH;
3880         cpu_reg.mips_view_base = 0x8000000;
3881
3882         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
3883                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
3884                 fw.ver_major = bce_TPAT_b09FwReleaseMajor;
3885                 fw.ver_minor = bce_TPAT_b09FwReleaseMinor;
3886                 fw.ver_fix = bce_TPAT_b09FwReleaseFix;
3887                 fw.start_addr = bce_TPAT_b09FwStartAddr;
3888
3889                 fw.text_addr = bce_TPAT_b09FwTextAddr;
3890                 fw.text_len = bce_TPAT_b09FwTextLen;
3891                 fw.text_index = 0;
3892                 fw.text = bce_TPAT_b09FwText;
3893
3894                 fw.data_addr = bce_TPAT_b09FwDataAddr;
3895                 fw.data_len = bce_TPAT_b09FwDataLen;
3896                 fw.data_index = 0;
3897                 fw.data = bce_TPAT_b09FwData;
3898
3899                 fw.sbss_addr = bce_TPAT_b09FwSbssAddr;
3900                 fw.sbss_len = bce_TPAT_b09FwSbssLen;
3901                 fw.sbss_index = 0;
3902                 fw.sbss = bce_TPAT_b09FwSbss;
3903
3904                 fw.bss_addr = bce_TPAT_b09FwBssAddr;
3905                 fw.bss_len = bce_TPAT_b09FwBssLen;
3906                 fw.bss_index = 0;
3907                 fw.bss = bce_TPAT_b09FwBss;
3908
3909                 fw.rodata_addr = bce_TPAT_b09FwRodataAddr;
3910                 fw.rodata_len = bce_TPAT_b09FwRodataLen;
3911                 fw.rodata_index = 0;
3912                 fw.rodata = bce_TPAT_b09FwRodata;
3913         } else {
3914                 fw.ver_major = bce_TPAT_b06FwReleaseMajor;
3915                 fw.ver_minor = bce_TPAT_b06FwReleaseMinor;
3916                 fw.ver_fix = bce_TPAT_b06FwReleaseFix;
3917                 fw.start_addr = bce_TPAT_b06FwStartAddr;
3918
3919                 fw.text_addr = bce_TPAT_b06FwTextAddr;
3920                 fw.text_len = bce_TPAT_b06FwTextLen;
3921                 fw.text_index = 0;
3922                 fw.text = bce_TPAT_b06FwText;
3923
3924                 fw.data_addr = bce_TPAT_b06FwDataAddr;
3925                 fw.data_len = bce_TPAT_b06FwDataLen;
3926                 fw.data_index = 0;
3927                 fw.data = bce_TPAT_b06FwData;
3928
3929                 fw.sbss_addr = bce_TPAT_b06FwSbssAddr;
3930                 fw.sbss_len = bce_TPAT_b06FwSbssLen;
3931                 fw.sbss_index = 0;
3932                 fw.sbss = bce_TPAT_b06FwSbss;
3933
3934                 fw.bss_addr = bce_TPAT_b06FwBssAddr;
3935                 fw.bss_len = bce_TPAT_b06FwBssLen;
3936                 fw.bss_index = 0;
3937                 fw.bss = bce_TPAT_b06FwBss;
3938
3939                 fw.rodata_addr = bce_TPAT_b06FwRodataAddr;
3940                 fw.rodata_len = bce_TPAT_b06FwRodataLen;
3941                 fw.rodata_index = 0;
3942                 fw.rodata = bce_TPAT_b06FwRodata;
3943         }
3944
3945         DBPRINT(sc, BCE_INFO_RESET, "Loading TPAT firmware.\n");
3946         bce_load_cpu_fw(sc, &cpu_reg, &fw);
3947
3948         DBEXIT(BCE_VERBOSE_RESET);
3949 }
3950
3951
3952 /****************************************************************************/
3953 /* Initialize the CP CPU.                                                   */
3954 /*                                                                          */
3955 /* Returns:                                                                 */
3956 /*   Nothing.                                                               */
3957 /****************************************************************************/
3958 static void
3959 bce_init_cp_cpu(struct bce_softc *sc)
3960 {
3961         struct cpu_reg cpu_reg;
3962         struct fw_info fw;
3963
3964         DBENTER(BCE_VERBOSE_RESET);
3965
3966         cpu_reg.mode = BCE_CP_CPU_MODE;
3967         cpu_reg.mode_value_halt = BCE_CP_CPU_MODE_SOFT_HALT;
3968         cpu_reg.mode_value_sstep = BCE_CP_CPU_MODE_STEP_ENA;
3969         cpu_reg.state = BCE_CP_CPU_STATE;
3970         cpu_reg.state_value_clear = 0xffffff;
3971         cpu_reg.gpr0 = BCE_CP_CPU_REG_FILE;
3972         cpu_reg.evmask = BCE_CP_CPU_EVENT_MASK;
3973         cpu_reg.pc = BCE_CP_CPU_PROGRAM_COUNTER;
3974         cpu_reg.inst = BCE_CP_CPU_INSTRUCTION;
3975         cpu_reg.bp = BCE_CP_CPU_HW_BREAKPOINT;
3976         cpu_reg.spad_base = BCE_CP_SCRATCH;
3977         cpu_reg.mips_view_base = 0x8000000;
3978
3979         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
3980                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
3981                 fw.ver_major = bce_CP_b09FwReleaseMajor;
3982                 fw.ver_minor = bce_CP_b09FwReleaseMinor;
3983                 fw.ver_fix = bce_CP_b09FwReleaseFix;
3984                 fw.start_addr = bce_CP_b09FwStartAddr;
3985
3986                 fw.text_addr = bce_CP_b09FwTextAddr;
3987                 fw.text_len = bce_CP_b09FwTextLen;
3988                 fw.text_index = 0;
3989                 fw.text = bce_CP_b09FwText;
3990
3991                 fw.data_addr = bce_CP_b09FwDataAddr;
3992                 fw.data_len = bce_CP_b09FwDataLen;
3993                 fw.data_index = 0;
3994                 fw.data = bce_CP_b09FwData;
3995
3996                 fw.sbss_addr = bce_CP_b09FwSbssAddr;
3997                 fw.sbss_len = bce_CP_b09FwSbssLen;
3998                 fw.sbss_index = 0;
3999                 fw.sbss = bce_CP_b09FwSbss;
4000
4001                 fw.bss_addr = bce_CP_b09FwBssAddr;
4002                 fw.bss_len = bce_CP_b09FwBssLen;
4003                 fw.bss_index = 0;
4004                 fw.bss = bce_CP_b09FwBss;
4005
4006                 fw.rodata_addr = bce_CP_b09FwRodataAddr;
4007                 fw.rodata_len = bce_CP_b09FwRodataLen;
4008                 fw.rodata_index = 0;
4009                 fw.rodata = bce_CP_b09FwRodata;
4010         } else {
4011                 fw.ver_major = bce_CP_b06FwReleaseMajor;
4012                 fw.ver_minor = bce_CP_b06FwReleaseMinor;
4013                 fw.ver_fix = bce_CP_b06FwReleaseFix;
4014                 fw.start_addr = bce_CP_b06FwStartAddr;
4015
4016                 fw.text_addr = bce_CP_b06FwTextAddr;
4017                 fw.text_len = bce_CP_b06FwTextLen;
4018                 fw.text_index = 0;
4019                 fw.text = bce_CP_b06FwText;
4020
4021                 fw.data_addr = bce_CP_b06FwDataAddr;
4022                 fw.data_len = bce_CP_b06FwDataLen;
4023                 fw.data_index = 0;
4024                 fw.data = bce_CP_b06FwData;
4025
4026                 fw.sbss_addr = bce_CP_b06FwSbssAddr;
4027                 fw.sbss_len = bce_CP_b06FwSbssLen;
4028                 fw.sbss_index = 0;
4029                 fw.sbss = bce_CP_b06FwSbss;
4030
4031                 fw.bss_addr = bce_CP_b06FwBssAddr;
4032                 fw.bss_len = bce_CP_b06FwBssLen;
4033                 fw.bss_index = 0;
4034                 fw.bss = bce_CP_b06FwBss;
4035
4036                 fw.rodata_addr = bce_CP_b06FwRodataAddr;
4037                 fw.rodata_len = bce_CP_b06FwRodataLen;
4038                 fw.rodata_index = 0;
4039                 fw.rodata = bce_CP_b06FwRodata;
4040         }
4041
4042         DBPRINT(sc, BCE_INFO_RESET, "Loading CP firmware.\n");
4043         bce_load_cpu_fw(sc, &cpu_reg, &fw);
4044
4045         DBEXIT(BCE_VERBOSE_RESET);
4046 }
4047
4048
4049 /****************************************************************************/
4050 /* Initialize the COM CPU.                                                 */
4051 /*                                                                          */
4052 /* Returns:                                                                 */
4053 /*   Nothing.                                                               */
4054 /****************************************************************************/
4055 static void
4056 bce_init_com_cpu(struct bce_softc *sc)
4057 {
4058         struct cpu_reg cpu_reg;
4059         struct fw_info fw;
4060
4061         DBENTER(BCE_VERBOSE_RESET);
4062
4063         cpu_reg.mode = BCE_COM_CPU_MODE;
4064         cpu_reg.mode_value_halt = BCE_COM_CPU_MODE_SOFT_HALT;
4065         cpu_reg.mode_value_sstep = BCE_COM_CPU_MODE_STEP_ENA;
4066         cpu_reg.state = BCE_COM_CPU_STATE;
4067         cpu_reg.state_value_clear = 0xffffff;
4068         cpu_reg.gpr0 = BCE_COM_CPU_REG_FILE;
4069         cpu_reg.evmask = BCE_COM_CPU_EVENT_MASK;
4070         cpu_reg.pc = BCE_COM_CPU_PROGRAM_COUNTER;
4071         cpu_reg.inst = BCE_COM_CPU_INSTRUCTION;
4072         cpu_reg.bp = BCE_COM_CPU_HW_BREAKPOINT;
4073         cpu_reg.spad_base = BCE_COM_SCRATCH;
4074         cpu_reg.mips_view_base = 0x8000000;
4075
4076         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
4077                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
4078                 fw.ver_major = bce_COM_b09FwReleaseMajor;
4079                 fw.ver_minor = bce_COM_b09FwReleaseMinor;
4080                 fw.ver_fix = bce_COM_b09FwReleaseFix;
4081                 fw.start_addr = bce_COM_b09FwStartAddr;
4082
4083                 fw.text_addr = bce_COM_b09FwTextAddr;
4084                 fw.text_len = bce_COM_b09FwTextLen;
4085                 fw.text_index = 0;
4086                 fw.text = bce_COM_b09FwText;
4087
4088                 fw.data_addr = bce_COM_b09FwDataAddr;
4089                 fw.data_len = bce_COM_b09FwDataLen;
4090                 fw.data_index = 0;
4091                 fw.data = bce_COM_b09FwData;
4092
4093                 fw.sbss_addr = bce_COM_b09FwSbssAddr;
4094                 fw.sbss_len = bce_COM_b09FwSbssLen;
4095                 fw.sbss_index = 0;
4096                 fw.sbss = bce_COM_b09FwSbss;
4097
4098                 fw.bss_addr = bce_COM_b09FwBssAddr;
4099                 fw.bss_len = bce_COM_b09FwBssLen;
4100                 fw.bss_index = 0;
4101                 fw.bss = bce_COM_b09FwBss;
4102
4103                 fw.rodata_addr = bce_COM_b09FwRodataAddr;
4104                 fw.rodata_len = bce_COM_b09FwRodataLen;
4105                 fw.rodata_index = 0;
4106                 fw.rodata = bce_COM_b09FwRodata;
4107         } else {
4108                 fw.ver_major = bce_COM_b06FwReleaseMajor;
4109                 fw.ver_minor = bce_COM_b06FwReleaseMinor;
4110                 fw.ver_fix = bce_COM_b06FwReleaseFix;
4111                 fw.start_addr = bce_COM_b06FwStartAddr;
4112
4113                 fw.text_addr = bce_COM_b06FwTextAddr;
4114                 fw.text_len = bce_COM_b06FwTextLen;
4115                 fw.text_index = 0;
4116                 fw.text = bce_COM_b06FwText;
4117
4118                 fw.data_addr = bce_COM_b06FwDataAddr;
4119                 fw.data_len = bce_COM_b06FwDataLen;
4120                 fw.data_index = 0;
4121                 fw.data = bce_COM_b06FwData;
4122
4123                 fw.sbss_addr = bce_COM_b06FwSbssAddr;
4124                 fw.sbss_len = bce_COM_b06FwSbssLen;
4125                 fw.sbss_index = 0;
4126                 fw.sbss = bce_COM_b06FwSbss;
4127
4128                 fw.bss_addr = bce_COM_b06FwBssAddr;
4129                 fw.bss_len = bce_COM_b06FwBssLen;
4130                 fw.bss_index = 0;
4131                 fw.bss = bce_COM_b06FwBss;
4132
4133                 fw.rodata_addr = bce_COM_b06FwRodataAddr;
4134                 fw.rodata_len = bce_COM_b06FwRodataLen;
4135                 fw.rodata_index = 0;
4136                 fw.rodata = bce_COM_b06FwRodata;
4137         }
4138
4139         DBPRINT(sc, BCE_INFO_RESET, "Loading COM firmware.\n");
4140         bce_load_cpu_fw(sc, &cpu_reg, &fw);
4141
4142         DBEXIT(BCE_VERBOSE_RESET);
4143 }
4144
4145
4146 /****************************************************************************/
4147 /* Initialize the RV2P, RX, TX, TPAT, COM, and CP CPUs.                     */
4148 /*                                                                          */
4149 /* Loads the firmware for each CPU and starts the CPU.                      */
4150 /*                                                                          */
4151 /* Returns:                                                                 */
4152 /*   Nothing.                                                               */
4153 /****************************************************************************/
4154 static void
4155 bce_init_cpus(struct bce_softc *sc)
4156 {
4157         DBENTER(BCE_VERBOSE_RESET);
4158
4159         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
4160                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
4161
4162                 if ((BCE_CHIP_REV(sc) == BCE_CHIP_REV_Ax)) {
4163                         bce_load_rv2p_fw(sc, bce_xi90_rv2p_proc1,
4164                                 sizeof(bce_xi90_rv2p_proc1), RV2P_PROC1);
4165                         bce_load_rv2p_fw(sc, bce_xi90_rv2p_proc2,
4166                                 sizeof(bce_xi90_rv2p_proc2), RV2P_PROC2);
4167                 } else {
4168                         bce_load_rv2p_fw(sc, bce_xi_rv2p_proc1,
4169                                 sizeof(bce_xi_rv2p_proc1), RV2P_PROC1);
4170                         bce_load_rv2p_fw(sc, bce_xi_rv2p_proc2,
4171                                 sizeof(bce_xi_rv2p_proc2), RV2P_PROC2);
4172                 }
4173
4174         } else {
4175                 bce_load_rv2p_fw(sc, bce_rv2p_proc1,
4176                         sizeof(bce_rv2p_proc1), RV2P_PROC1);
4177                 bce_load_rv2p_fw(sc, bce_rv2p_proc2,
4178                         sizeof(bce_rv2p_proc2), RV2P_PROC2);
4179         }
4180
4181         bce_init_rxp_cpu(sc);
4182         bce_init_txp_cpu(sc);
4183         bce_init_tpat_cpu(sc);
4184         bce_init_com_cpu(sc);
4185         bce_init_cp_cpu(sc);
4186
4187         DBEXIT(BCE_VERBOSE_RESET);
4188 }
4189
4190
4191 /****************************************************************************/
4192 /* Initialize context memory.                                               */
4193 /*                                                                          */
4194 /* Clears the memory associated with each Context ID (CID).                 */
4195 /*                                                                          */
4196 /* Returns:                                                                 */
4197 /*   Nothing.                                                               */
4198 /****************************************************************************/
4199 static void
4200 bce_init_ctx(struct bce_softc *sc)
4201 {
4202
4203         DBENTER(BCE_VERBOSE_RESET | BCE_VERBOSE_CTX);
4204
4205         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
4206                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
4207                 /* DRC: Replace this constant value with a #define. */
4208                 int i, retry_cnt = 10;
4209                 u32 val;
4210
4211                 DBPRINT(sc, BCE_INFO_CTX, "Initializing 5709 context.\n");
4212
4213                 /*
4214                  * BCM5709 context memory may be cached
4215                  * in host memory so prepare the host memory
4216                  * for access.
4217                  */
4218                 val = BCE_CTX_COMMAND_ENABLED | BCE_CTX_COMMAND_MEM_INIT | (1 << 12);
4219                 val |= (BCM_PAGE_BITS - 8) << 16;
4220                 REG_WR(sc, BCE_CTX_COMMAND, val);
4221
4222                 /* Wait for mem init command to complete. */
4223                 for (i = 0; i < retry_cnt; i++) {
4224                         val = REG_RD(sc, BCE_CTX_COMMAND);
4225                         if (!(val & BCE_CTX_COMMAND_MEM_INIT))
4226                                 break;
4227                         DELAY(2);
4228                 }
4229
4230                 /* ToDo: Consider returning an error here. */
4231                 DBRUNIF((val & BCE_CTX_COMMAND_MEM_INIT),
4232                         BCE_PRINTF("%s(): Context memory initialization failed!\n",
4233                         __FUNCTION__));
4234
4235                 for (i = 0; i < sc->ctx_pages; i++) {
4236                         int j;
4237
4238                         /* Set the physical address of the context memory cache. */
4239                         REG_WR(sc, BCE_CTX_HOST_PAGE_TBL_DATA0,
4240                                 BCE_ADDR_LO(sc->ctx_paddr[i] & 0xfffffff0) |
4241                                 BCE_CTX_HOST_PAGE_TBL_DATA0_VALID);
4242                         REG_WR(sc, BCE_CTX_HOST_PAGE_TBL_DATA1,
4243                                 BCE_ADDR_HI(sc->ctx_paddr[i]));
4244                         REG_WR(sc, BCE_CTX_HOST_PAGE_TBL_CTRL, i |
4245                                 BCE_CTX_HOST_PAGE_TBL_CTRL_WRITE_REQ);
4246
4247                         /* Verify that the context memory write was successful. */
4248                         for (j = 0; j < retry_cnt; j++) {
4249                                 val = REG_RD(sc, BCE_CTX_HOST_PAGE_TBL_CTRL);
4250                                 if ((val & BCE_CTX_HOST_PAGE_TBL_CTRL_WRITE_REQ) == 0)
4251                                         break;
4252                                 DELAY(5);
4253                         }
4254
4255                         /* ToDo: Consider returning an error here. */
4256                         DBRUNIF((val & BCE_CTX_HOST_PAGE_TBL_CTRL_WRITE_REQ),
4257                                 BCE_PRINTF("%s(): Failed to initialize context page %d!\n",
4258                                 __FUNCTION__, i));
4259                 }
4260         } else {
4261                 u32 vcid_addr, offset;
4262
4263                 DBPRINT(sc, BCE_INFO, "Initializing 5706/5708 context.\n");
4264
4265                 /*
4266                  * For the 5706/5708, context memory is local to
4267                  * the controller, so initialize the controller
4268                  * context memory.
4269                  */
4270
4271                 vcid_addr = GET_CID_ADDR(96);
4272                 while (vcid_addr) {
4273
4274                         vcid_addr -= PHY_CTX_SIZE;
4275
4276                         REG_WR(sc, BCE_CTX_VIRT_ADDR, 0);
4277                         REG_WR(sc, BCE_CTX_PAGE_TBL, vcid_addr);
4278
4279             for(offset = 0; offset < PHY_CTX_SIZE; offset += 4) {
4280                 CTX_WR(sc, 0x00, offset, 0);
4281             }
4282
4283                         REG_WR(sc, BCE_CTX_VIRT_ADDR, vcid_addr);
4284                         REG_WR(sc, BCE_CTX_PAGE_TBL, vcid_addr);
4285                 }
4286
4287         }
4288         DBEXIT(BCE_VERBOSE_RESET | BCE_VERBOSE_CTX);
4289 }
4290
4291
4292 /****************************************************************************/
4293 /* Fetch the permanent MAC address of the controller.                       */
4294 /*                                                                          */
4295 /* Returns:                                                                 */
4296 /*   Nothing.                                                               */
4297 /****************************************************************************/
4298 static void
4299 bce_get_mac_addr(struct bce_softc *sc)
4300 {
4301         u32 mac_lo = 0, mac_hi = 0;
4302
4303         DBENTER(BCE_VERBOSE_RESET);
4304         /*
4305          * The NetXtreme II bootcode populates various NIC
4306          * power-on and runtime configuration items in a
4307          * shared memory area.  The factory configured MAC
4308          * address is available from both NVRAM and the
4309          * shared memory area so we'll read the value from
4310          * shared memory for speed.
4311          */
4312
4313         mac_hi = REG_RD_IND(sc, sc->bce_shmem_base +
4314                 BCE_PORT_HW_CFG_MAC_UPPER);
4315         mac_lo = REG_RD_IND(sc, sc->bce_shmem_base +
4316                 BCE_PORT_HW_CFG_MAC_LOWER);
4317
4318         if ((mac_lo == 0) && (mac_hi == 0)) {
4319                 BCE_PRINTF("%s(%d): Invalid Ethernet address!\n",
4320                         __FILE__, __LINE__);
4321         } else {
4322                 sc->eaddr[0] = (u_char)(mac_hi >> 8);
4323                 sc->eaddr[1] = (u_char)(mac_hi >> 0);
4324                 sc->eaddr[2] = (u_char)(mac_lo >> 24);
4325                 sc->eaddr[3] = (u_char)(mac_lo >> 16);
4326                 sc->eaddr[4] = (u_char)(mac_lo >> 8);
4327                 sc->eaddr[5] = (u_char)(mac_lo >> 0);
4328         }
4329
4330         DBPRINT(sc, BCE_INFO_MISC, "Permanent Ethernet address = %6D\n", sc->eaddr, ":");
4331         DBEXIT(BCE_VERBOSE_RESET);
4332 }
4333
4334
4335 /****************************************************************************/
4336 /* Program the MAC address.                                                 */
4337 /*                                                                          */
4338 /* Returns:                                                                 */
4339 /*   Nothing.                                                               */
4340 /****************************************************************************/
4341 static void
4342 bce_set_mac_addr(struct bce_softc *sc)
4343 {
4344         u32 val;
4345         u8 *mac_addr = sc->eaddr;
4346
4347         /* ToDo: Add support for setting multiple MAC addresses. */
4348
4349         DBENTER(BCE_VERBOSE_RESET);
4350         DBPRINT(sc, BCE_INFO_MISC, "Setting Ethernet address = %6D\n", sc->eaddr, ":");
4351
4352         val = (mac_addr[0] << 8) | mac_addr[1];
4353
4354         REG_WR(sc, BCE_EMAC_MAC_MATCH0, val);
4355
4356         val = (mac_addr[2] << 24) | (mac_addr[3] << 16) |
4357                 (mac_addr[4] << 8) | mac_addr[5];
4358
4359         REG_WR(sc, BCE_EMAC_MAC_MATCH1, val);
4360
4361         DBEXIT(BCE_VERBOSE_RESET);
4362 }
4363
4364
4365 /****************************************************************************/
4366 /* Stop the controller.                                                     */
4367 /*                                                                          */
4368 /* Returns:                                                                 */
4369 /*   Nothing.                                                               */
4370 /****************************************************************************/
4371 static void
4372 bce_stop(struct bce_softc *sc)
4373 {
4374         struct ifnet *ifp;
4375         struct ifmedia_entry *ifm;
4376         struct mii_data *mii = NULL;
4377         int mtmp, itmp;
4378
4379         DBENTER(BCE_VERBOSE_RESET);
4380
4381         BCE_LOCK_ASSERT(sc);
4382
4383         ifp = sc->bce_ifp;
4384
4385         mii = device_get_softc(sc->bce_miibus);
4386
4387         callout_stop(&sc->bce_tick_callout);
4388
4389         /* Disable the transmit/receive blocks. */
4390         REG_WR(sc, BCE_MISC_ENABLE_CLR_BITS, BCE_MISC_ENABLE_CLR_DEFAULT);
4391         REG_RD(sc, BCE_MISC_ENABLE_CLR_BITS);
4392         DELAY(20);
4393
4394         bce_disable_intr(sc);
4395
4396         /* Free RX buffers. */
4397 #ifdef ZERO_COPY_SOCKETS
4398         bce_free_pg_chain(sc);
4399 #endif
4400         bce_free_rx_chain(sc);
4401
4402         /* Free TX buffers. */
4403         bce_free_tx_chain(sc);
4404
4405         /*
4406          * Isolate/power down the PHY, but leave the media selection
4407          * unchanged so that things will be put back to normal when
4408          * we bring the interface back up.
4409          */
4410
4411         itmp = ifp->if_flags;
4412         ifp->if_flags |= IFF_UP;
4413
4414         /* If we are called from bce_detach(), mii is already NULL. */
4415         if (mii != NULL) {
4416                 ifm = mii->mii_media.ifm_cur;
4417                 mtmp = ifm->ifm_media;
4418                 ifm->ifm_media = IFM_ETHER | IFM_NONE;
4419                 mii_mediachg(mii);
4420                 ifm->ifm_media = mtmp;
4421         }
4422
4423         ifp->if_flags = itmp;
4424         sc->watchdog_timer = 0;
4425
4426         sc->bce_link = 0;
4427
4428         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
4429
4430         DBEXIT(BCE_VERBOSE_RESET);
4431 }
4432
4433
4434 static int
4435 bce_reset(struct bce_softc *sc, u32 reset_code)
4436 {
4437         u32 val;
4438         int i, rc = 0;
4439
4440         DBENTER(BCE_VERBOSE_RESET);
4441
4442         DBPRINT(sc, BCE_VERBOSE_RESET, "%s(): reset_code = 0x%08X\n",
4443                 __FUNCTION__, reset_code);
4444
4445         /* Wait for pending PCI transactions to complete. */
4446         REG_WR(sc, BCE_MISC_ENABLE_CLR_BITS,
4447                BCE_MISC_ENABLE_CLR_BITS_TX_DMA_ENABLE |
4448                BCE_MISC_ENABLE_CLR_BITS_DMA_ENGINE_ENABLE |
4449                BCE_MISC_ENABLE_CLR_BITS_RX_DMA_ENABLE |
4450                BCE_MISC_ENABLE_CLR_BITS_HOST_COALESCE_ENABLE);
4451         val = REG_RD(sc, BCE_MISC_ENABLE_CLR_BITS);
4452         DELAY(5);
4453
4454         /* Disable DMA */
4455         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
4456                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
4457                 val = REG_RD(sc, BCE_MISC_NEW_CORE_CTL);
4458                 val &= ~BCE_MISC_NEW_CORE_CTL_DMA_ENABLE;
4459                 REG_WR(sc, BCE_MISC_NEW_CORE_CTL, val);
4460         }
4461
4462         /* Assume bootcode is running. */
4463         sc->bce_fw_timed_out = 0;
4464
4465         /* Give the firmware a chance to prepare for the reset. */
4466         rc = bce_fw_sync(sc, BCE_DRV_MSG_DATA_WAIT0 | reset_code);
4467         if (rc)
4468                 goto bce_reset_exit;
4469
4470         /* Set a firmware reminder that this is a soft reset. */
4471         REG_WR_IND(sc, sc->bce_shmem_base + BCE_DRV_RESET_SIGNATURE,
4472                    BCE_DRV_RESET_SIGNATURE_MAGIC);
4473
4474         /* Dummy read to force the chip to complete all current transactions. */
4475         val = REG_RD(sc, BCE_MISC_ID);
4476
4477         /* Chip reset. */
4478         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
4479                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
4480                 REG_WR(sc, BCE_MISC_COMMAND, BCE_MISC_COMMAND_SW_RESET);
4481                 REG_RD(sc, BCE_MISC_COMMAND);
4482                 DELAY(5);
4483
4484                 val = BCE_PCICFG_MISC_CONFIG_REG_WINDOW_ENA |
4485                       BCE_PCICFG_MISC_CONFIG_TARGET_MB_WORD_SWAP;
4486
4487                 pci_write_config(sc->bce_dev, BCE_PCICFG_MISC_CONFIG, val, 4);
4488         } else {
4489                 val = BCE_PCICFG_MISC_CONFIG_CORE_RST_REQ |
4490                         BCE_PCICFG_MISC_CONFIG_REG_WINDOW_ENA |
4491                         BCE_PCICFG_MISC_CONFIG_TARGET_MB_WORD_SWAP;
4492                 REG_WR(sc, BCE_PCICFG_MISC_CONFIG, val);
4493
4494                 /* Allow up to 30us for reset to complete. */
4495                 for (i = 0; i < 10; i++) {
4496                         val = REG_RD(sc, BCE_PCICFG_MISC_CONFIG);
4497                         if ((val & (BCE_PCICFG_MISC_CONFIG_CORE_RST_REQ |
4498                                 BCE_PCICFG_MISC_CONFIG_CORE_RST_BSY)) == 0) {
4499                                 break;
4500                         }
4501                         DELAY(10);
4502                 }
4503
4504                 /* Check that reset completed successfully. */
4505                 if (val & (BCE_PCICFG_MISC_CONFIG_CORE_RST_REQ |
4506                         BCE_PCICFG_MISC_CONFIG_CORE_RST_BSY)) {
4507                         BCE_PRINTF("%s(%d): Reset failed!\n",
4508                                 __FILE__, __LINE__);
4509                         rc = EBUSY;
4510                         goto bce_reset_exit;
4511                 }
4512         }
4513
4514         /* Make sure byte swapping is properly configured. */
4515         val = REG_RD(sc, BCE_PCI_SWAP_DIAG0);
4516         if (val != 0x01020304) {
4517                 BCE_PRINTF("%s(%d): Byte swap is incorrect!\n",
4518                         __FILE__, __LINE__);
4519                 rc = ENODEV;
4520                 goto bce_reset_exit;
4521         }
4522
4523         /* Just completed a reset, assume that firmware is running again. */
4524         sc->bce_fw_timed_out = 0;
4525
4526         /* Wait for the firmware to finish its initialization. */
4527         rc = bce_fw_sync(sc, BCE_DRV_MSG_DATA_WAIT1 | reset_code);
4528         if (rc)
4529                 BCE_PRINTF("%s(%d): Firmware did not complete initialization!\n",
4530                         __FILE__, __LINE__);
4531
4532 bce_reset_exit:
4533         DBEXIT(BCE_VERBOSE_RESET);
4534         return (rc);
4535 }
4536
4537
4538 static int
4539 bce_chipinit(struct bce_softc *sc)
4540 {
4541         u32 val;
4542         int rc = 0;
4543
4544         DBENTER(BCE_VERBOSE_RESET);
4545
4546         bce_disable_intr(sc);
4547
4548         /*
4549          * Initialize DMA byte/word swapping, configure the number of DMA
4550          * channels and PCI clock compensation delay.
4551          */
4552         val = BCE_DMA_CONFIG_DATA_BYTE_SWAP |
4553               BCE_DMA_CONFIG_DATA_WORD_SWAP |
4554 #if BYTE_ORDER == BIG_ENDIAN
4555               BCE_DMA_CONFIG_CNTL_BYTE_SWAP |
4556 #endif
4557               BCE_DMA_CONFIG_CNTL_WORD_SWAP |
4558               DMA_READ_CHANS << 12 |
4559               DMA_WRITE_CHANS << 16;
4560
4561         val |= (0x2 << 20) | BCE_DMA_CONFIG_CNTL_PCI_COMP_DLY;
4562
4563         if ((sc->bce_flags & BCE_PCIX_FLAG) && (sc->bus_speed_mhz == 133))
4564                 val |= BCE_DMA_CONFIG_PCI_FAST_CLK_CMP;
4565
4566         /*
4567          * This setting resolves a problem observed on certain Intel PCI
4568          * chipsets that cannot handle multiple outstanding DMA operations.
4569          * See errata E9_5706A1_65.
4570          */
4571         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5706) &&
4572             (BCE_CHIP_ID(sc) != BCE_CHIP_ID_5706_A0) &&
4573             !(sc->bce_flags & BCE_PCIX_FLAG))
4574                 val |= BCE_DMA_CONFIG_CNTL_PING_PONG_DMA;
4575
4576         REG_WR(sc, BCE_DMA_CONFIG, val);
4577
4578         /* Enable the RX_V2P and Context state machines before access. */
4579         REG_WR(sc, BCE_MISC_ENABLE_SET_BITS,
4580                BCE_MISC_ENABLE_SET_BITS_HOST_COALESCE_ENABLE |
4581                BCE_MISC_ENABLE_STATUS_BITS_RX_V2P_ENABLE |
4582                BCE_MISC_ENABLE_STATUS_BITS_CONTEXT_ENABLE);
4583
4584         /* Initialize context mapping and zero out the quick contexts. */
4585         bce_init_ctx(sc);
4586
4587         /* Initialize the on-boards CPUs */
4588         bce_init_cpus(sc);
4589
4590         /* Prepare NVRAM for access. */
4591         if (bce_init_nvram(sc)) {
4592                 rc = ENODEV;
4593                 goto bce_chipinit_exit;
4594         }
4595
4596         /* Set the kernel bypass block size */
4597         val = REG_RD(sc, BCE_MQ_CONFIG);
4598         val &= ~BCE_MQ_CONFIG_KNL_BYP_BLK_SIZE;
4599         val |= BCE_MQ_CONFIG_KNL_BYP_BLK_SIZE_256;
4600
4601         /* Enable bins used on the 5709. */
4602         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
4603                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
4604                 val |= BCE_MQ_CONFIG_BIN_MQ_MODE;
4605                 if (BCE_CHIP_ID(sc) == BCE_CHIP_ID_5709_A1)
4606                         val |= BCE_MQ_CONFIG_HALT_DIS;
4607         }
4608
4609         REG_WR(sc, BCE_MQ_CONFIG, val);
4610
4611         val = 0x10000 + (MAX_CID_CNT * MB_KERNEL_CTX_SIZE);
4612         REG_WR(sc, BCE_MQ_KNL_BYP_WIND_START, val);
4613         REG_WR(sc, BCE_MQ_KNL_WIND_END, val);
4614
4615         /* Set the page size and clear the RV2P processor stall bits. */
4616         val = (BCM_PAGE_BITS - 8) << 24;
4617         REG_WR(sc, BCE_RV2P_CONFIG, val);
4618
4619         /* Configure page size. */
4620         val = REG_RD(sc, BCE_TBDR_CONFIG);
4621         val &= ~BCE_TBDR_CONFIG_PAGE_SIZE;
4622         val |= (BCM_PAGE_BITS - 8) << 24 | 0x40;
4623         REG_WR(sc, BCE_TBDR_CONFIG, val);
4624
4625         /* Set the perfect match control register to default. */
4626         REG_WR_IND(sc, BCE_RXP_PM_CTRL, 0);
4627
4628 bce_chipinit_exit:
4629         DBEXIT(BCE_VERBOSE_RESET);
4630
4631         return(rc);
4632 }
4633
4634
4635 /****************************************************************************/
4636 /* Initialize the controller in preparation to send/receive traffic.        */
4637 /*                                                                          */
4638 /* Returns:                                                                 */
4639 /*   0 for success, positive value for failure.                             */
4640 /****************************************************************************/
4641 static int
4642 bce_blockinit(struct bce_softc *sc)
4643 {
4644         u32 reg, val;
4645         int rc = 0;
4646
4647         DBENTER(BCE_VERBOSE_RESET);
4648
4649         /* Load the hardware default MAC address. */
4650         bce_set_mac_addr(sc);
4651
4652         /* Set the Ethernet backoff seed value */
4653         val = sc->eaddr[0]         + (sc->eaddr[1] << 8) +
4654               (sc->eaddr[2] << 16) + (sc->eaddr[3]     ) +
4655               (sc->eaddr[4] << 8)  + (sc->eaddr[5] << 16);
4656         REG_WR(sc, BCE_EMAC_BACKOFF_SEED, val);
4657
4658         sc->last_status_idx = 0;
4659         sc->rx_mode = BCE_EMAC_RX_MODE_SORT_MODE;
4660
4661         /* Set up link change interrupt generation. */
4662         REG_WR(sc, BCE_EMAC_ATTENTION_ENA, BCE_EMAC_ATTENTION_ENA_LINK);
4663
4664         /* Program the physical address of the status block. */
4665         REG_WR(sc, BCE_HC_STATUS_ADDR_L,
4666                 BCE_ADDR_LO(sc->status_block_paddr));
4667         REG_WR(sc, BCE_HC_STATUS_ADDR_H,
4668                 BCE_ADDR_HI(sc->status_block_paddr));
4669
4670         /* Program the physical address of the statistics block. */
4671         REG_WR(sc, BCE_HC_STATISTICS_ADDR_L,
4672                 BCE_ADDR_LO(sc->stats_block_paddr));
4673         REG_WR(sc, BCE_HC_STATISTICS_ADDR_H,
4674                 BCE_ADDR_HI(sc->stats_block_paddr));
4675
4676         /* Program various host coalescing parameters. */
4677         REG_WR(sc, BCE_HC_TX_QUICK_CONS_TRIP,
4678                 (sc->bce_tx_quick_cons_trip_int << 16) | sc->bce_tx_quick_cons_trip);
4679         REG_WR(sc, BCE_HC_RX_QUICK_CONS_TRIP,
4680                 (sc->bce_rx_quick_cons_trip_int << 16) | sc->bce_rx_quick_cons_trip);
4681         REG_WR(sc, BCE_HC_COMP_PROD_TRIP,
4682                 (sc->bce_comp_prod_trip_int << 16) | sc->bce_comp_prod_trip);
4683         REG_WR(sc, BCE_HC_TX_TICKS,
4684                 (sc->bce_tx_ticks_int << 16) | sc->bce_tx_ticks);
4685         REG_WR(sc, BCE_HC_RX_TICKS,
4686                 (sc->bce_rx_ticks_int << 16) | sc->bce_rx_ticks);
4687         REG_WR(sc, BCE_HC_COM_TICKS,
4688                 (sc->bce_com_ticks_int << 16) | sc->bce_com_ticks);
4689         REG_WR(sc, BCE_HC_CMD_TICKS,
4690                 (sc->bce_cmd_ticks_int << 16) | sc->bce_cmd_ticks);
4691         REG_WR(sc, BCE_HC_STATS_TICKS,
4692                 (sc->bce_stats_ticks & 0xffff00));
4693         REG_WR(sc, BCE_HC_STAT_COLLECT_TICKS, 0xbb8);  /* 3ms */
4694
4695         /* Configure the Host Coalescing block. */
4696         val = BCE_HC_CONFIG_RX_TMR_MODE | BCE_HC_CONFIG_TX_TMR_MODE |
4697                       BCE_HC_CONFIG_COLLECT_STATS;
4698
4699 #if 0
4700         /* ToDo: Add MSI-X support. */
4701         if (sc->bce_flags & BCE_USING_MSIX_FLAG) {
4702                 u32 base = ((BCE_TX_VEC - 1) * BCE_HC_SB_CONFIG_SIZE) +
4703                            BCE_HC_SB_CONFIG_1;
4704
4705                 REG_WR(sc, BCE_HC_MSIX_BIT_VECTOR, BCE_HC_MSIX_BIT_VECTOR_VAL);
4706
4707                 REG_WR(sc, base, BCE_HC_SB_CONFIG_1_TX_TMR_MODE |
4708                         BCE_HC_SB_CONFIG_1_ONE_SHOT);
4709
4710                 REG_WR(sc, base + BCE_HC_TX_QUICK_CONS_TRIP_OFF,
4711                         (sc->tx_quick_cons_trip_int << 16) |
4712                          sc->tx_quick_cons_trip);
4713
4714                 REG_WR(sc, base + BCE_HC_TX_TICKS_OFF,
4715                         (sc->tx_ticks_int << 16) | sc->tx_ticks);
4716
4717                 val |= BCE_HC_CONFIG_SB_ADDR_INC_128B;
4718         }
4719
4720         /*
4721          * Tell the HC block to automatically set the
4722          * INT_MASK bit after an MSI/MSI-X interrupt
4723          * is generated so the driver doesn't have to.
4724          */
4725         if (sc->bce_flags & BCE_ONE_SHOT_MSI_FLAG)
4726                 val |= BCE_HC_CONFIG_ONE_SHOT;
4727
4728         /* Set the MSI-X status blocks to 128 byte boundaries. */
4729         if (sc->bce_flags & BCE_USING_MSIX_FLAG)
4730                 val |= BCE_HC_CONFIG_SB_ADDR_INC_128B;
4731 #endif
4732
4733         REG_WR(sc, BCE_HC_CONFIG, val);
4734
4735         /* Clear the internal statistics counters. */
4736         REG_WR(sc, BCE_HC_COMMAND, BCE_HC_COMMAND_CLR_STAT_NOW);
4737
4738         /* Verify that bootcode is running. */
4739         reg = REG_RD_IND(sc, sc->bce_shmem_base + BCE_DEV_INFO_SIGNATURE);
4740
4741         DBRUNIF(DB_RANDOMTRUE(bootcode_running_failure_sim_control),
4742                 BCE_PRINTF("%s(%d): Simulating bootcode failure.\n",
4743                         __FILE__, __LINE__);
4744                 reg = 0);
4745
4746         if ((reg & BCE_DEV_INFO_SIGNATURE_MAGIC_MASK) !=
4747             BCE_DEV_INFO_SIGNATURE_MAGIC) {
4748                 BCE_PRINTF("%s(%d): Bootcode not running! Found: 0x%08X, "
4749                         "Expected: 08%08X\n", __FILE__, __LINE__,
4750                         (reg & BCE_DEV_INFO_SIGNATURE_MAGIC_MASK),
4751                         BCE_DEV_INFO_SIGNATURE_MAGIC);
4752                 rc = ENODEV;
4753                 goto bce_blockinit_exit;
4754         }
4755
4756         /* Enable DMA */
4757         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
4758                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
4759                 val = REG_RD(sc, BCE_MISC_NEW_CORE_CTL);
4760                 val |= BCE_MISC_NEW_CORE_CTL_DMA_ENABLE;
4761                 REG_WR(sc, BCE_MISC_NEW_CORE_CTL, val);
4762         }
4763
4764         /* Allow bootcode to apply any additional fixes before enabling MAC. */
4765         rc = bce_fw_sync(sc, BCE_DRV_MSG_DATA_WAIT2 | BCE_DRV_MSG_CODE_RESET);
4766
4767         /* Enable link state change interrupt generation. */
4768         REG_WR(sc, BCE_HC_ATTN_BITS_ENABLE, STATUS_ATTN_BITS_LINK_STATE);
4769
4770         /* Enable all remaining blocks in the MAC. */
4771         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709)     ||
4772                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716))
4773                 REG_WR(sc, BCE_MISC_ENABLE_SET_BITS, BCE_MISC_ENABLE_DEFAULT_XI);
4774         else
4775                 REG_WR(sc, BCE_MISC_ENABLE_SET_BITS, BCE_MISC_ENABLE_DEFAULT);
4776
4777         REG_RD(sc, BCE_MISC_ENABLE_SET_BITS);
4778         DELAY(20);
4779
4780         /* Save the current host coalescing block settings. */
4781         sc->hc_command = REG_RD(sc, BCE_HC_COMMAND);
4782
4783 bce_blockinit_exit:
4784         DBEXIT(BCE_VERBOSE_RESET);
4785
4786         return (rc);
4787 }
4788
4789
4790 /****************************************************************************/
4791 /* Encapsulate an mbuf into the rx_bd chain.                                */
4792 /*                                                                          */
4793 /* Returns:                                                                 */
4794 /*   0 for success, positive value for failure.                             */
4795 /****************************************************************************/
4796 static int
4797 bce_get_rx_buf(struct bce_softc *sc, struct mbuf *m, u16 *prod,
4798         u16 *chain_prod, u32 *prod_bseq)
4799 {
4800         bus_dmamap_t map;
4801         bus_dma_segment_t segs[BCE_MAX_SEGMENTS];
4802         struct mbuf *m_new = NULL;
4803         struct rx_bd *rxbd;
4804         int nsegs, error, rc = 0;
4805 #ifdef BCE_DEBUG
4806         u16 debug_chain_prod = *chain_prod;
4807 #endif
4808
4809         DBENTER(BCE_EXTREME_RESET | BCE_EXTREME_RECV | BCE_EXTREME_LOAD);
4810
4811         /* Make sure the inputs are valid. */
4812         DBRUNIF((*chain_prod > MAX_RX_BD),
4813                 BCE_PRINTF("%s(%d): RX producer out of range: 0x%04X > 0x%04X\n",
4814                 __FILE__, __LINE__, *chain_prod, (u16) MAX_RX_BD));
4815
4816         DBPRINT(sc, BCE_EXTREME_RECV, "%s(enter): prod = 0x%04X, chain_prod = 0x%04X, "
4817                 "prod_bseq = 0x%08X\n", __FUNCTION__, *prod, *chain_prod, *prod_bseq);
4818
4819         /* Update some debug statistic counters */
4820         DBRUNIF((sc->free_rx_bd < sc->rx_low_watermark),
4821                 sc->rx_low_watermark = sc->free_rx_bd);
4822         DBRUNIF((sc->free_rx_bd == sc->max_rx_bd), sc->rx_empty_count++);
4823
4824         /* Check whether this is a new mbuf allocation. */
4825         if (m == NULL) {
4826
4827                 /* Simulate an mbuf allocation failure. */
4828                 DBRUNIF(DB_RANDOMTRUE(mbuf_alloc_failed_sim_control),
4829                         sc->mbuf_alloc_failed_count++;
4830                         sc->mbuf_alloc_failed_sim_count++;
4831                         rc = ENOBUFS;
4832                         goto bce_get_rx_buf_exit);
4833
4834                 /* This is a new mbuf allocation. */
4835 #ifdef ZERO_COPY_SOCKETS
4836                 MGETHDR(m_new, M_DONTWAIT, MT_DATA);
4837 #else
4838                 if (sc->rx_bd_mbuf_alloc_size <= MCLBYTES)
4839                         m_new = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR);
4840                 else
4841                         m_new = m_getjcl(M_DONTWAIT, MT_DATA, M_PKTHDR, sc->rx_bd_mbuf_alloc_size);
4842 #endif
4843
4844                 if (m_new == NULL) {
4845                         sc->mbuf_alloc_failed_count++;
4846                         rc = ENOBUFS;
4847                         goto bce_get_rx_buf_exit;
4848                 }
4849
4850                 DBRUN(sc->debug_rx_mbuf_alloc++);
4851         } else {
4852                 /* Reuse an existing mbuf. */
4853                 m_new = m;
4854         }
4855
4856         /* Make sure we have a valid packet header. */
4857         M_ASSERTPKTHDR(m_new);
4858
4859         /* Initialize the mbuf size and pad if necessary for alignment. */
4860         m_new->m_pkthdr.len = m_new->m_len = sc->rx_bd_mbuf_alloc_size;
4861         m_adj(m_new, sc->rx_bd_mbuf_align_pad);
4862
4863         /* ToDo: Consider calling m_fragment() to test error handling. */
4864
4865         /* Map the mbuf cluster into device memory. */
4866         map = sc->rx_mbuf_map[*chain_prod];
4867         error = bus_dmamap_load_mbuf_sg(sc->rx_mbuf_tag, map, m_new,
4868             segs, &nsegs, BUS_DMA_NOWAIT);
4869
4870         /* Handle any mapping errors. */
4871         if (error) {
4872                 BCE_PRINTF("%s(%d): Error mapping mbuf into RX chain (%d)!\n",
4873                         __FILE__, __LINE__, error);
4874
4875                 sc->dma_map_addr_rx_failed_count++;
4876                 m_freem(m_new);
4877
4878                 DBRUN(sc->debug_rx_mbuf_alloc--);
4879
4880                 rc = ENOBUFS;
4881                 goto bce_get_rx_buf_exit;
4882         }
4883
4884         /* All mbufs must map to a single segment. */
4885         KASSERT(nsegs == 1, ("%s(): Too many segments returned (%d)!",
4886                  __FUNCTION__, nsegs));
4887
4888         /* ToDo: Do we need bus_dmamap_sync(,,BUS_DMASYNC_PREWRITE) here? */
4889
4890         /* Setup the rx_bd for the segment. */
4891         rxbd = &sc->rx_bd_chain[RX_PAGE(*chain_prod)][RX_IDX(*chain_prod)];
4892
4893         rxbd->rx_bd_haddr_lo  = htole32(BCE_ADDR_LO(segs[0].ds_addr));
4894         rxbd->rx_bd_haddr_hi  = htole32(BCE_ADDR_HI(segs[0].ds_addr));
4895         rxbd->rx_bd_len       = htole32(segs[0].ds_len);
4896         rxbd->rx_bd_flags     = htole32(RX_BD_FLAGS_START | RX_BD_FLAGS_END);
4897         *prod_bseq += segs[0].ds_len;
4898
4899         /* Save the mbuf and update our counter. */
4900         sc->rx_mbuf_ptr[*chain_prod] = m_new;
4901         sc->free_rx_bd -= nsegs;
4902
4903         DBRUNMSG(BCE_INSANE_RECV, bce_dump_rx_mbuf_chain(sc, debug_chain_prod,
4904                 nsegs));
4905
4906         DBPRINT(sc, BCE_EXTREME_RECV, "%s(exit): prod = 0x%04X, chain_prod = 0x%04X, "
4907                 "prod_bseq = 0x%08X\n", __FUNCTION__, *prod, *chain_prod, *prod_bseq);
4908
4909 bce_get_rx_buf_exit:
4910         DBEXIT(BCE_EXTREME_RESET | BCE_EXTREME_RECV | BCE_EXTREME_LOAD);
4911
4912         return(rc);
4913 }
4914
4915
4916 #ifdef ZERO_COPY_SOCKETS
4917 /****************************************************************************/
4918 /* Encapsulate an mbuf cluster into the page chain.                        */
4919 /*                                                                          */
4920 /* Returns:                                                                 */
4921 /*   0 for success, positive value for failure.                             */
4922 /****************************************************************************/
4923 static int
4924 bce_get_pg_buf(struct bce_softc *sc, struct mbuf *m, u16 *prod,
4925         u16 *prod_idx)
4926 {
4927         bus_dmamap_t map;
4928         bus_addr_t busaddr;
4929         struct mbuf *m_new = NULL;
4930         struct rx_bd *pgbd;
4931         int error, rc = 0;
4932 #ifdef BCE_DEBUG
4933         u16 debug_prod_idx = *prod_idx;
4934 #endif
4935
4936         DBENTER(BCE_EXTREME_RESET | BCE_EXTREME_RECV | BCE_EXTREME_LOAD);
4937
4938         /* Make sure the inputs are valid. */
4939         DBRUNIF((*prod_idx > MAX_PG_BD),
4940                 BCE_PRINTF("%s(%d): page producer out of range: 0x%04X > 0x%04X\n",
4941                 __FILE__, __LINE__, *prod_idx, (u16) MAX_PG_BD));
4942
4943         DBPRINT(sc, BCE_EXTREME_RECV, "%s(enter): prod = 0x%04X, "
4944                 "chain_prod = 0x%04X\n", __FUNCTION__, *prod, *prod_idx);
4945
4946         /* Update counters if we've hit a new low or run out of pages. */
4947         DBRUNIF((sc->free_pg_bd < sc->pg_low_watermark),
4948                 sc->pg_low_watermark = sc->free_pg_bd);
4949         DBRUNIF((sc->free_pg_bd == sc->max_pg_bd), sc->pg_empty_count++);
4950
4951         /* Check whether this is a new mbuf allocation. */
4952         if (m == NULL) {
4953
4954                 /* Simulate an mbuf allocation failure. */
4955                 DBRUNIF(DB_RANDOMTRUE(mbuf_alloc_failed_sim_control),
4956                         sc->mbuf_alloc_failed_count++;
4957                         sc->mbuf_alloc_failed_sim_count++;
4958                         rc = ENOBUFS;
4959                         goto bce_get_pg_buf_exit);
4960
4961                 /* This is a new mbuf allocation. */
4962                 m_new = m_getcl(M_DONTWAIT, MT_DATA, 0);
4963                 if (m_new == NULL) {
4964                         sc->mbuf_alloc_failed_count++;
4965                         rc = ENOBUFS;
4966                         goto bce_get_pg_buf_exit;
4967                 }
4968
4969                 DBRUN(sc->debug_pg_mbuf_alloc++);
4970         } else {
4971                 /* Reuse an existing mbuf. */
4972                 m_new = m;
4973                 m_new->m_data = m_new->m_ext.ext_buf;
4974         }
4975
4976         m_new->m_len = sc->pg_bd_mbuf_alloc_size;
4977
4978         /* ToDo: Consider calling m_fragment() to test error handling. */
4979
4980         /* Map the mbuf cluster into device memory. */
4981         map = sc->pg_mbuf_map[*prod_idx];
4982         error = bus_dmamap_load(sc->pg_mbuf_tag, map, mtod(m_new, void *),
4983             sc->pg_bd_mbuf_alloc_size, bce_dma_map_addr, &busaddr, BUS_DMA_NOWAIT);
4984
4985         /* Handle any mapping errors. */
4986         if (error) {
4987                 BCE_PRINTF("%s(%d): Error mapping mbuf into page chain!\n",
4988                         __FILE__, __LINE__);
4989
4990                 m_freem(m_new);
4991                 DBRUN(sc->debug_pg_mbuf_alloc--);
4992
4993                 rc = ENOBUFS;
4994                 goto bce_get_pg_buf_exit;
4995         }
4996
4997         /* ToDo: Do we need bus_dmamap_sync(,,BUS_DMASYNC_PREWRITE) here? */
4998
4999         /*
5000          * The page chain uses the same rx_bd data structure
5001          * as the receive chain but doesn't require a byte sequence (bseq).
5002          */
5003         pgbd = &sc->pg_bd_chain[PG_PAGE(*prod_idx)][PG_IDX(*prod_idx)];
5004
5005         pgbd->rx_bd_haddr_lo  = htole32(BCE_ADDR_LO(busaddr));
5006         pgbd->rx_bd_haddr_hi  = htole32(BCE_ADDR_HI(busaddr));
5007         pgbd->rx_bd_len       = htole32(sc->pg_bd_mbuf_alloc_size);
5008         pgbd->rx_bd_flags     = htole32(RX_BD_FLAGS_START | RX_BD_FLAGS_END);
5009
5010         /* Save the mbuf and update our counter. */
5011         sc->pg_mbuf_ptr[*prod_idx] = m_new;
5012         sc->free_pg_bd--;
5013
5014         DBRUNMSG(BCE_INSANE_RECV, bce_dump_pg_mbuf_chain(sc, debug_prod_idx,
5015                 1));
5016
5017         DBPRINT(sc, BCE_EXTREME_RECV, "%s(exit): prod = 0x%04X, "
5018                 "prod_idx = 0x%04X\n", __FUNCTION__, *prod, *prod_idx);
5019
5020 bce_get_pg_buf_exit:
5021         DBEXIT(BCE_EXTREME_RESET | BCE_EXTREME_RECV | BCE_EXTREME_LOAD);
5022
5023         return(rc);
5024 }
5025 #endif /* ZERO_COPY_SOCKETS */
5026
5027 /****************************************************************************/
5028 /* Initialize the TX context memory.                                        */
5029 /*                                                                          */
5030 /* Returns:                                                                 */
5031 /*   Nothing                                                                */
5032 /****************************************************************************/
5033 static void
5034 bce_init_tx_context(struct bce_softc *sc)
5035 {
5036         u32 val;
5037
5038         DBENTER(BCE_VERBOSE_RESET | BCE_VERBOSE_SEND | BCE_VERBOSE_CTX);
5039
5040         /* Initialize the context ID for an L2 TX chain. */
5041         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
5042                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
5043                 /* Set the CID type to support an L2 connection. */
5044                 val = BCE_L2CTX_TX_TYPE_TYPE_L2_XI | BCE_L2CTX_TX_TYPE_SIZE_L2_XI;
5045                 CTX_WR(sc, GET_CID_ADDR(TX_CID), BCE_L2CTX_TX_TYPE_XI, val);
5046                 val = BCE_L2CTX_TX_CMD_TYPE_TYPE_L2_XI | (8 << 16);
5047                 CTX_WR(sc, GET_CID_ADDR(TX_CID), BCE_L2CTX_TX_CMD_TYPE_XI, val);
5048
5049                 /* Point the hardware to the first page in the chain. */
5050                 val = BCE_ADDR_HI(sc->tx_bd_chain_paddr[0]);
5051                 CTX_WR(sc, GET_CID_ADDR(TX_CID), BCE_L2CTX_TX_TBDR_BHADDR_HI_XI, val);
5052                 val = BCE_ADDR_LO(sc->tx_bd_chain_paddr[0]);
5053                 CTX_WR(sc, GET_CID_ADDR(TX_CID), BCE_L2CTX_TX_TBDR_BHADDR_LO_XI, val);
5054         } else {
5055                 /* Set the CID type to support an L2 connection. */
5056                 val = BCE_L2CTX_TX_TYPE_TYPE_L2 | BCE_L2CTX_TX_TYPE_SIZE_L2;
5057                 CTX_WR(sc, GET_CID_ADDR(TX_CID), BCE_L2CTX_TX_TYPE, val);
5058                 val = BCE_L2CTX_TX_CMD_TYPE_TYPE_L2 | (8 << 16);
5059                 CTX_WR(sc, GET_CID_ADDR(TX_CID), BCE_L2CTX_TX_CMD_TYPE, val);
5060
5061                 /* Point the hardware to the first page in the chain. */
5062                 val = BCE_ADDR_HI(sc->tx_bd_chain_paddr[0]);
5063                 CTX_WR(sc, GET_CID_ADDR(TX_CID), BCE_L2CTX_TX_TBDR_BHADDR_HI, val);
5064                 val = BCE_ADDR_LO(sc->tx_bd_chain_paddr[0]);
5065                 CTX_WR(sc, GET_CID_ADDR(TX_CID), BCE_L2CTX_TX_TBDR_BHADDR_LO, val);
5066         }
5067
5068         DBEXIT(BCE_VERBOSE_RESET | BCE_VERBOSE_SEND | BCE_VERBOSE_CTX);
5069 }
5070
5071
5072 /****************************************************************************/
5073 /* Allocate memory and initialize the TX data structures.                   */
5074 /*                                                                          */
5075 /* Returns:                                                                 */
5076 /*   0 for success, positive value for failure.                             */
5077 /****************************************************************************/
5078 static int
5079 bce_init_tx_chain(struct bce_softc *sc)
5080 {
5081         struct tx_bd *txbd;
5082         int i, rc = 0;
5083
5084         DBENTER(BCE_VERBOSE_RESET | BCE_VERBOSE_SEND | BCE_VERBOSE_LOAD);
5085
5086         /* Set the initial TX producer/consumer indices. */
5087         sc->tx_prod        = 0;
5088         sc->tx_cons        = 0;
5089         sc->tx_prod_bseq   = 0;
5090         sc->used_tx_bd     = 0;
5091         sc->max_tx_bd      = USABLE_TX_BD;
5092         DBRUN(sc->tx_hi_watermark = USABLE_TX_BD);
5093         DBRUN(sc->tx_full_count = 0);
5094
5095         /*
5096          * The NetXtreme II supports a linked-list structre called
5097          * a Buffer Descriptor Chain (or BD chain).  A BD chain
5098          * consists of a series of 1 or more chain pages, each of which
5099          * consists of a fixed number of BD entries.
5100          * The last BD entry on each page is a pointer to the next page
5101          * in the chain, and the last pointer in the BD chain
5102          * points back to the beginning of the chain.
5103          */
5104
5105         /* Set the TX next pointer chain entries. */
5106         for (i = 0; i < TX_PAGES; i++) {
5107                 int j;
5108
5109                 txbd = &sc->tx_bd_chain[i][USABLE_TX_BD_PER_PAGE];
5110
5111                 /* Check if we've reached the last page. */
5112                 if (i == (TX_PAGES - 1))
5113                         j = 0;
5114                 else
5115                         j = i + 1;
5116
5117                 txbd->tx_bd_haddr_hi = htole32(BCE_ADDR_HI(sc->tx_bd_chain_paddr[j]));
5118                 txbd->tx_bd_haddr_lo = htole32(BCE_ADDR_LO(sc->tx_bd_chain_paddr[j]));
5119         }
5120
5121         bce_init_tx_context(sc);
5122
5123         DBRUNMSG(BCE_INSANE_SEND, bce_dump_tx_chain(sc, 0, TOTAL_TX_BD));
5124         DBEXIT(BCE_VERBOSE_RESET | BCE_VERBOSE_SEND | BCE_VERBOSE_LOAD);
5125
5126         return(rc);
5127 }
5128
5129
5130 /****************************************************************************/
5131 /* Free memory and clear the TX data structures.                            */
5132 /*                                                                          */
5133 /* Returns:                                                                 */
5134 /*   Nothing.                                                               */
5135 /****************************************************************************/
5136 static void
5137 bce_free_tx_chain(struct bce_softc *sc)
5138 {
5139         int i;
5140
5141         DBENTER(BCE_VERBOSE_RESET | BCE_VERBOSE_SEND | BCE_VERBOSE_UNLOAD);
5142
5143         /* Unmap, unload, and free any mbufs still in the TX mbuf chain. */
5144         for (i = 0; i < TOTAL_TX_BD; i++) {
5145                 if (sc->tx_mbuf_ptr[i] != NULL) {
5146                         if (sc->tx_mbuf_map[i] != NULL)
5147                                 bus_dmamap_sync(sc->tx_mbuf_tag, sc->tx_mbuf_map[i],
5148                                         BUS_DMASYNC_POSTWRITE);
5149                         m_freem(sc->tx_mbuf_ptr[i]);
5150                         sc->tx_mbuf_ptr[i] = NULL;
5151                         DBRUN(sc->debug_tx_mbuf_alloc--);
5152                 }
5153         }
5154
5155         /* Clear each TX chain page. */
5156         for (i = 0; i < TX_PAGES; i++)
5157                 bzero((char *)sc->tx_bd_chain[i], BCE_TX_CHAIN_PAGE_SZ);
5158
5159         sc->used_tx_bd     = 0;
5160
5161         /* Check if we lost any mbufs in the process. */
5162         DBRUNIF((sc->debug_tx_mbuf_alloc),
5163                 BCE_PRINTF("%s(%d): Memory leak! Lost %d mbufs "
5164                         "from tx chain!\n",
5165                         __FILE__, __LINE__, sc->debug_tx_mbuf_alloc));
5166
5167         DBEXIT(BCE_VERBOSE_RESET | BCE_VERBOSE_SEND | BCE_VERBOSE_UNLOAD);
5168 }
5169
5170
5171 /****************************************************************************/
5172 /* Initialize the RX context memory.                                        */
5173 /*                                                                          */
5174 /* Returns:                                                                 */
5175 /*   Nothing                                                                */
5176 /****************************************************************************/
5177 static void
5178 bce_init_rx_context(struct bce_softc *sc)
5179 {
5180         u32 val;
5181
5182         DBENTER(BCE_VERBOSE_RESET | BCE_VERBOSE_RECV | BCE_VERBOSE_CTX);
5183
5184         /* Initialize the type, size, and BD cache levels for the RX context. */
5185         val = BCE_L2CTX_RX_CTX_TYPE_CTX_BD_CHN_TYPE_VALUE |
5186                 BCE_L2CTX_RX_CTX_TYPE_SIZE_L2 |
5187                 (0x02 << BCE_L2CTX_RX_BD_PRE_READ_SHIFT);
5188
5189         /*
5190          * Set the level for generating pause frames
5191          * when the number of available rx_bd's gets
5192          * too low (the low watermark) and the level
5193          * when pause frames can be stopped (the high
5194          * watermark).
5195          */
5196         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
5197                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
5198                 u32 lo_water, hi_water;
5199
5200                 lo_water = BCE_L2CTX_RX_LO_WATER_MARK_DEFAULT;
5201                 hi_water = USABLE_RX_BD / 4;
5202
5203                 lo_water /= BCE_L2CTX_RX_LO_WATER_MARK_SCALE;
5204                 hi_water /= BCE_L2CTX_RX_HI_WATER_MARK_SCALE;
5205
5206                 if (hi_water > 0xf)
5207                         hi_water = 0xf;
5208                 else if (hi_water == 0)
5209                         lo_water = 0;
5210                 val |= (lo_water << BCE_L2CTX_RX_LO_WATER_MARK_SHIFT) |
5211                         (hi_water << BCE_L2CTX_RX_HI_WATER_MARK_SHIFT);
5212         }
5213
5214         CTX_WR(sc, GET_CID_ADDR(RX_CID), BCE_L2CTX_RX_CTX_TYPE, val);
5215
5216         /* Setup the MQ BIN mapping for l2_ctx_host_bseq. */
5217         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
5218                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
5219                 val = REG_RD(sc, BCE_MQ_MAP_L2_5);
5220                 REG_WR(sc, BCE_MQ_MAP_L2_5, val | BCE_MQ_MAP_L2_5_ARM);
5221         }
5222
5223         /* Point the hardware to the first page in the chain. */
5224         val = BCE_ADDR_HI(sc->rx_bd_chain_paddr[0]);
5225         CTX_WR(sc, GET_CID_ADDR(RX_CID), BCE_L2CTX_RX_NX_BDHADDR_HI, val);
5226         val = BCE_ADDR_LO(sc->rx_bd_chain_paddr[0]);
5227         CTX_WR(sc, GET_CID_ADDR(RX_CID), BCE_L2CTX_RX_NX_BDHADDR_LO, val);
5228
5229         DBEXIT(BCE_VERBOSE_RESET | BCE_VERBOSE_RECV | BCE_VERBOSE_CTX);
5230 }
5231
5232
5233 /****************************************************************************/
5234 /* Allocate memory and initialize the RX data structures.                   */
5235 /*                                                                          */
5236 /* Returns:                                                                 */
5237 /*   0 for success, positive value for failure.                             */
5238 /****************************************************************************/
5239 static int
5240 bce_init_rx_chain(struct bce_softc *sc)
5241 {
5242         struct rx_bd *rxbd;
5243         int i, rc = 0;
5244
5245         DBENTER(BCE_VERBOSE_RESET | BCE_VERBOSE_RECV | BCE_VERBOSE_LOAD |
5246                 BCE_VERBOSE_CTX);
5247
5248         /* Initialize the RX producer and consumer indices. */
5249         sc->rx_prod        = 0;
5250         sc->rx_cons        = 0;
5251         sc->rx_prod_bseq   = 0;
5252         sc->free_rx_bd     = USABLE_RX_BD;
5253         sc->max_rx_bd      = USABLE_RX_BD;
5254         DBRUN(sc->rx_low_watermark = sc->max_rx_bd);
5255         DBRUN(sc->rx_empty_count = 0);
5256
5257         /* Initialize the RX next pointer chain entries. */
5258         for (i = 0; i < RX_PAGES; i++) {
5259                 int j;
5260
5261                 rxbd = &sc->rx_bd_chain[i][USABLE_RX_BD_PER_PAGE];
5262
5263                 /* Check if we've reached the last page. */
5264                 if (i == (RX_PAGES - 1))
5265                         j = 0;
5266                 else
5267                         j = i + 1;
5268
5269                 /* Setup the chain page pointers. */
5270                 rxbd->rx_bd_haddr_hi = htole32(BCE_ADDR_HI(sc->rx_bd_chain_paddr[j]));
5271                 rxbd->rx_bd_haddr_lo = htole32(BCE_ADDR_LO(sc->rx_bd_chain_paddr[j]));
5272         }
5273
5274 /* Fill up the RX chain. */
5275         bce_fill_rx_chain(sc);
5276
5277         for (i = 0; i < RX_PAGES; i++) {
5278                 bus_dmamap_sync(
5279                         sc->rx_bd_chain_tag,
5280                 sc->rx_bd_chain_map[i],
5281                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
5282         }
5283
5284         bce_init_rx_context(sc);
5285
5286         DBRUNMSG(BCE_EXTREME_RECV, bce_dump_rx_chain(sc, 0, TOTAL_RX_BD));
5287         DBEXIT(BCE_VERBOSE_RESET | BCE_VERBOSE_RECV | BCE_VERBOSE_LOAD |
5288                 BCE_VERBOSE_CTX);
5289         /* ToDo: Are there possible failure modes here? */
5290         return(rc);
5291 }
5292
5293
5294 /****************************************************************************/
5295 /* Add mbufs to the RX chain until its full or an mbuf allocation error     */
5296 /* occurs.                                                                  */
5297 /*                                                                          */
5298 /* Returns:                                                                 */
5299 /*   Nothing                                                                */
5300 /****************************************************************************/
5301 static void
5302 bce_fill_rx_chain(struct bce_softc *sc)
5303 {
5304         u16 prod, prod_idx;
5305         u32 prod_bseq;
5306
5307         DBENTER(BCE_VERBOSE_RESET | BCE_EXTREME_RECV | BCE_VERBOSE_LOAD |
5308                 BCE_VERBOSE_CTX);
5309
5310         /* Get the RX chain producer indices. */
5311         prod      = sc->rx_prod;
5312         prod_bseq = sc->rx_prod_bseq;
5313
5314         /* Keep filling the RX chain until it's full. */
5315         while (sc->free_rx_bd > 0) {
5316                 prod_idx = RX_CHAIN_IDX(prod);
5317                 if (bce_get_rx_buf(sc, NULL, &prod, &prod_idx, &prod_bseq)) {
5318                         /* Bail out if we can't add an mbuf to the chain. */
5319                         break;
5320                 }
5321                 prod = NEXT_RX_BD(prod);
5322         }
5323
5324         /* Save the RX chain producer indices. */
5325         sc->rx_prod      = prod;
5326         sc->rx_prod_bseq = prod_bseq;
5327
5328         DBRUNIF(((prod & USABLE_RX_BD_PER_PAGE) == USABLE_RX_BD_PER_PAGE),
5329                 BCE_PRINTF("%s(): Invalid rx_prod value: 0x%04X\n",
5330                 __FUNCTION__, sc->rx_prod));
5331
5332         /* Write the mailbox and tell the chip about the waiting rx_bd's. */
5333         REG_WR16(sc, MB_GET_CID_ADDR(RX_CID) + BCE_L2MQ_RX_HOST_BDIDX,
5334                 sc->rx_prod);
5335         REG_WR(sc, MB_GET_CID_ADDR(RX_CID) + BCE_L2MQ_RX_HOST_BSEQ,
5336                 sc->rx_prod_bseq);
5337
5338         DBEXIT(BCE_VERBOSE_RESET | BCE_EXTREME_RECV | BCE_VERBOSE_LOAD |
5339                 BCE_VERBOSE_CTX);
5340 }
5341
5342
5343 /****************************************************************************/
5344 /* Free memory and clear the RX data structures.                            */
5345 /*                                                                          */
5346 /* Returns:                                                                 */
5347 /*   Nothing.                                                               */
5348 /****************************************************************************/
5349 static void
5350 bce_free_rx_chain(struct bce_softc *sc)
5351 {
5352         int i;
5353
5354         DBENTER(BCE_VERBOSE_RESET | BCE_VERBOSE_RECV | BCE_VERBOSE_UNLOAD);
5355
5356         /* Free any mbufs still in the RX mbuf chain. */
5357         for (i = 0; i < TOTAL_RX_BD; i++) {
5358                 if (sc->rx_mbuf_ptr[i] != NULL) {
5359                         if (sc->rx_mbuf_map[i] != NULL)
5360                                 bus_dmamap_sync(sc->rx_mbuf_tag, sc->rx_mbuf_map[i],
5361                                         BUS_DMASYNC_POSTREAD);
5362                         m_freem(sc->rx_mbuf_ptr[i]);
5363                         sc->rx_mbuf_ptr[i] = NULL;
5364                         DBRUN(sc->debug_rx_mbuf_alloc--);
5365                 }
5366         }
5367
5368         /* Clear each RX chain page. */
5369         for (i = 0; i < RX_PAGES; i++)
5370                 bzero((char *)sc->rx_bd_chain[i], BCE_RX_CHAIN_PAGE_SZ);
5371
5372         sc->free_rx_bd = sc->max_rx_bd;
5373
5374         /* Check if we lost any mbufs in the process. */
5375         DBRUNIF((sc->debug_rx_mbuf_alloc),
5376                 BCE_PRINTF("%s(): Memory leak! Lost %d mbufs from rx chain!\n",
5377                         __FUNCTION__, sc->debug_rx_mbuf_alloc));
5378
5379         DBEXIT(BCE_VERBOSE_RESET | BCE_VERBOSE_RECV | BCE_VERBOSE_UNLOAD);
5380 }
5381
5382
5383 #ifdef ZERO_COPY_SOCKETS
5384 /****************************************************************************/
5385 /* Allocate memory and initialize the page data structures.                 */
5386 /* Assumes that bce_init_rx_chain() has not already been called.            */
5387 /*                                                                          */
5388 /* Returns:                                                                 */
5389 /*   0 for success, positive value for failure.                             */
5390 /****************************************************************************/
5391 static int
5392 bce_init_pg_chain(struct bce_softc *sc)
5393 {
5394         struct rx_bd *pgbd;
5395         int i, rc = 0;
5396         u32 val;
5397
5398         DBENTER(BCE_VERBOSE_RESET | BCE_VERBOSE_RECV | BCE_VERBOSE_LOAD |
5399                 BCE_VERBOSE_CTX);
5400
5401         /* Initialize the page producer and consumer indices. */
5402         sc->pg_prod        = 0;
5403         sc->pg_cons        = 0;
5404         sc->free_pg_bd     = USABLE_PG_BD;
5405         sc->max_pg_bd      = USABLE_PG_BD;
5406         DBRUN(sc->pg_low_watermark = sc->max_pg_bd);
5407         DBRUN(sc->pg_empty_count = 0);
5408
5409         /* Initialize the page next pointer chain entries. */
5410         for (i = 0; i < PG_PAGES; i++) {
5411                 int j;
5412
5413                 pgbd = &sc->pg_bd_chain[i][USABLE_PG_BD_PER_PAGE];
5414
5415                 /* Check if we've reached the last page. */
5416                 if (i == (PG_PAGES - 1))
5417                         j = 0;
5418                 else
5419                         j = i + 1;
5420
5421                 /* Setup the chain page pointers. */
5422                 pgbd->rx_bd_haddr_hi = htole32(BCE_ADDR_HI(sc->pg_bd_chain_paddr[j]));
5423                 pgbd->rx_bd_haddr_lo = htole32(BCE_ADDR_LO(sc->pg_bd_chain_paddr[j]));
5424         }
5425
5426         /* Setup the MQ BIN mapping for host_pg_bidx. */
5427         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709)     ||
5428                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716))
5429                 REG_WR(sc, BCE_MQ_MAP_L2_3, BCE_MQ_MAP_L2_3_DEFAULT);
5430
5431         CTX_WR(sc, GET_CID_ADDR(RX_CID), BCE_L2CTX_RX_PG_BUF_SIZE, 0);
5432
5433         /* Configure the rx_bd and page chain mbuf cluster size. */
5434         val = (sc->rx_bd_mbuf_data_len << 16) | sc->pg_bd_mbuf_alloc_size;
5435         CTX_WR(sc, GET_CID_ADDR(RX_CID), BCE_L2CTX_RX_PG_BUF_SIZE, val);
5436
5437         /* Configure the context reserved for jumbo support. */
5438         CTX_WR(sc, GET_CID_ADDR(RX_CID), BCE_L2CTX_RX_RBDC_KEY,
5439                 BCE_L2CTX_RX_RBDC_JUMBO_KEY);
5440
5441         /* Point the hardware to the first page in the page chain. */
5442         val = BCE_ADDR_HI(sc->pg_bd_chain_paddr[0]);
5443         CTX_WR(sc, GET_CID_ADDR(RX_CID), BCE_L2CTX_RX_NX_PG_BDHADDR_HI, val);
5444         val = BCE_ADDR_LO(sc->pg_bd_chain_paddr[0]);
5445         CTX_WR(sc, GET_CID_ADDR(RX_CID), BCE_L2CTX_RX_NX_PG_BDHADDR_LO, val);
5446
5447         /* Fill up the page chain. */
5448         bce_fill_pg_chain(sc);
5449
5450         for (i = 0; i < PG_PAGES; i++) {
5451                 bus_dmamap_sync(
5452                         sc->pg_bd_chain_tag,
5453                 sc->pg_bd_chain_map[i],
5454                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
5455         }
5456
5457         DBRUNMSG(BCE_EXTREME_RECV, bce_dump_pg_chain(sc, 0, TOTAL_PG_BD));
5458         DBEXIT(BCE_VERBOSE_RESET | BCE_VERBOSE_RECV | BCE_VERBOSE_LOAD |
5459                 BCE_VERBOSE_CTX);
5460         return(rc);
5461 }
5462
5463
5464 /****************************************************************************/
5465 /* Add mbufs to the page chain until its full or an mbuf allocation error   */
5466 /* occurs.                                                                  */
5467 /*                                                                          */
5468 /* Returns:                                                                 */
5469 /*   Nothing                                                                */
5470 /****************************************************************************/
5471 static void
5472 bce_fill_pg_chain(struct bce_softc *sc)
5473 {
5474         u16 prod, prod_idx;
5475
5476         DBENTER(BCE_VERBOSE_RESET | BCE_EXTREME_RECV | BCE_VERBOSE_LOAD |
5477                 BCE_VERBOSE_CTX);
5478
5479         /* Get the page chain prodcuer index. */
5480         prod = sc->pg_prod;
5481
5482         /* Keep filling the page chain until it's full. */
5483         while (sc->free_pg_bd > 0) {
5484                 prod_idx = PG_CHAIN_IDX(prod);
5485                 if (bce_get_pg_buf(sc, NULL, &prod, &prod_idx)) {
5486                         /* Bail out if we can't add an mbuf to the chain. */
5487                         break;
5488                 }
5489                 prod = NEXT_PG_BD(prod);
5490         }
5491
5492         /* Save the page chain producer index. */
5493         sc->pg_prod = prod;
5494
5495         DBRUNIF(((prod & USABLE_RX_BD_PER_PAGE) == USABLE_RX_BD_PER_PAGE),
5496                 BCE_PRINTF("%s(): Invalid pg_prod value: 0x%04X\n",
5497                 __FUNCTION__, sc->pg_prod));
5498
5499         /*
5500          * Write the mailbox and tell the chip about
5501          * the new rx_bd's in the page chain.
5502          */
5503         REG_WR16(sc, MB_GET_CID_ADDR(RX_CID) + BCE_L2MQ_RX_HOST_PG_BDIDX,
5504                 sc->pg_prod);
5505
5506         DBEXIT(BCE_VERBOSE_RESET | BCE_EXTREME_RECV | BCE_VERBOSE_LOAD |
5507                 BCE_VERBOSE_CTX);
5508 }
5509
5510
5511 /****************************************************************************/
5512 /* Free memory and clear the RX data structures.                            */
5513 /*                                                                          */
5514 /* Returns:                                                                 */
5515 /*   Nothing.                                                               */
5516 /****************************************************************************/
5517 static void
5518 bce_free_pg_chain(struct bce_softc *sc)
5519 {
5520         int i;
5521
5522         DBENTER(BCE_VERBOSE_RESET | BCE_VERBOSE_RECV | BCE_VERBOSE_UNLOAD);
5523
5524         /* Free any mbufs still in the mbuf page chain. */
5525         for (i = 0; i < TOTAL_PG_BD; i++) {
5526                 if (sc->pg_mbuf_ptr[i] != NULL) {
5527                         if (sc->pg_mbuf_map[i] != NULL)
5528                                 bus_dmamap_sync(sc->pg_mbuf_tag, sc->pg_mbuf_map[i],
5529                                         BUS_DMASYNC_POSTREAD);
5530                         m_freem(sc->pg_mbuf_ptr[i]);
5531                         sc->pg_mbuf_ptr[i] = NULL;
5532                         DBRUN(sc->debug_pg_mbuf_alloc--);
5533                 }
5534         }
5535
5536         /* Clear each page chain pages. */
5537         for (i = 0; i < PG_PAGES; i++)
5538                 bzero((char *)sc->pg_bd_chain[i], BCE_PG_CHAIN_PAGE_SZ);
5539
5540         sc->free_pg_bd = sc->max_pg_bd;
5541
5542         /* Check if we lost any mbufs in the process. */
5543         DBRUNIF((sc->debug_pg_mbuf_alloc),
5544                 BCE_PRINTF("%s(): Memory leak! Lost %d mbufs from page chain!\n",
5545                         __FUNCTION__, sc->debug_pg_mbuf_alloc));
5546
5547         DBEXIT(BCE_VERBOSE_RESET | BCE_VERBOSE_RECV | BCE_VERBOSE_UNLOAD);
5548 }
5549 #endif /* ZERO_COPY_SOCKETS */
5550
5551
5552 /****************************************************************************/
5553 /* Set media options.                                                       */
5554 /*                                                                          */
5555 /* Returns:                                                                 */
5556 /*   0 for success, positive value for failure.                             */
5557 /****************************************************************************/
5558 static int
5559 bce_ifmedia_upd(struct ifnet *ifp)
5560 {
5561         struct bce_softc *sc = ifp->if_softc;
5562
5563         DBENTER(BCE_VERBOSE);
5564
5565         BCE_LOCK(sc);
5566         bce_ifmedia_upd_locked(ifp);
5567         BCE_UNLOCK(sc);
5568
5569         DBEXIT(BCE_VERBOSE);
5570         return (0);
5571 }
5572
5573
5574 /****************************************************************************/
5575 /* Set media options.                                                       */
5576 /*                                                                          */
5577 /* Returns:                                                                 */
5578 /*   Nothing.                                                               */
5579 /****************************************************************************/
5580 static void
5581 bce_ifmedia_upd_locked(struct ifnet *ifp)
5582 {
5583         struct bce_softc *sc = ifp->if_softc;
5584         struct mii_data *mii;
5585
5586         DBENTER(BCE_VERBOSE);
5587
5588         BCE_LOCK_ASSERT(sc);
5589
5590         mii = device_get_softc(sc->bce_miibus);
5591
5592         /* Make sure the MII bus has been enumerated. */
5593         if (mii) {
5594                 sc->bce_link = 0;
5595                 if (mii->mii_instance) {
5596                         struct mii_softc *miisc;
5597
5598                         LIST_FOREACH(miisc, &mii->mii_phys, mii_list)
5599                                 mii_phy_reset(miisc);
5600                 }
5601                 mii_mediachg(mii);
5602         }
5603
5604         DBEXIT(BCE_VERBOSE);
5605 }
5606
5607
5608 /****************************************************************************/
5609 /* Reports current media status.                                            */
5610 /*                                                                          */
5611 /* Returns:                                                                 */
5612 /*   Nothing.                                                               */
5613 /****************************************************************************/
5614 static void
5615 bce_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
5616 {
5617         struct bce_softc *sc = ifp->if_softc;
5618         struct mii_data *mii;
5619
5620         DBENTER(BCE_VERBOSE);
5621
5622         BCE_LOCK(sc);
5623
5624         mii = device_get_softc(sc->bce_miibus);
5625
5626         mii_pollstat(mii);
5627         ifmr->ifm_active = mii->mii_media_active;
5628         ifmr->ifm_status = mii->mii_media_status;
5629
5630         BCE_UNLOCK(sc);
5631
5632         DBEXIT(BCE_VERBOSE);
5633 }
5634
5635
5636 /****************************************************************************/
5637 /* Handles PHY generated interrupt events.                                  */
5638 /*                                                                          */
5639 /* Returns:                                                                 */
5640 /*   Nothing.                                                               */
5641 /****************************************************************************/
5642 static void
5643 bce_phy_intr(struct bce_softc *sc)
5644 {
5645         u32 new_link_state, old_link_state;
5646
5647         DBENTER(BCE_VERBOSE_PHY | BCE_VERBOSE_INTR);
5648
5649         new_link_state = sc->status_block->status_attn_bits &
5650                 STATUS_ATTN_BITS_LINK_STATE;
5651         old_link_state = sc->status_block->status_attn_bits_ack &
5652                 STATUS_ATTN_BITS_LINK_STATE;
5653
5654         /* Handle any changes if the link state has changed. */
5655         if (new_link_state != old_link_state) {
5656
5657                 /* Update the status_attn_bits_ack field in the status block. */
5658                 if (new_link_state) {
5659                         REG_WR(sc, BCE_PCICFG_STATUS_BIT_SET_CMD,
5660                                 STATUS_ATTN_BITS_LINK_STATE);
5661                         DBPRINT(sc, BCE_INFO_PHY, "%s(): Link is now UP.\n",
5662                                 __FUNCTION__);
5663                 }
5664                 else {
5665                         REG_WR(sc, BCE_PCICFG_STATUS_BIT_CLEAR_CMD,
5666                                 STATUS_ATTN_BITS_LINK_STATE);
5667                         DBPRINT(sc, BCE_INFO_PHY, "%s(): Link is now DOWN.\n",
5668                                 __FUNCTION__);
5669                 }
5670
5671                 /*
5672                  * Assume link is down and allow
5673                  * tick routine to update the state
5674                  * based on the actual media state.
5675                  */
5676                 sc->bce_link = 0;
5677                 callout_stop(&sc->bce_tick_callout);
5678                 bce_tick(sc);
5679         }
5680
5681         /* Acknowledge the link change interrupt. */
5682         REG_WR(sc, BCE_EMAC_STATUS, BCE_EMAC_STATUS_LINK_CHANGE);
5683
5684         DBEXIT(BCE_VERBOSE_PHY | BCE_VERBOSE_INTR);
5685 }
5686
5687
5688 /****************************************************************************/
5689 /* Reads the receive consumer value from the status block (skipping over    */
5690 /* chain page pointer if necessary).                                        */
5691 /*                                                                          */
5692 /* Returns:                                                                 */
5693 /*   hw_cons                                                                */
5694 /****************************************************************************/
5695 static inline u16
5696 bce_get_hw_rx_cons(struct bce_softc *sc)
5697 {
5698         u16 hw_cons;
5699
5700         rmb();
5701         hw_cons = sc->status_block->status_rx_quick_consumer_index0;
5702         if ((hw_cons & USABLE_RX_BD_PER_PAGE) == USABLE_RX_BD_PER_PAGE)
5703                 hw_cons++;
5704
5705         return hw_cons;
5706 }
5707
5708 /****************************************************************************/
5709 /* Handles received frame interrupt events.                                 */
5710 /*                                                                          */
5711 /* Returns:                                                                 */
5712 /*   Nothing.                                                               */
5713 /****************************************************************************/
5714 static void
5715 bce_rx_intr(struct bce_softc *sc)
5716 {
5717         struct ifnet *ifp = sc->bce_ifp;
5718         struct l2_fhdr *l2fhdr;
5719         unsigned int pkt_len;
5720         u16 sw_rx_cons, sw_rx_cons_idx, hw_rx_cons;
5721         u32 status;
5722 #ifdef ZERO_COPY_SOCKETS
5723         unsigned int rem_len;
5724         u16 sw_pg_cons, sw_pg_cons_idx;
5725 #endif
5726
5727         DBENTER(BCE_VERBOSE_RECV | BCE_VERBOSE_INTR);
5728         DBRUN(sc->rx_interrupts++);
5729         DBPRINT(sc, BCE_EXTREME_RECV, "%s(enter): rx_prod = 0x%04X, "
5730                 "rx_cons = 0x%04X, rx_prod_bseq = 0x%08X\n",
5731                 __FUNCTION__, sc->rx_prod, sc->rx_cons, sc->rx_prod_bseq);
5732
5733         /* Prepare the RX chain pages to be accessed by the host CPU. */
5734         for (int i = 0; i < RX_PAGES; i++)
5735                 bus_dmamap_sync(sc->rx_bd_chain_tag,
5736                     sc->rx_bd_chain_map[i], BUS_DMASYNC_POSTWRITE);
5737
5738 #ifdef ZERO_COPY_SOCKETS
5739         /* Prepare the page chain pages to be accessed by the host CPU. */
5740         for (int i = 0; i < PG_PAGES; i++)
5741                 bus_dmamap_sync(sc->pg_bd_chain_tag,
5742                     sc->pg_bd_chain_map[i], BUS_DMASYNC_POSTWRITE);
5743 #endif
5744
5745         /* Get the hardware's view of the RX consumer index. */
5746         hw_rx_cons = sc->hw_rx_cons = bce_get_hw_rx_cons(sc);
5747
5748         /* Get working copies of the driver's view of the consumer indices. */
5749         sw_rx_cons = sc->rx_cons;
5750 #ifdef ZERO_COPY_SOCKETS
5751         sw_pg_cons = sc->pg_cons;
5752 #endif
5753
5754         /* Update some debug statistics counters */
5755         DBRUNIF((sc->free_rx_bd < sc->rx_low_watermark),
5756                 sc->rx_low_watermark = sc->free_rx_bd);
5757         DBRUNIF((sc->free_rx_bd == sc->max_rx_bd), sc->rx_empty_count++);
5758
5759         /* Scan through the receive chain as long as there is work to do */
5760         /* ToDo: Consider setting a limit on the number of packets processed. */
5761         rmb();
5762         while (sw_rx_cons != hw_rx_cons) {
5763                 struct mbuf *m0;
5764
5765                 /* Convert the producer/consumer indices to an actual rx_bd index. */
5766                 sw_rx_cons_idx = RX_CHAIN_IDX(sw_rx_cons);
5767
5768                 /* Unmap the mbuf from DMA space. */
5769                 bus_dmamap_sync(sc->rx_mbuf_tag,
5770                     sc->rx_mbuf_map[sw_rx_cons_idx],
5771                 BUS_DMASYNC_POSTREAD);
5772                 bus_dmamap_unload(sc->rx_mbuf_tag,
5773                     sc->rx_mbuf_map[sw_rx_cons_idx]);
5774
5775                 /* Remove the mbuf from the RX chain. */
5776                 m0 = sc->rx_mbuf_ptr[sw_rx_cons_idx];
5777                 sc->rx_mbuf_ptr[sw_rx_cons_idx] = NULL;
5778                 DBRUN(sc->debug_rx_mbuf_alloc--);
5779                 sc->free_rx_bd++;
5780
5781                 /*
5782                  * Frames received on the NetXteme II are prepended     with an
5783                  * l2_fhdr structure which provides status information about
5784                  * the received frame (including VLAN tags and checksum info).
5785                  * The frames are also automatically adjusted to align the IP
5786                  * header (i.e. two null bytes are inserted before the Ethernet
5787                  * header).  As a result the data DMA'd by the controller into
5788                  * the mbuf is as follows:
5789                  *
5790                  * +---------+-----+---------------------+-----+
5791                  * | l2_fhdr | pad | packet data         | FCS |
5792                  * +---------+-----+---------------------+-----+
5793                  *
5794                  * The l2_fhdr needs to be checked and skipped and the FCS needs
5795                  * to be stripped before sending the packet up the stack.
5796                  */
5797                 l2fhdr  = mtod(m0, struct l2_fhdr *);
5798
5799                 /* Get the packet data + FCS length and the status. */
5800                 pkt_len = l2fhdr->l2_fhdr_pkt_len;
5801                 status  = l2fhdr->l2_fhdr_status;
5802
5803                 /*
5804                  * Skip over the l2_fhdr and pad, resulting in the
5805                  * following data in the mbuf:
5806                  * +---------------------+-----+
5807                  * | packet data         | FCS |
5808                  * +---------------------+-----+
5809                  */
5810                 m_adj(m0, sizeof(struct l2_fhdr) + ETHER_ALIGN);
5811
5812 #ifdef ZERO_COPY_SOCKETS
5813                 /*
5814                  * Check whether the received frame fits in a single
5815                  * mbuf or not (i.e. packet data + FCS <=
5816                  * sc->rx_bd_mbuf_data_len bytes).
5817                  */
5818                 if (pkt_len > m0->m_len) {
5819                         /*
5820                          * The received frame is larger than a single mbuf.
5821                          * If the frame was a TCP frame then only the TCP
5822                          * header is placed in the mbuf, the remaining
5823                          * payload (including FCS) is placed in the page
5824                          * chain, the SPLIT flag is set, and the header
5825                          * length is placed in the IP checksum field.
5826                          * If the frame is not a TCP frame then the mbuf
5827                          * is filled and the remaining bytes are placed
5828                          * in the page chain.
5829                          */
5830
5831                         DBPRINT(sc, BCE_INFO_RECV, "%s(): Found a large packet.\n",
5832                                 __FUNCTION__);
5833
5834                         /*
5835                          * When the page chain is enabled and the TCP
5836                          * header has been split from the TCP payload,
5837                          * the ip_xsum structure will reflect the length
5838                          * of the TCP header, not the IP checksum.  Set
5839                          * the packet length of the mbuf accordingly.
5840                          */
5841                         if (status & L2_FHDR_STATUS_SPLIT)
5842                                 m0->m_len = l2fhdr->l2_fhdr_ip_xsum;
5843
5844                         rem_len = pkt_len - m0->m_len;
5845
5846                         /* Pull mbufs off the page chain for the remaining data. */
5847                         while (rem_len > 0) {
5848                                 struct mbuf *m_pg;
5849
5850                                 sw_pg_cons_idx = PG_CHAIN_IDX(sw_pg_cons);
5851
5852                                 /* Remove the mbuf from the page chain. */
5853                                 m_pg = sc->pg_mbuf_ptr[sw_pg_cons_idx];
5854                                 sc->pg_mbuf_ptr[sw_pg_cons_idx] = NULL;
5855                                 DBRUN(sc->debug_pg_mbuf_alloc--);
5856                                 sc->free_pg_bd++;
5857
5858                                 /* Unmap the page chain mbuf from DMA space. */
5859                                 bus_dmamap_sync(sc->pg_mbuf_tag,
5860                                         sc->pg_mbuf_map[sw_pg_cons_idx],
5861                                         BUS_DMASYNC_POSTREAD);
5862                                 bus_dmamap_unload(sc->pg_mbuf_tag,
5863                                         sc->pg_mbuf_map[sw_pg_cons_idx]);
5864
5865                                 /* Adjust the mbuf length. */
5866                                 if (rem_len < m_pg->m_len) {
5867                                         /* The mbuf chain is complete. */
5868                                         m_pg->m_len = rem_len;
5869                                         rem_len = 0;
5870                                 } else {
5871                                         /* More packet data is waiting. */
5872                                         rem_len -= m_pg->m_len;
5873                                 }
5874
5875                                 /* Concatenate the mbuf cluster to the mbuf. */
5876                                 m_cat(m0, m_pg);
5877
5878                                 sw_pg_cons = NEXT_PG_BD(sw_pg_cons);
5879                         }
5880
5881                         /* Set the total packet length. */
5882                         m0->m_pkthdr.len = pkt_len;
5883
5884                 } else {
5885                         /*
5886                          * The received packet is small and fits in a
5887                          * single mbuf (i.e. the l2_fhdr + pad + packet +
5888                          * FCS <= MHLEN).  In other words, the packet is
5889                          * 154 bytes or less in size.
5890                          */
5891
5892                         DBPRINT(sc, BCE_INFO_RECV, "%s(): Found a small packet.\n",
5893                                 __FUNCTION__);
5894
5895                         /* Set the total packet length. */
5896                         m0->m_pkthdr.len = m0->m_len = pkt_len;
5897                 }
5898 #endif
5899
5900                 /* Remove the trailing Ethernet FCS. */
5901                 m_adj(m0, -ETHER_CRC_LEN);
5902
5903                 /* Check that the resulting mbuf chain is valid. */
5904                 DBRUN(m_sanity(m0, FALSE));
5905                 DBRUNIF(((m0->m_len < ETHER_HDR_LEN) |
5906                         (m0->m_pkthdr.len > BCE_MAX_JUMBO_ETHER_MTU_VLAN)),
5907                         BCE_PRINTF("Invalid Ethernet frame size!\n");
5908                         m_print(m0, 128));
5909
5910                 DBRUNIF(DB_RANDOMTRUE(l2fhdr_error_sim_control),
5911                         BCE_PRINTF("Simulating l2_fhdr status error.\n");
5912                         sc->l2fhdr_error_sim_count++;
5913                         status = status | L2_FHDR_ERRORS_PHY_DECODE);
5914
5915                 /* Check the received frame for errors. */
5916                 if (status & (L2_FHDR_ERRORS_BAD_CRC |
5917                         L2_FHDR_ERRORS_PHY_DECODE | L2_FHDR_ERRORS_ALIGNMENT |
5918                         L2_FHDR_ERRORS_TOO_SHORT  | L2_FHDR_ERRORS_GIANT_FRAME)) {
5919
5920                         /* Log the error and release the mbuf. */
5921                         ifp->if_ierrors++;
5922                         sc->l2fhdr_error_count++;
5923
5924                         m_freem(m0);
5925                         m0 = NULL;
5926                         goto bce_rx_int_next_rx;
5927                 }
5928
5929                 /* Send the packet to the appropriate interface. */
5930                 m0->m_pkthdr.rcvif = ifp;
5931
5932                 /* Assume no hardware checksum. */
5933                 m0->m_pkthdr.csum_flags = 0;
5934
5935                 /* Validate the checksum if offload enabled. */
5936                 if (ifp->if_capenable & IFCAP_RXCSUM) {
5937
5938                         /* Check for an IP datagram. */
5939                         if (!(status & L2_FHDR_STATUS_SPLIT) &&
5940                                 (status & L2_FHDR_STATUS_IP_DATAGRAM)) {
5941                                 m0->m_pkthdr.csum_flags |= CSUM_IP_CHECKED;
5942
5943                                 /* Check if the IP checksum is valid. */
5944                                 if ((l2fhdr->l2_fhdr_ip_xsum ^ 0xffff) == 0)
5945                                         m0->m_pkthdr.csum_flags |= CSUM_IP_VALID;
5946                         }
5947
5948                         /* Check for a valid TCP/UDP frame. */
5949                         if (status & (L2_FHDR_STATUS_TCP_SEGMENT |
5950                                 L2_FHDR_STATUS_UDP_DATAGRAM)) {
5951
5952                                 /* Check for a good TCP/UDP checksum. */
5953                                 if ((status & (L2_FHDR_ERRORS_TCP_XSUM |
5954                                               L2_FHDR_ERRORS_UDP_XSUM)) == 0) {
5955                                         m0->m_pkthdr.csum_data =
5956                                             l2fhdr->l2_fhdr_tcp_udp_xsum;
5957                                         m0->m_pkthdr.csum_flags |= (CSUM_DATA_VALID
5958                                                 | CSUM_PSEUDO_HDR);
5959                                 }
5960                         }
5961                 }
5962
5963                 /* Attach the VLAN tag. */
5964                 if (status & L2_FHDR_STATUS_L2_VLAN_TAG) {
5965 #if __FreeBSD_version < 700000
5966                         VLAN_INPUT_TAG(ifp, m0, l2fhdr->l2_fhdr_vlan_tag, continue);
5967 #else
5968                         m0->m_pkthdr.ether_vtag = l2fhdr->l2_fhdr_vlan_tag;
5969                         m0->m_flags |= M_VLANTAG;
5970 #endif
5971                 }
5972
5973                 /* Increment received packet statistics. */
5974                 ifp->if_ipackets++;
5975
5976 bce_rx_int_next_rx:
5977                 sw_rx_cons = NEXT_RX_BD(sw_rx_cons);
5978
5979                 /* If we have a packet, pass it up the stack */
5980                 if (m0) {
5981                         /* Make sure we don't lose our place when we release the lock. */
5982                         sc->rx_cons = sw_rx_cons;
5983 #ifdef ZERO_COPY_SOCKETS
5984                         sc->pg_cons = sw_pg_cons;
5985 #endif
5986
5987                         BCE_UNLOCK(sc);
5988                         (*ifp->if_input)(ifp, m0);
5989                         BCE_LOCK(sc);
5990
5991                         /* Recover our place. */
5992                         sw_rx_cons = sc->rx_cons;
5993 #ifdef ZERO_COPY_SOCKETS
5994                         sw_pg_cons = sc->pg_cons;
5995 #endif
5996                 }
5997
5998                 /* Refresh hw_cons to see if there's new work */
5999                 if (sw_rx_cons == hw_rx_cons)
6000                         hw_rx_cons = sc->hw_rx_cons = bce_get_hw_rx_cons(sc);
6001         }
6002
6003         /* No new packets to process.  Refill the RX and page chains and exit. */
6004 #ifdef ZERO_COPY_SOCKETS
6005         sc->pg_cons = sw_pg_cons;
6006         bce_fill_pg_chain(sc);
6007 #endif
6008
6009         sc->rx_cons = sw_rx_cons;
6010         bce_fill_rx_chain(sc);
6011
6012         for (int i = 0; i < RX_PAGES; i++)
6013                 bus_dmamap_sync(sc->rx_bd_chain_tag,
6014                     sc->rx_bd_chain_map[i], BUS_DMASYNC_PREWRITE);
6015
6016 #ifdef ZERO_COPY_SOCKETS
6017         for (int i = 0; i < PG_PAGES; i++)
6018                 bus_dmamap_sync(sc->pg_bd_chain_tag,
6019                     sc->pg_bd_chain_map[i], BUS_DMASYNC_PREWRITE);
6020 #endif
6021
6022         DBPRINT(sc, BCE_EXTREME_RECV, "%s(exit): rx_prod = 0x%04X, "
6023                 "rx_cons = 0x%04X, rx_prod_bseq = 0x%08X\n",
6024                 __FUNCTION__, sc->rx_prod, sc->rx_cons, sc->rx_prod_bseq);
6025         DBEXIT(BCE_VERBOSE_RECV | BCE_VERBOSE_INTR);
6026 }
6027
6028
6029 /****************************************************************************/
6030 /* Reads the transmit consumer value from the status block (skipping over   */
6031 /* chain page pointer if necessary).                                        */
6032 /*                                                                          */
6033 /* Returns:                                                                 */
6034 /*   hw_cons                                                                */
6035 /****************************************************************************/
6036 static inline u16
6037 bce_get_hw_tx_cons(struct bce_softc *sc)
6038 {
6039         u16 hw_cons;
6040
6041         mb();
6042         hw_cons = sc->status_block->status_tx_quick_consumer_index0;
6043         if ((hw_cons & USABLE_TX_BD_PER_PAGE) == USABLE_TX_BD_PER_PAGE)
6044                 hw_cons++;
6045
6046         return hw_cons;
6047 }
6048
6049
6050 /****************************************************************************/
6051 /* Handles transmit completion interrupt events.                            */
6052 /*                                                                          */
6053 /* Returns:                                                                 */
6054 /*   Nothing.                                                               */
6055 /****************************************************************************/
6056 static void
6057 bce_tx_intr(struct bce_softc *sc)
6058 {
6059         struct ifnet *ifp = sc->bce_ifp;
6060         u16 hw_tx_cons, sw_tx_cons, sw_tx_chain_cons;
6061
6062         DBENTER(BCE_VERBOSE_SEND | BCE_VERBOSE_INTR);
6063         DBRUN(sc->tx_interrupts++);
6064         DBPRINT(sc, BCE_EXTREME_SEND, "%s(enter): tx_prod = 0x%04X, "
6065                 "tx_cons = 0x%04X, tx_prod_bseq = 0x%08X\n",
6066                 __FUNCTION__, sc->tx_prod, sc->tx_cons, sc->tx_prod_bseq);
6067
6068         BCE_LOCK_ASSERT(sc);
6069
6070         /* Get the hardware's view of the TX consumer index. */
6071         hw_tx_cons = sc->hw_tx_cons = bce_get_hw_tx_cons(sc);
6072         sw_tx_cons = sc->tx_cons;
6073
6074         /* Prevent speculative reads from getting ahead of the status block. */
6075         bus_space_barrier(sc->bce_btag, sc->bce_bhandle, 0, 0,
6076                 BUS_SPACE_BARRIER_READ);
6077
6078         /* Cycle through any completed TX chain page entries. */
6079         while (sw_tx_cons != hw_tx_cons) {
6080 #ifdef BCE_DEBUG
6081                 struct tx_bd *txbd = NULL;
6082 #endif
6083                 sw_tx_chain_cons = TX_CHAIN_IDX(sw_tx_cons);
6084
6085                 DBPRINT(sc, BCE_INFO_SEND,
6086                         "%s(): hw_tx_cons = 0x%04X, sw_tx_cons = 0x%04X, "
6087                         "sw_tx_chain_cons = 0x%04X\n",
6088                         __FUNCTION__, hw_tx_cons, sw_tx_cons, sw_tx_chain_cons);
6089
6090                 DBRUNIF((sw_tx_chain_cons > MAX_TX_BD),
6091                         BCE_PRINTF("%s(%d): TX chain consumer out of range! "
6092                                 " 0x%04X > 0x%04X\n", __FILE__, __LINE__, sw_tx_chain_cons,
6093                                 (int) MAX_TX_BD);
6094                         bce_breakpoint(sc));
6095
6096                 DBRUN(txbd = &sc->tx_bd_chain[TX_PAGE(sw_tx_chain_cons)]
6097                                 [TX_IDX(sw_tx_chain_cons)]);
6098
6099                 DBRUNIF((txbd == NULL),
6100                         BCE_PRINTF("%s(%d): Unexpected NULL tx_bd[0x%04X]!\n",
6101                                 __FILE__, __LINE__, sw_tx_chain_cons);
6102                         bce_breakpoint(sc));
6103
6104                 DBRUNMSG(BCE_INFO_SEND, BCE_PRINTF("%s(): ", __FUNCTION__);
6105                         bce_dump_txbd(sc, sw_tx_chain_cons, txbd));
6106
6107                 /*
6108                  * Free the associated mbuf. Remember
6109                  * that only the last tx_bd of a packet
6110                  * has an mbuf pointer and DMA map.
6111                  */
6112                 if (sc->tx_mbuf_ptr[sw_tx_chain_cons] != NULL) {
6113
6114                         /* Validate that this is the last tx_bd. */
6115                         DBRUNIF((!(txbd->tx_bd_flags & TX_BD_FLAGS_END)),
6116                                 BCE_PRINTF("%s(%d): tx_bd END flag not set but "
6117                                 "txmbuf == NULL!\n", __FILE__, __LINE__);
6118                                 bce_breakpoint(sc));
6119
6120                         DBRUNMSG(BCE_INFO_SEND,
6121                                 BCE_PRINTF("%s(): Unloading map/freeing mbuf "
6122                                         "from tx_bd[0x%04X]\n", __FUNCTION__, sw_tx_chain_cons));
6123
6124                         /* Unmap the mbuf. */
6125                         bus_dmamap_unload(sc->tx_mbuf_tag,
6126                             sc->tx_mbuf_map[sw_tx_chain_cons]);
6127
6128                         /* Free the mbuf. */
6129                         m_freem(sc->tx_mbuf_ptr[sw_tx_chain_cons]);
6130                         sc->tx_mbuf_ptr[sw_tx_chain_cons] = NULL;
6131                         DBRUN(sc->debug_tx_mbuf_alloc--);
6132
6133                         ifp->if_opackets++;
6134                 }
6135
6136                 sc->used_tx_bd--;
6137                 sw_tx_cons = NEXT_TX_BD(sw_tx_cons);
6138
6139                 /* Refresh hw_cons to see if there's new work. */
6140                 hw_tx_cons = sc->hw_tx_cons = bce_get_hw_tx_cons(sc);
6141
6142                 /* Prevent speculative reads from getting ahead of the status block. */
6143                 bus_space_barrier(sc->bce_btag, sc->bce_bhandle, 0, 0,
6144                         BUS_SPACE_BARRIER_READ);
6145         }
6146
6147         /* Clear the TX timeout timer. */
6148         sc->watchdog_timer = 0;
6149
6150         /* Clear the tx hardware queue full flag. */
6151         if (sc->used_tx_bd < sc->max_tx_bd) {
6152                 DBRUNIF((ifp->if_drv_flags & IFF_DRV_OACTIVE),
6153                         DBPRINT(sc, BCE_INFO_SEND,
6154                                 "%s(): Open TX chain! %d/%d (used/total)\n",
6155                                 __FUNCTION__, sc->used_tx_bd, sc->max_tx_bd));
6156                 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
6157         }
6158
6159         sc->tx_cons = sw_tx_cons;
6160
6161         DBPRINT(sc, BCE_EXTREME_SEND, "%s(exit): tx_prod = 0x%04X, "
6162                 "tx_cons = 0x%04X, tx_prod_bseq = 0x%08X\n",
6163                 __FUNCTION__, sc->tx_prod, sc->tx_cons, sc->tx_prod_bseq);
6164         DBEXIT(BCE_VERBOSE_SEND | BCE_VERBOSE_INTR);
6165 }
6166
6167
6168 /****************************************************************************/
6169 /* Disables interrupt generation.                                           */
6170 /*                                                                          */
6171 /* Returns:                                                                 */
6172 /*   Nothing.                                                               */
6173 /****************************************************************************/
6174 static void
6175 bce_disable_intr(struct bce_softc *sc)
6176 {
6177         DBENTER(BCE_VERBOSE_INTR);
6178
6179         REG_WR(sc, BCE_PCICFG_INT_ACK_CMD, BCE_PCICFG_INT_ACK_CMD_MASK_INT);
6180         REG_RD(sc, BCE_PCICFG_INT_ACK_CMD);
6181
6182         DBEXIT(BCE_VERBOSE_INTR);
6183 }
6184
6185
6186 /****************************************************************************/
6187 /* Enables interrupt generation.                                            */
6188 /*                                                                          */
6189 /* Returns:                                                                 */
6190 /*   Nothing.                                                               */
6191 /****************************************************************************/
6192 static void
6193 bce_enable_intr(struct bce_softc *sc, int coal_now)
6194 {
6195         DBENTER(BCE_VERBOSE_INTR);
6196
6197         REG_WR(sc, BCE_PCICFG_INT_ACK_CMD,
6198                BCE_PCICFG_INT_ACK_CMD_INDEX_VALID |
6199                BCE_PCICFG_INT_ACK_CMD_MASK_INT | sc->last_status_idx);
6200
6201         REG_WR(sc, BCE_PCICFG_INT_ACK_CMD,
6202                BCE_PCICFG_INT_ACK_CMD_INDEX_VALID | sc->last_status_idx);
6203
6204         /* Force an immediate interrupt (whether there is new data or not). */
6205         if (coal_now)
6206                 REG_WR(sc, BCE_HC_COMMAND, sc->hc_command | BCE_HC_COMMAND_COAL_NOW);
6207
6208         DBEXIT(BCE_VERBOSE_INTR);
6209 }
6210
6211
6212 /****************************************************************************/
6213 /* Handles controller initialization.                                       */
6214 /*                                                                          */
6215 /* Returns:                                                                 */
6216 /*   Nothing.                                                               */
6217 /****************************************************************************/
6218 static void
6219 bce_init_locked(struct bce_softc *sc)
6220 {
6221         struct ifnet *ifp;
6222         u32 ether_mtu = 0;
6223
6224         DBENTER(BCE_VERBOSE_RESET);
6225
6226         BCE_LOCK_ASSERT(sc);
6227
6228         ifp = sc->bce_ifp;
6229
6230         /* Check if the driver is still running and bail out if it is. */
6231         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
6232                 goto bce_init_locked_exit;
6233
6234         bce_stop(sc);
6235
6236         if (bce_reset(sc, BCE_DRV_MSG_CODE_RESET)) {
6237                 BCE_PRINTF("%s(%d): Controller reset failed!\n",
6238                         __FILE__, __LINE__);
6239                 goto bce_init_locked_exit;
6240         }
6241
6242         if (bce_chipinit(sc)) {
6243                 BCE_PRINTF("%s(%d): Controller initialization failed!\n",
6244                         __FILE__, __LINE__);
6245                 goto bce_init_locked_exit;
6246         }
6247
6248         if (bce_blockinit(sc)) {
6249                 BCE_PRINTF("%s(%d): Block initialization failed!\n",
6250                         __FILE__, __LINE__);
6251                 goto bce_init_locked_exit;
6252         }
6253
6254         /* Load our MAC address. */
6255         bcopy(IF_LLADDR(sc->bce_ifp), sc->eaddr, ETHER_ADDR_LEN);
6256         bce_set_mac_addr(sc);
6257
6258         /*
6259          * Calculate and program the hardware Ethernet MTU
6260          * size. Be generous on the receive if we have room.
6261          */
6262 #ifdef ZERO_COPY_SOCKETS
6263         if (ifp->if_mtu <= (sc->rx_bd_mbuf_data_len + sc->pg_bd_mbuf_alloc_size))
6264                 ether_mtu = sc->rx_bd_mbuf_data_len + sc->pg_bd_mbuf_alloc_size;
6265 #else
6266         if (ifp->if_mtu <= sc->rx_bd_mbuf_data_len)
6267                 ether_mtu = sc->rx_bd_mbuf_data_len;
6268 #endif
6269         else
6270                 ether_mtu = ifp->if_mtu;
6271
6272         ether_mtu += ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + ETHER_CRC_LEN;
6273
6274         DBPRINT(sc, BCE_INFO_MISC, "%s(): setting h/w mtu = %d\n", __FUNCTION__,
6275                 ether_mtu);
6276
6277         /* Program the mtu, enabling jumbo frame support if necessary. */
6278         if (ether_mtu > (ETHER_MAX_LEN + ETHER_VLAN_ENCAP_LEN))
6279                 REG_WR(sc, BCE_EMAC_RX_MTU_SIZE,
6280                         min(ether_mtu, BCE_MAX_JUMBO_ETHER_MTU) |
6281                         BCE_EMAC_RX_MTU_SIZE_JUMBO_ENA);
6282         else
6283                 REG_WR(sc, BCE_EMAC_RX_MTU_SIZE, ether_mtu);
6284
6285         DBPRINT(sc, BCE_INFO_LOAD,
6286                 "%s(): rx_bd_mbuf_alloc_size = %d, rx_bce_mbuf_data_len = %d, "
6287                 "rx_bd_mbuf_align_pad = %d\n", __FUNCTION__,
6288                 sc->rx_bd_mbuf_alloc_size, sc->rx_bd_mbuf_data_len,
6289                 sc->rx_bd_mbuf_align_pad);
6290
6291         /* Program appropriate promiscuous/multicast filtering. */
6292         bce_set_rx_mode(sc);
6293
6294 #ifdef ZERO_COPY_SOCKETS
6295         DBPRINT(sc, BCE_INFO_LOAD, "%s(): pg_bd_mbuf_alloc_size = %d\n",
6296                 __FUNCTION__, sc->pg_bd_mbuf_alloc_size);
6297
6298         /* Init page buffer descriptor chain. */
6299         bce_init_pg_chain(sc);
6300 #endif
6301
6302         /* Init RX buffer descriptor chain. */
6303         bce_init_rx_chain(sc);
6304
6305         /* Init TX buffer descriptor chain. */
6306         bce_init_tx_chain(sc);
6307
6308         /* Enable host interrupts. */
6309         bce_enable_intr(sc, 1);
6310
6311         bce_ifmedia_upd_locked(ifp);
6312
6313         ifp->if_drv_flags |= IFF_DRV_RUNNING;
6314         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
6315
6316         callout_reset(&sc->bce_tick_callout, hz, bce_tick, sc);
6317
6318 bce_init_locked_exit:
6319         DBEXIT(BCE_VERBOSE_RESET);
6320 }
6321
6322
6323 /****************************************************************************/
6324 /* Initialize the controller just enough so that any management firmware    */
6325 /* running on the device will continue to operate correctly.                */
6326 /*                                                                          */
6327 /* Returns:                                                                 */
6328 /*   Nothing.                                                               */
6329 /****************************************************************************/
6330 static void
6331 bce_mgmt_init_locked(struct bce_softc *sc)
6332 {
6333         struct ifnet *ifp;
6334
6335         DBENTER(BCE_VERBOSE_RESET);
6336
6337         BCE_LOCK_ASSERT(sc);
6338
6339         /* Bail out if management firmware is not running. */
6340         if (!(sc->bce_flags & BCE_MFW_ENABLE_FLAG)) {
6341                 DBPRINT(sc, BCE_VERBOSE_SPECIAL,
6342                         "No management firmware running...\n");
6343                 goto bce_mgmt_init_locked_exit;
6344         }
6345
6346         ifp = sc->bce_ifp;
6347
6348         /* Enable all critical blocks in the MAC. */
6349         REG_WR(sc, BCE_MISC_ENABLE_SET_BITS, BCE_MISC_ENABLE_DEFAULT);
6350         REG_RD(sc, BCE_MISC_ENABLE_SET_BITS);
6351         DELAY(20);
6352
6353         bce_ifmedia_upd_locked(ifp);
6354
6355 bce_mgmt_init_locked_exit:
6356         DBEXIT(BCE_VERBOSE_RESET);
6357 }
6358
6359
6360 /****************************************************************************/
6361 /* Handles controller initialization when called from an unlocked routine.  */
6362 /*                                                                          */
6363 /* Returns:                                                                 */
6364 /*   Nothing.                                                               */
6365 /****************************************************************************/
6366 static void
6367 bce_init(void *xsc)
6368 {
6369         struct bce_softc *sc = xsc;
6370
6371         DBENTER(BCE_VERBOSE_RESET);
6372
6373         BCE_LOCK(sc);
6374         bce_init_locked(sc);
6375         BCE_UNLOCK(sc);
6376
6377         DBEXIT(BCE_VERBOSE_RESET);
6378 }
6379
6380
6381 /****************************************************************************/
6382 /* Encapsultes an mbuf cluster into the tx_bd chain structure and makes the */
6383 /* memory visible to the controller.                                        */
6384 /*                                                                          */
6385 /* Returns:                                                                 */
6386 /*   0 for success, positive value for failure.                             */
6387 /* Modified:                                                                */
6388 /*   m_head: May be set to NULL if MBUF is excessively fragmented.          */
6389 /****************************************************************************/
6390 static int
6391 bce_tx_encap(struct bce_softc *sc, struct mbuf **m_head)
6392 {
6393         bus_dma_segment_t segs[BCE_MAX_SEGMENTS];
6394         bus_dmamap_t map;
6395         struct tx_bd *txbd = NULL;
6396         struct mbuf *m0;
6397         struct ether_vlan_header *eh;
6398         struct ip *ip;
6399         struct tcphdr *th;
6400         u16 prod, chain_prod, etype, mss = 0, vlan_tag = 0, flags = 0;
6401         u32 prod_bseq;
6402         int hdr_len = 0, e_hlen = 0, ip_hlen = 0, tcp_hlen = 0, ip_len = 0;
6403
6404 #ifdef BCE_DEBUG
6405         u16 debug_prod;
6406 #endif
6407         int i, error, nsegs, rc = 0;
6408
6409         DBENTER(BCE_VERBOSE_SEND);
6410         DBPRINT(sc, BCE_INFO_SEND,
6411                 "%s(enter): tx_prod = 0x%04X, tx_chain_prod = %04X, "
6412                 "tx_prod_bseq = 0x%08X\n",
6413                 __FUNCTION__, sc->tx_prod, (u16) TX_CHAIN_IDX(sc->tx_prod),
6414                 sc->tx_prod_bseq);
6415
6416         /* Transfer any checksum offload flags to the bd. */
6417         m0 = *m_head;
6418         if (m0->m_pkthdr.csum_flags) {
6419                 if (m0->m_pkthdr.csum_flags & CSUM_IP)
6420                         flags |= TX_BD_FLAGS_IP_CKSUM;
6421                 if (m0->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP))
6422                         flags |= TX_BD_FLAGS_TCP_UDP_CKSUM;
6423                 if (m0->m_pkthdr.csum_flags & CSUM_TSO) {
6424                         /* For TSO the controller needs two pieces of info, */
6425                         /* the MSS and the IP+TCP options length.           */
6426                         mss = htole16(m0->m_pkthdr.tso_segsz);
6427
6428                         /* Map the header and find the Ethernet type & header length */
6429                         eh = mtod(m0, struct ether_vlan_header *);
6430                         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
6431                                 etype = ntohs(eh->evl_proto);
6432                                 e_hlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
6433                         } else {
6434                                 etype = ntohs(eh->evl_encap_proto);
6435                                 e_hlen = ETHER_HDR_LEN;
6436                         }
6437
6438                         /* Check for supported TSO Ethernet types (only IPv4 for now) */
6439                         switch (etype) {
6440                                 case ETHERTYPE_IP:
6441                                         ip = (struct ip *)(m0->m_data + e_hlen);
6442
6443                                         /* TSO only supported for TCP protocol */
6444                                         if (ip->ip_p != IPPROTO_TCP) {
6445                                                 BCE_PRINTF("%s(%d): TSO enabled for non-TCP frame!.\n",
6446                                                         __FILE__, __LINE__);
6447                                                 goto bce_tx_encap_skip_tso;
6448                                         }
6449
6450                                         /* Get IP header length in bytes (min 20) */
6451                                         ip_hlen = ip->ip_hl << 2;
6452
6453                                         /* Get the TCP header length in bytes (min 20) */
6454                                         th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
6455                                         tcp_hlen = (th->th_off << 2);
6456
6457                                         /* IP header length and checksum will be calc'd by hardware */
6458                                         ip_len = ip->ip_len;
6459                                         ip->ip_len = 0;
6460                                         ip->ip_sum = 0;
6461                                         break;
6462                                 case ETHERTYPE_IPV6:
6463                                         BCE_PRINTF("%s(%d): TSO over IPv6 not supported!.\n",
6464                                                 __FILE__, __LINE__);
6465                                         goto bce_tx_encap_skip_tso;
6466                                 default:
6467                                         BCE_PRINTF("%s(%d): TSO enabled for unsupported protocol!.\n",
6468                                                 __FILE__, __LINE__);
6469                                         goto bce_tx_encap_skip_tso;
6470                         }
6471
6472                         hdr_len = e_hlen + ip_hlen + tcp_hlen;
6473
6474                         DBPRINT(sc, BCE_EXTREME_SEND,
6475                                 "%s(): hdr_len = %d, e_hlen = %d, ip_hlen = %d, tcp_hlen = %d, ip_len = %d\n",
6476                                  __FUNCTION__, hdr_len, e_hlen, ip_hlen, tcp_hlen, ip_len);
6477
6478                         /* Set the LSO flag in the TX BD */
6479                         flags |= TX_BD_FLAGS_SW_LSO;
6480                         /* Set the length of IP + TCP options (in 32 bit words) */
6481                         flags |= (((ip_hlen + tcp_hlen - 40) >> 2) << 8);
6482
6483 bce_tx_encap_skip_tso:
6484                         DBRUN(sc->requested_tso_frames++);
6485                 }
6486         }
6487
6488         /* Transfer any VLAN tags to the bd. */
6489         if (m0->m_flags & M_VLANTAG) {
6490                 flags |= TX_BD_FLAGS_VLAN_TAG;
6491                 vlan_tag = m0->m_pkthdr.ether_vtag;
6492         }
6493
6494         /* Map the mbuf into DMAable memory. */
6495         prod = sc->tx_prod;
6496         chain_prod = TX_CHAIN_IDX(prod);
6497         map = sc->tx_mbuf_map[chain_prod];
6498
6499         /* Map the mbuf into our DMA address space. */
6500         error = bus_dmamap_load_mbuf_sg(sc->tx_mbuf_tag, map, m0,
6501             segs, &nsegs, BUS_DMA_NOWAIT);
6502
6503         /* Check if the DMA mapping was successful */
6504         if (error == EFBIG) {
6505
6506                 sc->fragmented_mbuf_count++;
6507
6508                 /* Try to defrag the mbuf. */
6509                 m0 = m_defrag(*m_head, M_DONTWAIT);
6510                 if (m0 == NULL) {
6511                         /* Defrag was unsuccessful */
6512                         m_freem(*m_head);
6513                         *m_head = NULL;
6514                         sc->mbuf_alloc_failed_count++;
6515                         rc = ENOBUFS;
6516                         goto bce_tx_encap_exit;
6517                 }
6518
6519                 /* Defrag was successful, try mapping again */
6520                 *m_head = m0;
6521                 error = bus_dmamap_load_mbuf_sg(sc->tx_mbuf_tag, map, m0,
6522                     segs, &nsegs, BUS_DMA_NOWAIT);
6523
6524                 /* Still getting an error after a defrag. */
6525                 if (error == ENOMEM) {
6526                         /* Insufficient DMA buffers available. */
6527                         sc->dma_map_addr_tx_failed_count++;
6528                         rc = error;
6529                         goto bce_tx_encap_exit;
6530                 } else if (error != 0) {
6531                         /* Still can't map the mbuf, release it and return an error. */
6532                         BCE_PRINTF(
6533                             "%s(%d): Unknown error mapping mbuf into TX chain!\n",
6534                             __FILE__, __LINE__);
6535                         m_freem(m0);
6536                         *m_head = NULL;
6537                         sc->dma_map_addr_tx_failed_count++;
6538                         rc = ENOBUFS;
6539                         goto bce_tx_encap_exit;
6540                 }
6541         } else if (error == ENOMEM) {
6542                 /* Insufficient DMA buffers available. */
6543                 sc->dma_map_addr_tx_failed_count++;
6544                 rc = error;
6545                 goto bce_tx_encap_exit;
6546         } else if (error != 0) {
6547                 m_freem(m0);
6548                 *m_head = NULL;
6549                 sc->dma_map_addr_tx_failed_count++;
6550                 rc = error;
6551                 goto bce_tx_encap_exit;
6552         }
6553
6554         /* Make sure there's room in the chain */
6555         if (nsegs > (sc->max_tx_bd - sc->used_tx_bd)) {
6556                 bus_dmamap_unload(sc->tx_mbuf_tag, map);
6557                 rc = ENOBUFS;
6558                 goto bce_tx_encap_exit;
6559         }
6560
6561         /* prod points to an empty tx_bd at this point. */
6562         prod_bseq  = sc->tx_prod_bseq;
6563
6564 #ifdef BCE_DEBUG
6565         debug_prod = chain_prod;
6566 #endif
6567
6568         DBPRINT(sc, BCE_INFO_SEND,
6569                 "%s(start): prod = 0x%04X, chain_prod = 0x%04X, "
6570                 "prod_bseq = 0x%08X\n",
6571                 __FUNCTION__, prod, chain_prod, prod_bseq);
6572
6573         /*
6574          * Cycle through each mbuf segment that makes up
6575          * the outgoing frame, gathering the mapping info
6576          * for that segment and creating a tx_bd for
6577          * the mbuf.
6578          */
6579         for (i = 0; i < nsegs ; i++) {
6580
6581                 chain_prod = TX_CHAIN_IDX(prod);
6582                 txbd= &sc->tx_bd_chain[TX_PAGE(chain_prod)][TX_IDX(chain_prod)];
6583
6584                 txbd->tx_bd_haddr_lo = htole32(BCE_ADDR_LO(segs[i].ds_addr));
6585                 txbd->tx_bd_haddr_hi = htole32(BCE_ADDR_HI(segs[i].ds_addr));
6586                 txbd->tx_bd_mss_nbytes = htole32(mss << 16) | htole16(segs[i].ds_len);
6587                 txbd->tx_bd_vlan_tag = htole16(vlan_tag);
6588                 txbd->tx_bd_flags = htole16(flags);
6589                 prod_bseq += segs[i].ds_len;
6590                 if (i == 0)
6591                         txbd->tx_bd_flags |= htole16(TX_BD_FLAGS_START);
6592                 prod = NEXT_TX_BD(prod);
6593         }
6594
6595         /* Set the END flag on the last TX buffer descriptor. */
6596         txbd->tx_bd_flags |= htole16(TX_BD_FLAGS_END);
6597
6598         DBRUNMSG(BCE_EXTREME_SEND, bce_dump_tx_chain(sc, debug_prod, nsegs));
6599
6600         DBPRINT(sc, BCE_INFO_SEND,
6601                 "%s( end ): prod = 0x%04X, chain_prod = 0x%04X, "
6602                 "prod_bseq = 0x%08X\n",
6603                 __FUNCTION__, prod, chain_prod, prod_bseq);
6604
6605         /*
6606          * Ensure that the mbuf pointer for this transmission
6607          * is placed at the array index of the last
6608          * descriptor in this chain.  This is done
6609          * because a single map is used for all
6610          * segments of the mbuf and we don't want to
6611          * unload the map before all of the segments
6612          * have been freed.
6613          */
6614         sc->tx_mbuf_ptr[chain_prod] = m0;
6615         sc->used_tx_bd += nsegs;
6616
6617         /* Update some debug statistic counters */
6618         DBRUNIF((sc->used_tx_bd > sc->tx_hi_watermark),
6619                 sc->tx_hi_watermark = sc->used_tx_bd);
6620         DBRUNIF((sc->used_tx_bd == sc->max_tx_bd), sc->tx_full_count++);
6621         DBRUNIF(sc->debug_tx_mbuf_alloc++);
6622
6623         DBRUNMSG(BCE_EXTREME_SEND, bce_dump_tx_mbuf_chain(sc, chain_prod, 1));
6624
6625         /* prod points to the next free tx_bd at this point. */
6626         sc->tx_prod = prod;
6627         sc->tx_prod_bseq = prod_bseq;
6628
6629         DBPRINT(sc, BCE_INFO_SEND,
6630                 "%s(exit): prod = 0x%04X, chain_prod = %04X, "
6631                 "prod_bseq = 0x%08X\n",
6632                 __FUNCTION__, sc->tx_prod, (u16) TX_CHAIN_IDX(sc->tx_prod),
6633                 sc->tx_prod_bseq);
6634
6635 bce_tx_encap_exit:
6636         DBEXIT(BCE_VERBOSE_SEND);
6637         return(rc);
6638 }
6639
6640
6641 /****************************************************************************/
6642 /* Main transmit routine when called from another routine with a lock.      */
6643 /*                                                                          */
6644 /* Returns:                                                                 */
6645 /*   Nothing.                                                               */
6646 /****************************************************************************/
6647 static void
6648 bce_start_locked(struct ifnet *ifp)
6649 {
6650         struct bce_softc *sc = ifp->if_softc;
6651         struct mbuf *m_head = NULL;
6652         int count = 0;
6653         u16 tx_prod, tx_chain_prod;
6654
6655         DBENTER(BCE_VERBOSE_SEND | BCE_VERBOSE_CTX);
6656
6657         BCE_LOCK_ASSERT(sc);
6658
6659         /* prod points to the next free tx_bd. */
6660         tx_prod = sc->tx_prod;
6661         tx_chain_prod = TX_CHAIN_IDX(tx_prod);
6662
6663         DBPRINT(sc, BCE_INFO_SEND,
6664                 "%s(enter): tx_prod = 0x%04X, tx_chain_prod = 0x%04X, "
6665                 "tx_prod_bseq = 0x%08X\n",
6666                 __FUNCTION__, tx_prod, tx_chain_prod, sc->tx_prod_bseq);
6667
6668         /* If there's no link or the transmit queue is empty then just exit. */
6669         if (!sc->bce_link) {
6670                 DBPRINT(sc, BCE_INFO_SEND, "%s(): No link.\n",
6671                         __FUNCTION__);
6672                 goto bce_start_locked_exit;
6673         }
6674
6675         if (IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
6676                 DBPRINT(sc, BCE_INFO_SEND, "%s(): Transmit queue empty.\n",
6677                         __FUNCTION__);
6678                 goto bce_start_locked_exit;
6679         }
6680
6681         /*
6682          * Keep adding entries while there is space in the ring.
6683          */
6684         while (sc->used_tx_bd < sc->max_tx_bd) {
6685
6686                 /* Check for any frames to send. */
6687                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
6688
6689                 /* Stop when the transmit queue is empty. */
6690                 if (m_head == NULL)
6691                         break;
6692
6693                 /*
6694                  * Pack the data into the transmit ring. If we
6695                  * don't have room, place the mbuf back at the
6696                  * head of the queue and set the OACTIVE flag
6697                  * to wait for the NIC to drain the chain.
6698                  */
6699                 if (bce_tx_encap(sc, &m_head)) {
6700                         /* No room, put the frame back on the transmit queue. */
6701                         if (m_head != NULL)
6702                                 IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
6703                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
6704                         DBPRINT(sc, BCE_INFO_SEND,
6705                                 "TX chain is closed for business! Total tx_bd used = %d\n",
6706                                 sc->used_tx_bd);
6707                         break;
6708                 }
6709
6710                 count++;
6711
6712                 /* Send a copy of the frame to any BPF listeners. */
6713                 ETHER_BPF_MTAP(ifp, m_head);
6714         }
6715
6716         /* Exit if no packets were dequeued. */
6717         if (count == 0) {
6718                 DBPRINT(sc, BCE_VERBOSE_SEND, "%s(): No packets were dequeued\n",
6719                         __FUNCTION__);
6720                 goto bce_start_locked_exit;
6721         }
6722
6723         DBPRINT(sc, BCE_VERBOSE_SEND, "%s(): Inserted %d frames into send queue.\n",
6724                 __FUNCTION__, count);
6725
6726         REG_WR(sc, BCE_MQ_COMMAND, REG_RD(sc, BCE_MQ_COMMAND) | BCE_MQ_COMMAND_NO_MAP_ERROR);
6727
6728         /* Write the mailbox and tell the chip about the waiting tx_bd's. */
6729         DBPRINT(sc, BCE_VERBOSE_SEND, "%s(): MB_GET_CID_ADDR(TX_CID) = 0x%08X; "
6730                 "BCE_L2MQ_TX_HOST_BIDX = 0x%08X, sc->tx_prod = 0x%04X\n",
6731                 __FUNCTION__,
6732                 MB_GET_CID_ADDR(TX_CID), BCE_L2MQ_TX_HOST_BIDX, sc->tx_prod);
6733         REG_WR16(sc, MB_GET_CID_ADDR(TX_CID) + BCE_L2MQ_TX_HOST_BIDX, sc->tx_prod);
6734         DBPRINT(sc, BCE_VERBOSE_SEND, "%s(): MB_GET_CID_ADDR(TX_CID) = 0x%08X; "
6735                 "BCE_L2MQ_TX_HOST_BSEQ = 0x%08X, sc->tx_prod_bseq = 0x%04X\n",
6736                 __FUNCTION__,
6737                 MB_GET_CID_ADDR(TX_CID), BCE_L2MQ_TX_HOST_BSEQ, sc->tx_prod_bseq);
6738         REG_WR(sc, MB_GET_CID_ADDR(TX_CID) + BCE_L2MQ_TX_HOST_BSEQ, sc->tx_prod_bseq);
6739
6740         /* Set the tx timeout. */
6741         sc->watchdog_timer = BCE_TX_TIMEOUT;
6742
6743         DBRUNMSG(BCE_VERBOSE_SEND, bce_dump_ctx(sc, TX_CID));
6744         DBRUNMSG(BCE_VERBOSE_SEND, bce_dump_mq_regs(sc));
6745
6746 bce_start_locked_exit:
6747         DBEXIT(BCE_VERBOSE_SEND | BCE_VERBOSE_CTX);
6748         return;
6749 }
6750
6751
6752 /****************************************************************************/
6753 /* Main transmit routine when called from another routine without a lock.   */
6754 /*                                                                          */
6755 /* Returns:                                                                 */
6756 /*   Nothing.                                                               */
6757 /****************************************************************************/
6758 static void
6759 bce_start(struct ifnet *ifp)
6760 {
6761         struct bce_softc *sc = ifp->if_softc;
6762
6763         DBENTER(BCE_VERBOSE_SEND);
6764
6765         BCE_LOCK(sc);
6766         bce_start_locked(ifp);
6767         BCE_UNLOCK(sc);
6768
6769         DBEXIT(BCE_VERBOSE_SEND);
6770 }
6771
6772
6773 /****************************************************************************/
6774 /* Handles any IOCTL calls from the operating system.                       */
6775 /*                                                                          */
6776 /* Returns:                                                                 */
6777 /*   0 for success, positive value for failure.                             */
6778 /****************************************************************************/
6779 static int
6780 bce_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
6781 {
6782         struct bce_softc *sc = ifp->if_softc;
6783         struct ifreq *ifr = (struct ifreq *) data;
6784         struct mii_data *mii;
6785         int mask, error = 0;
6786
6787         DBENTER(BCE_VERBOSE_MISC);
6788
6789         switch(command) {
6790
6791                 /* Set the interface MTU. */
6792                 case SIOCSIFMTU:
6793                         /* Check that the MTU setting is supported. */
6794                         if ((ifr->ifr_mtu < BCE_MIN_MTU) ||
6795                                 (ifr->ifr_mtu > BCE_MAX_JUMBO_MTU)) {
6796                                 error = EINVAL;
6797                                 break;
6798                         }
6799
6800                         DBPRINT(sc, BCE_INFO_MISC,
6801                                 "SIOCSIFMTU: Changing MTU from %d to %d\n",
6802                                 (int) ifp->if_mtu, (int) ifr->ifr_mtu);
6803
6804                         BCE_LOCK(sc);
6805                         ifp->if_mtu = ifr->ifr_mtu;
6806                         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
6807 #ifdef ZERO_COPY_SOCKETS
6808                         /* No buffer allocation size changes are necessary. */
6809 #else
6810                         /* Recalculate our buffer allocation sizes. */
6811                         if ((ifp->if_mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + ETHER_CRC_LEN) > MCLBYTES) {
6812                                 sc->rx_bd_mbuf_alloc_size = MJUM9BYTES;
6813                                 sc->rx_bd_mbuf_align_pad  = roundup2(MJUM9BYTES, 16) - MJUM9BYTES;
6814                                 sc->rx_bd_mbuf_data_len   = sc->rx_bd_mbuf_alloc_size -
6815                                         sc->rx_bd_mbuf_align_pad;
6816                         } else {
6817                                 sc->rx_bd_mbuf_alloc_size = MCLBYTES;
6818                                 sc->rx_bd_mbuf_align_pad  = roundup2(MCLBYTES, 16) - MCLBYTES;
6819                                 sc->rx_bd_mbuf_data_len   = sc->rx_bd_mbuf_alloc_size -
6820                                         sc->rx_bd_mbuf_align_pad;
6821                         }
6822 #endif
6823
6824                         bce_init_locked(sc);
6825                         BCE_UNLOCK(sc);
6826                         break;
6827
6828                 /* Set interface flags. */
6829                 case SIOCSIFFLAGS:
6830                         DBPRINT(sc, BCE_VERBOSE_SPECIAL, "Received SIOCSIFFLAGS\n");
6831
6832                         BCE_LOCK(sc);
6833
6834                         /* Check if the interface is up. */
6835                         if (ifp->if_flags & IFF_UP) {
6836                                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
6837                                         /* Change promiscuous/multicast flags as necessary. */
6838                                         bce_set_rx_mode(sc);
6839                                 } else {
6840                                         /* Start the HW */
6841                                         bce_init_locked(sc);
6842                                 }
6843                         } else {
6844                                 /* The interface is down, check if driver is running. */
6845                                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
6846                                         bce_stop(sc);
6847
6848                                         /* If MFW is running, restart the controller a bit. */
6849                                         if (sc->bce_flags & BCE_MFW_ENABLE_FLAG) {
6850                                                 bce_reset(sc, BCE_DRV_MSG_CODE_RESET);
6851                                                 bce_chipinit(sc);
6852                                                 bce_mgmt_init_locked(sc);
6853                                         }
6854                                 }
6855                         }
6856
6857                         BCE_UNLOCK(sc);
6858                         error = 0;
6859
6860                         break;
6861
6862                 /* Add/Delete multicast address */
6863                 case SIOCADDMULTI:
6864                 case SIOCDELMULTI:
6865                         DBPRINT(sc, BCE_VERBOSE_MISC, "Received SIOCADDMULTI/SIOCDELMULTI\n");
6866
6867                         BCE_LOCK(sc);
6868                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
6869                                 bce_set_rx_mode(sc);
6870                                 error = 0;
6871                         }
6872                         BCE_UNLOCK(sc);
6873
6874                         break;
6875
6876                 /* Set/Get Interface media */
6877                 case SIOCSIFMEDIA:
6878                 case SIOCGIFMEDIA:
6879                         DBPRINT(sc, BCE_VERBOSE_MISC, "Received SIOCSIFMEDIA/SIOCGIFMEDIA\n");
6880
6881                         mii = device_get_softc(sc->bce_miibus);
6882                         error = ifmedia_ioctl(ifp, ifr,
6883                             &mii->mii_media, command);
6884                         break;
6885
6886                 /* Set interface capability */
6887                 case SIOCSIFCAP:
6888                         mask = ifr->ifr_reqcap ^ ifp->if_capenable;
6889                         DBPRINT(sc, BCE_INFO_MISC, "Received SIOCSIFCAP = 0x%08X\n", (u32) mask);
6890
6891                         /* Toggle the TX checksum capabilites enable flag. */
6892                         if (mask & IFCAP_TXCSUM) {
6893                                 ifp->if_capenable ^= IFCAP_TXCSUM;
6894                                 if (IFCAP_TXCSUM & ifp->if_capenable)
6895                                         ifp->if_hwassist = BCE_IF_HWASSIST;
6896                                 else
6897                                         ifp->if_hwassist = 0;
6898                         }
6899
6900                         /* Toggle the RX checksum capabilities enable flag. */
6901                         if (mask & IFCAP_RXCSUM) {
6902                                 ifp->if_capenable ^= IFCAP_RXCSUM;
6903                                 if (IFCAP_RXCSUM & ifp->if_capenable)
6904                                         ifp->if_hwassist = BCE_IF_HWASSIST;
6905                                 else
6906                                         ifp->if_hwassist = 0;
6907                         }
6908
6909                         /* Toggle the TSO capabilities enable flag. */
6910                         if (bce_tso_enable && (mask & IFCAP_TSO4)) {
6911                                 ifp->if_capenable ^= IFCAP_TSO4;
6912                                 if (IFCAP_RXCSUM & ifp->if_capenable)
6913                                         ifp->if_hwassist = BCE_IF_HWASSIST;
6914                                 else
6915                                         ifp->if_hwassist = 0;
6916                         }
6917
6918                         /* Toggle VLAN_MTU capabilities enable flag. */
6919                         if (mask & IFCAP_VLAN_MTU) {
6920                                 BCE_PRINTF("%s(%d): Changing VLAN_MTU not supported.\n",
6921                                         __FILE__, __LINE__);
6922                         }
6923
6924                         /* Toggle VLANHWTAG capabilities enabled flag. */
6925                         if (mask & IFCAP_VLAN_HWTAGGING) {
6926                                 if (sc->bce_flags & BCE_MFW_ENABLE_FLAG)
6927                                         BCE_PRINTF("%s(%d): Cannot change VLAN_HWTAGGING while "
6928                                                 "management firmware (ASF/IPMI/UMP) is running!\n",
6929                                                 __FILE__, __LINE__);
6930                                 else
6931                                         BCE_PRINTF("%s(%d): Changing VLAN_HWTAGGING not supported!\n",
6932                                                 __FILE__, __LINE__);
6933                         }
6934
6935                         break;
6936                 default:
6937                         /* We don't know how to handle the IOCTL, pass it on. */
6938                         error = ether_ioctl(ifp, command, data);
6939                         break;
6940         }
6941
6942         DBEXIT(BCE_VERBOSE_MISC);
6943         return(error);
6944 }
6945
6946
6947 /****************************************************************************/
6948 /* Transmit timeout handler.                                                */
6949 /*                                                                          */
6950 /* Returns:                                                                 */
6951 /*   Nothing.                                                               */
6952 /****************************************************************************/
6953 static void
6954 bce_watchdog(struct bce_softc *sc)
6955 {
6956         DBENTER(BCE_EXTREME_SEND);
6957
6958         BCE_LOCK_ASSERT(sc);
6959
6960         /* If the watchdog timer hasn't expired then just exit. */
6961         if (sc->watchdog_timer == 0 || --sc->watchdog_timer)
6962                 goto bce_watchdog_exit;
6963
6964         /* If pause frames are active then don't reset the hardware. */
6965         /* ToDo: Should we reset the timer here? */
6966         if (REG_RD(sc, BCE_EMAC_TX_STATUS) & BCE_EMAC_TX_STATUS_XOFFED)
6967                 goto bce_watchdog_exit;
6968
6969         BCE_PRINTF("%s(%d): Watchdog timeout occurred, resetting!\n",
6970                 __FILE__, __LINE__);
6971
6972         DBRUNMSG(BCE_INFO,
6973                 bce_dump_driver_state(sc);
6974                 bce_dump_status_block(sc);
6975                 bce_dump_stats_block(sc);
6976                 bce_dump_ftqs(sc);
6977                 bce_dump_txp_state(sc, 0);
6978                 bce_dump_rxp_state(sc, 0);
6979                 bce_dump_tpat_state(sc, 0);
6980                 bce_dump_cp_state(sc, 0);
6981                 bce_dump_com_state(sc, 0));
6982
6983         DBRUN(bce_breakpoint(sc));
6984
6985         sc->bce_ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
6986
6987         bce_init_locked(sc);
6988         sc->bce_ifp->if_oerrors++;
6989
6990 bce_watchdog_exit:
6991         DBEXIT(BCE_EXTREME_SEND);
6992 }
6993
6994
6995 /*
6996  * Interrupt handler.
6997  */
6998 /****************************************************************************/
6999 /* Main interrupt entry point.  Verifies that the controller generated the  */
7000 /* interrupt and then calls a separate routine for handle the various       */
7001 /* interrupt causes (PHY, TX, RX).                                          */
7002 /*                                                                          */
7003 /* Returns:                                                                 */
7004 /*   0 for success, positive value for failure.                             */
7005 /****************************************************************************/
7006 static void
7007 bce_intr(void *xsc)
7008 {
7009         struct bce_softc *sc;
7010         struct ifnet *ifp;
7011         u32 status_attn_bits;
7012         u16 hw_rx_cons, hw_tx_cons;
7013
7014         sc = xsc;
7015         ifp = sc->bce_ifp;
7016
7017         DBENTER(BCE_VERBOSE_SEND | BCE_VERBOSE_RECV | BCE_VERBOSE_INTR);
7018         DBRUNMSG(BCE_VERBOSE_INTR, bce_dump_status_block(sc));
7019
7020         BCE_LOCK(sc);
7021
7022         DBRUN(sc->interrupts_generated++);
7023
7024         bus_dmamap_sync(sc->status_tag, sc->status_map,
7025             BUS_DMASYNC_POSTWRITE);
7026
7027         /*
7028          * If the hardware status block index
7029          * matches the last value read by the
7030          * driver and we haven't asserted our
7031          * interrupt then there's nothing to do.
7032          */
7033         if ((sc->status_block->status_idx == sc->last_status_idx) &&
7034                 (REG_RD(sc, BCE_PCICFG_MISC_STATUS) & BCE_PCICFG_MISC_STATUS_INTA_VALUE)) {
7035                         DBPRINT(sc, BCE_VERBOSE_INTR, "%s(): Spurious interrupt.\n",
7036                                 __FUNCTION__);
7037                         goto bce_intr_exit;
7038         }
7039
7040         /* Ack the interrupt and stop others from occuring. */
7041         REG_WR(sc, BCE_PCICFG_INT_ACK_CMD,
7042                 BCE_PCICFG_INT_ACK_CMD_USE_INT_HC_PARAM |
7043                 BCE_PCICFG_INT_ACK_CMD_MASK_INT);
7044
7045         /* Check if the hardware has finished any work. */
7046         hw_rx_cons = bce_get_hw_rx_cons(sc);
7047         hw_tx_cons = bce_get_hw_tx_cons(sc);
7048
7049         /* Keep processing data as long as there is work to do. */
7050         for (;;) {
7051
7052                 status_attn_bits = sc->status_block->status_attn_bits;
7053
7054         DBRUNIF(DB_RANDOMTRUE(unexpected_attention_sim_control),
7055                 BCE_PRINTF("Simulating unexpected status attention bit set.");
7056                 sc->unexpected_attention_sim_count++;
7057                 status_attn_bits = status_attn_bits | STATUS_ATTN_BITS_PARITY_ERROR);
7058
7059                 /* Was it a link change interrupt? */
7060                 if ((status_attn_bits & STATUS_ATTN_BITS_LINK_STATE) !=
7061                         (sc->status_block->status_attn_bits_ack & STATUS_ATTN_BITS_LINK_STATE)) {
7062                         bce_phy_intr(sc);
7063
7064                         /* Clear any transient status updates during link state change. */
7065                         REG_WR(sc, BCE_HC_COMMAND,
7066                                 sc->hc_command | BCE_HC_COMMAND_COAL_NOW_WO_INT);
7067                         REG_RD(sc, BCE_HC_COMMAND);
7068                 }
7069
7070                 /* If any other attention is asserted then the chip is toast. */
7071                 if (((status_attn_bits & ~STATUS_ATTN_BITS_LINK_STATE) !=
7072                         (sc->status_block->status_attn_bits_ack &
7073                         ~STATUS_ATTN_BITS_LINK_STATE))) {
7074
7075                 sc->unexpected_attention_count++;
7076
7077                         BCE_PRINTF("%s(%d): Fatal attention detected: 0x%08X\n",
7078                                 __FILE__, __LINE__, sc->status_block->status_attn_bits);
7079
7080                         DBRUNMSG(BCE_FATAL,
7081                                 if (unexpected_attention_sim_control == 0)
7082                                         bce_breakpoint(sc));
7083
7084                         bce_init_locked(sc);
7085                         goto bce_intr_exit;
7086                 }
7087
7088                 /* Check for any completed RX frames. */
7089                 if (hw_rx_cons != sc->hw_rx_cons)
7090                         bce_rx_intr(sc);
7091
7092                 /* Check for any completed TX frames. */
7093                 if (hw_tx_cons != sc->hw_tx_cons)
7094                         bce_tx_intr(sc);
7095
7096                 /* Save the status block index value for use during the next interrupt. */
7097                 sc->last_status_idx = sc->status_block->status_idx;
7098
7099                 /* Prevent speculative reads from getting ahead of the status block. */
7100                 bus_space_barrier(sc->bce_btag, sc->bce_bhandle, 0, 0,
7101                         BUS_SPACE_BARRIER_READ);
7102
7103                 /* If there's no work left then exit the interrupt service routine. */
7104                 hw_rx_cons = bce_get_hw_rx_cons(sc);
7105                 hw_tx_cons = bce_get_hw_tx_cons(sc);
7106
7107                 if ((hw_rx_cons == sc->hw_rx_cons) && (hw_tx_cons == sc->hw_tx_cons))
7108                         break;
7109
7110         }
7111
7112         bus_dmamap_sync(sc->status_tag, sc->status_map,
7113             BUS_DMASYNC_PREWRITE);
7114
7115         /* Re-enable interrupts. */
7116         bce_enable_intr(sc, 0);
7117
7118         /* Handle any frames that arrived while handling the interrupt. */
7119         if (ifp->if_drv_flags & IFF_DRV_RUNNING && !IFQ_DRV_IS_EMPTY(&ifp->if_snd))
7120                 bce_start_locked(ifp);
7121
7122 bce_intr_exit:
7123         BCE_UNLOCK(sc);
7124
7125         DBEXIT(BCE_VERBOSE_SEND | BCE_VERBOSE_RECV | BCE_VERBOSE_INTR);
7126 }
7127
7128
7129 /****************************************************************************/
7130 /* Programs the various packet receive modes (broadcast and multicast).     */
7131 /*                                                                          */
7132 /* Returns:                                                                 */
7133 /*   Nothing.                                                               */
7134 /****************************************************************************/
7135 static void
7136 bce_set_rx_mode(struct bce_softc *sc)
7137 {
7138         struct ifnet *ifp;
7139         struct ifmultiaddr *ifma;
7140         u32 hashes[NUM_MC_HASH_REGISTERS] = { 0, 0, 0, 0, 0, 0, 0, 0 };
7141         u32 rx_mode, sort_mode;
7142         int h, i;
7143
7144         DBENTER(BCE_VERBOSE_MISC);
7145
7146         BCE_LOCK_ASSERT(sc);
7147
7148         ifp = sc->bce_ifp;
7149
7150         /* Initialize receive mode default settings. */
7151         rx_mode   = sc->rx_mode & ~(BCE_EMAC_RX_MODE_PROMISCUOUS |
7152                             BCE_EMAC_RX_MODE_KEEP_VLAN_TAG);
7153         sort_mode = 1 | BCE_RPM_SORT_USER0_BC_EN;
7154
7155         /*
7156          * ASF/IPMI/UMP firmware requires that VLAN tag stripping
7157          * be enbled.
7158          */
7159         if (!(BCE_IF_CAPABILITIES & IFCAP_VLAN_HWTAGGING) &&
7160                 (!(sc->bce_flags & BCE_MFW_ENABLE_FLAG)))
7161                 rx_mode |= BCE_EMAC_RX_MODE_KEEP_VLAN_TAG;
7162
7163         /*
7164          * Check for promiscuous, all multicast, or selected
7165          * multicast address filtering.
7166          */
7167         if (ifp->if_flags & IFF_PROMISC) {
7168                 DBPRINT(sc, BCE_INFO_MISC, "Enabling promiscuous mode.\n");
7169
7170                 /* Enable promiscuous mode. */
7171                 rx_mode |= BCE_EMAC_RX_MODE_PROMISCUOUS;
7172                 sort_mode |= BCE_RPM_SORT_USER0_PROM_EN;
7173         } else if (ifp->if_flags & IFF_ALLMULTI) {
7174                 DBPRINT(sc, BCE_INFO_MISC, "Enabling all multicast mode.\n");
7175
7176                 /* Enable all multicast addresses. */
7177                 for (i = 0; i < NUM_MC_HASH_REGISTERS; i++) {
7178                         REG_WR(sc, BCE_EMAC_MULTICAST_HASH0 + (i * 4), 0xffffffff);
7179         }
7180                 sort_mode |= BCE_RPM_SORT_USER0_MC_EN;
7181         } else {
7182                 /* Accept one or more multicast(s). */
7183                 DBPRINT(sc, BCE_INFO_MISC, "Enabling selective multicast mode.\n");
7184
7185                 IF_ADDR_LOCK(ifp);
7186                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
7187                         if (ifma->ifma_addr->sa_family != AF_LINK)
7188                                 continue;
7189                         h = ether_crc32_le(LLADDR((struct sockaddr_dl *)
7190                             ifma->ifma_addr), ETHER_ADDR_LEN) & 0xFF;
7191                             hashes[(h & 0xE0) >> 5] |= 1 << (h & 0x1F);
7192                 }
7193                 IF_ADDR_UNLOCK(ifp);
7194
7195                 for (i = 0; i < NUM_MC_HASH_REGISTERS; i++)
7196                         REG_WR(sc, BCE_EMAC_MULTICAST_HASH0 + (i * 4), hashes[i]);
7197
7198                 sort_mode |= BCE_RPM_SORT_USER0_MC_HSH_EN;
7199         }
7200
7201         /* Only make changes if the recive mode has actually changed. */
7202         if (rx_mode != sc->rx_mode) {
7203                 DBPRINT(sc, BCE_VERBOSE_MISC, "Enabling new receive mode: 0x%08X\n",
7204                         rx_mode);
7205
7206                 sc->rx_mode = rx_mode;
7207                 REG_WR(sc, BCE_EMAC_RX_MODE, rx_mode);
7208         }
7209
7210         /* Disable and clear the exisitng sort before enabling a new sort. */
7211         REG_WR(sc, BCE_RPM_SORT_USER0, 0x0);
7212         REG_WR(sc, BCE_RPM_SORT_USER0, sort_mode);
7213         REG_WR(sc, BCE_RPM_SORT_USER0, sort_mode | BCE_RPM_SORT_USER0_ENA);
7214
7215         DBEXIT(BCE_VERBOSE_MISC);
7216 }
7217
7218
7219 /****************************************************************************/
7220 /* Called periodically to updates statistics from the controllers           */
7221 /* statistics block.                                                        */
7222 /*                                                                          */
7223 /* Returns:                                                                 */
7224 /*   Nothing.                                                               */
7225 /****************************************************************************/
7226 static void
7227 bce_stats_update(struct bce_softc *sc)
7228 {
7229         struct ifnet *ifp;
7230         struct statistics_block *stats;
7231
7232         DBENTER(BCE_EXTREME_MISC);
7233
7234         ifp = sc->bce_ifp;
7235
7236         stats = (struct statistics_block *) sc->stats_block;
7237
7238         /*
7239          * Certain controllers don't report
7240          * carrier sense errors correctly.
7241          * See errata E11_5708CA0_1165.
7242          */
7243         if (!(BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5706) &&
7244             !(BCE_CHIP_ID(sc) == BCE_CHIP_ID_5708_A0))
7245                 ifp->if_oerrors += (u_long) stats->stat_Dot3StatsCarrierSenseErrors;
7246
7247         /*
7248          * Update the sysctl statistics from the
7249          * hardware statistics.
7250          */
7251         sc->stat_IfHCInOctets =
7252                 ((u64) stats->stat_IfHCInOctets_hi << 32) +
7253                  (u64) stats->stat_IfHCInOctets_lo;
7254
7255         sc->stat_IfHCInBadOctets =
7256                 ((u64) stats->stat_IfHCInBadOctets_hi << 32) +
7257                  (u64) stats->stat_IfHCInBadOctets_lo;
7258
7259         sc->stat_IfHCOutOctets =
7260                 ((u64) stats->stat_IfHCOutOctets_hi << 32) +
7261                  (u64) stats->stat_IfHCOutOctets_lo;
7262
7263         sc->stat_IfHCOutBadOctets =
7264                 ((u64) stats->stat_IfHCOutBadOctets_hi << 32) +
7265                  (u64) stats->stat_IfHCOutBadOctets_lo;
7266
7267         sc->stat_IfHCInUcastPkts =
7268                 ((u64) stats->stat_IfHCInUcastPkts_hi << 32) +
7269                  (u64) stats->stat_IfHCInUcastPkts_lo;
7270
7271         sc->stat_IfHCInMulticastPkts =
7272                 ((u64) stats->stat_IfHCInMulticastPkts_hi << 32) +
7273                  (u64) stats->stat_IfHCInMulticastPkts_lo;
7274
7275         sc->stat_IfHCInBroadcastPkts =
7276                 ((u64) stats->stat_IfHCInBroadcastPkts_hi << 32) +
7277                  (u64) stats->stat_IfHCInBroadcastPkts_lo;
7278
7279         sc->stat_IfHCOutUcastPkts =
7280                 ((u64) stats->stat_IfHCOutUcastPkts_hi << 32) +
7281                  (u64) stats->stat_IfHCOutUcastPkts_lo;
7282
7283         sc->stat_IfHCOutMulticastPkts =
7284                 ((u64) stats->stat_IfHCOutMulticastPkts_hi << 32) +
7285                  (u64) stats->stat_IfHCOutMulticastPkts_lo;
7286
7287         sc->stat_IfHCOutBroadcastPkts =
7288                 ((u64) stats->stat_IfHCOutBroadcastPkts_hi << 32) +
7289                  (u64) stats->stat_IfHCOutBroadcastPkts_lo;
7290
7291         sc->stat_emac_tx_stat_dot3statsinternalmactransmiterrors =
7292                 stats->stat_emac_tx_stat_dot3statsinternalmactransmiterrors;
7293
7294         sc->stat_Dot3StatsCarrierSenseErrors =
7295                 stats->stat_Dot3StatsCarrierSenseErrors;
7296
7297         sc->stat_Dot3StatsFCSErrors =
7298                 stats->stat_Dot3StatsFCSErrors;
7299
7300         sc->stat_Dot3StatsAlignmentErrors =
7301                 stats->stat_Dot3StatsAlignmentErrors;
7302
7303         sc->stat_Dot3StatsSingleCollisionFrames =
7304                 stats->stat_Dot3StatsSingleCollisionFrames;
7305
7306         sc->stat_Dot3StatsMultipleCollisionFrames =
7307                 stats->stat_Dot3StatsMultipleCollisionFrames;
7308
7309         sc->stat_Dot3StatsDeferredTransmissions =
7310                 stats->stat_Dot3StatsDeferredTransmissions;
7311
7312         sc->stat_Dot3StatsExcessiveCollisions =
7313                 stats->stat_Dot3StatsExcessiveCollisions;
7314
7315         sc->stat_Dot3StatsLateCollisions =
7316                 stats->stat_Dot3StatsLateCollisions;
7317
7318         sc->stat_EtherStatsCollisions =
7319                 stats->stat_EtherStatsCollisions;
7320
7321         sc->stat_EtherStatsFragments =
7322                 stats->stat_EtherStatsFragments;
7323
7324         sc->stat_EtherStatsJabbers =
7325                 stats->stat_EtherStatsJabbers;
7326
7327         sc->stat_EtherStatsUndersizePkts =
7328                 stats->stat_EtherStatsUndersizePkts;
7329
7330         sc->stat_EtherStatsOversizePkts =
7331                 stats->stat_EtherStatsOversizePkts;
7332
7333         sc->stat_EtherStatsPktsRx64Octets =
7334                 stats->stat_EtherStatsPktsRx64Octets;
7335
7336         sc->stat_EtherStatsPktsRx65Octetsto127Octets =
7337                 stats->stat_EtherStatsPktsRx65Octetsto127Octets;
7338
7339         sc->stat_EtherStatsPktsRx128Octetsto255Octets =
7340                 stats->stat_EtherStatsPktsRx128Octetsto255Octets;
7341
7342         sc->stat_EtherStatsPktsRx256Octetsto511Octets =
7343                 stats->stat_EtherStatsPktsRx256Octetsto511Octets;
7344
7345         sc->stat_EtherStatsPktsRx512Octetsto1023Octets =
7346                 stats->stat_EtherStatsPktsRx512Octetsto1023Octets;
7347
7348         sc->stat_EtherStatsPktsRx1024Octetsto1522Octets =
7349                 stats->stat_EtherStatsPktsRx1024Octetsto1522Octets;
7350
7351         sc->stat_EtherStatsPktsRx1523Octetsto9022Octets =
7352                 stats->stat_EtherStatsPktsRx1523Octetsto9022Octets;
7353
7354         sc->stat_EtherStatsPktsTx64Octets =
7355                 stats->stat_EtherStatsPktsTx64Octets;
7356
7357         sc->stat_EtherStatsPktsTx65Octetsto127Octets =
7358                 stats->stat_EtherStatsPktsTx65Octetsto127Octets;
7359
7360         sc->stat_EtherStatsPktsTx128Octetsto255Octets =
7361                 stats->stat_EtherStatsPktsTx128Octetsto255Octets;
7362
7363         sc->stat_EtherStatsPktsTx256Octetsto511Octets =
7364                 stats->stat_EtherStatsPktsTx256Octetsto511Octets;
7365
7366         sc->stat_EtherStatsPktsTx512Octetsto1023Octets =
7367                 stats->stat_EtherStatsPktsTx512Octetsto1023Octets;
7368
7369         sc->stat_EtherStatsPktsTx1024Octetsto1522Octets =
7370                 stats->stat_EtherStatsPktsTx1024Octetsto1522Octets;
7371
7372         sc->stat_EtherStatsPktsTx1523Octetsto9022Octets =
7373                 stats->stat_EtherStatsPktsTx1523Octetsto9022Octets;
7374
7375         sc->stat_XonPauseFramesReceived =
7376                 stats->stat_XonPauseFramesReceived;
7377
7378         sc->stat_XoffPauseFramesReceived =
7379                 stats->stat_XoffPauseFramesReceived;
7380
7381         sc->stat_OutXonSent =
7382                 stats->stat_OutXonSent;
7383
7384         sc->stat_OutXoffSent =
7385                 stats->stat_OutXoffSent;
7386
7387         sc->stat_FlowControlDone =
7388                 stats->stat_FlowControlDone;
7389
7390         sc->stat_MacControlFramesReceived =
7391                 stats->stat_MacControlFramesReceived;
7392
7393         sc->stat_XoffStateEntered =
7394                 stats->stat_XoffStateEntered;
7395
7396         sc->stat_IfInFramesL2FilterDiscards =
7397                 stats->stat_IfInFramesL2FilterDiscards;
7398
7399         sc->stat_IfInRuleCheckerDiscards =
7400                 stats->stat_IfInRuleCheckerDiscards;
7401
7402         sc->stat_IfInFTQDiscards =
7403                 stats->stat_IfInFTQDiscards;
7404
7405         sc->stat_IfInMBUFDiscards =
7406                 stats->stat_IfInMBUFDiscards;
7407
7408         sc->stat_IfInRuleCheckerP4Hit =
7409                 stats->stat_IfInRuleCheckerP4Hit;
7410
7411         sc->stat_CatchupInRuleCheckerDiscards =
7412                 stats->stat_CatchupInRuleCheckerDiscards;
7413
7414         sc->stat_CatchupInFTQDiscards =
7415                 stats->stat_CatchupInFTQDiscards;
7416
7417         sc->stat_CatchupInMBUFDiscards =
7418                 stats->stat_CatchupInMBUFDiscards;
7419
7420         sc->stat_CatchupInRuleCheckerP4Hit =
7421                 stats->stat_CatchupInRuleCheckerP4Hit;
7422
7423         sc->com_no_buffers = REG_RD_IND(sc, 0x120084);
7424
7425         /*
7426          * Update the interface statistics from the
7427          * hardware statistics.
7428          */
7429         ifp->if_collisions =
7430                 (u_long) sc->stat_EtherStatsCollisions;
7431
7432         /* ToDo: This method loses soft errors. */
7433         ifp->if_ierrors =
7434                 (u_long) sc->stat_EtherStatsUndersizePkts +
7435                 (u_long) sc->stat_EtherStatsOversizePkts +
7436                 (u_long) sc->stat_IfInMBUFDiscards +
7437                 (u_long) sc->stat_Dot3StatsAlignmentErrors +
7438                 (u_long) sc->stat_Dot3StatsFCSErrors +
7439                 (u_long) sc->stat_IfInRuleCheckerDiscards +
7440                 (u_long) sc->stat_IfInFTQDiscards +
7441                 (u_long) sc->com_no_buffers;
7442
7443         /* ToDo: This method loses soft errors. */
7444         ifp->if_oerrors =
7445                 (u_long) sc->stat_emac_tx_stat_dot3statsinternalmactransmiterrors +
7446                 (u_long) sc->stat_Dot3StatsExcessiveCollisions +
7447                 (u_long) sc->stat_Dot3StatsLateCollisions;
7448
7449         /* ToDo: Add additional statistics. */
7450
7451         DBEXIT(BCE_EXTREME_MISC);
7452 }
7453
7454
7455 /****************************************************************************/
7456 /* Periodic function to notify the bootcode that the driver is still        */
7457 /* present.                                                                 */
7458 /*                                                                          */
7459 /* Returns:                                                                 */
7460 /*   Nothing.                                                               */
7461 /****************************************************************************/
7462 static void
7463 bce_pulse(void *xsc)
7464 {
7465         struct bce_softc *sc = xsc;
7466         u32 msg;
7467
7468         DBENTER(BCE_EXTREME_MISC);
7469
7470         BCE_LOCK_ASSERT(sc);
7471
7472         /* Tell the firmware that the driver is still running. */
7473         msg = (u32) ++sc->bce_fw_drv_pulse_wr_seq;
7474         REG_WR_IND(sc, sc->bce_shmem_base + BCE_DRV_PULSE_MB, msg);
7475
7476         /* Schedule the next pulse. */
7477         callout_reset(&sc->bce_pulse_callout, hz, bce_pulse, sc);
7478
7479         DBEXIT(BCE_EXTREME_MISC);
7480 }
7481
7482
7483 /****************************************************************************/
7484 /* Periodic function to perform maintenance tasks.                          */
7485 /*                                                                          */
7486 /* Returns:                                                                 */
7487 /*   Nothing.                                                               */
7488 /****************************************************************************/
7489 static void
7490 bce_tick(void *xsc)
7491 {
7492         struct bce_softc *sc = xsc;
7493         struct mii_data *mii;
7494         struct ifnet *ifp;
7495
7496         ifp = sc->bce_ifp;
7497
7498         DBENTER(BCE_EXTREME_MISC);
7499
7500         BCE_LOCK_ASSERT(sc);
7501
7502         /* Schedule the next tick. */
7503         callout_reset(&sc->bce_tick_callout, hz, bce_tick, sc);
7504
7505         /* Update the statistics from the hardware statistics block. */
7506         bce_stats_update(sc);
7507
7508         /* Top off the receive and page chains. */
7509 #ifdef ZERO_COPY_SOCKETS
7510         bce_fill_pg_chain(sc);
7511 #endif
7512         bce_fill_rx_chain(sc);
7513
7514         /* Check that chip hasn't hung. */
7515         bce_watchdog(sc);
7516
7517         /* If link is up already up then we're done. */
7518         if (sc->bce_link)
7519                 goto bce_tick_exit;
7520
7521         /* Link is down.  Check what the PHY's doing. */
7522         mii = device_get_softc(sc->bce_miibus);
7523         mii_tick(mii);
7524
7525         /* Check if the link has come up. */
7526         if ((mii->mii_media_status & IFM_ACTIVE) &&
7527             (IFM_SUBTYPE(mii->mii_media_active) != IFM_NONE)) {
7528                 DBPRINT(sc, BCE_VERBOSE_MISC, "%s(): Link up!\n", __FUNCTION__);
7529                 sc->bce_link++;
7530                 if ((IFM_SUBTYPE(mii->mii_media_active) == IFM_1000_T ||
7531                     IFM_SUBTYPE(mii->mii_media_active) == IFM_1000_SX) &&
7532                     bootverbose)
7533                         BCE_PRINTF("Gigabit link up!\n");
7534                 /* Now that link is up, handle any outstanding TX traffic. */
7535                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
7536                         DBPRINT(sc, BCE_VERBOSE_MISC, "%s(): Found pending TX traffic.\n",
7537                                  __FUNCTION__);
7538                         bce_start_locked(ifp);
7539                 }
7540         }
7541
7542 bce_tick_exit:
7543         DBEXIT(BCE_EXTREME_MISC);
7544         return;
7545 }
7546
7547
7548 #ifdef BCE_DEBUG
7549 /****************************************************************************/
7550 /* Allows the driver state to be dumped through the sysctl interface.       */
7551 /*                                                                          */
7552 /* Returns:                                                                 */
7553 /*   0 for success, positive value for failure.                             */
7554 /****************************************************************************/
7555 static int
7556 bce_sysctl_driver_state(SYSCTL_HANDLER_ARGS)
7557 {
7558         int error;
7559         int result;
7560         struct bce_softc *sc;
7561
7562         result = -1;
7563         error = sysctl_handle_int(oidp, &result, 0, req);
7564
7565         if (error || !req->newptr)
7566                 return (error);
7567
7568         if (result == 1) {
7569                 sc = (struct bce_softc *)arg1;
7570                 bce_dump_driver_state(sc);
7571         }
7572
7573         return error;
7574 }
7575
7576
7577 /****************************************************************************/
7578 /* Allows the hardware state to be dumped through the sysctl interface.     */
7579 /*                                                                          */
7580 /* Returns:                                                                 */
7581 /*   0 for success, positive value for failure.                             */
7582 /****************************************************************************/
7583 static int
7584 bce_sysctl_hw_state(SYSCTL_HANDLER_ARGS)
7585 {
7586         int error;
7587         int result;
7588         struct bce_softc *sc;
7589
7590         result = -1;
7591         error = sysctl_handle_int(oidp, &result, 0, req);
7592
7593         if (error || !req->newptr)
7594                 return (error);
7595
7596         if (result == 1) {
7597                 sc = (struct bce_softc *)arg1;
7598                 bce_dump_hw_state(sc);
7599         }
7600
7601         return error;
7602 }
7603
7604
7605 /****************************************************************************/
7606 /* Allows the bootcode state to be dumped through the sysctl interface.     */
7607 /*                                                                          */
7608 /* Returns:                                                                 */
7609 /*   0 for success, positive value for failure.                             */
7610 /****************************************************************************/
7611 static int
7612 bce_sysctl_bc_state(SYSCTL_HANDLER_ARGS)
7613 {
7614         int error;
7615         int result;
7616         struct bce_softc *sc;
7617
7618         result = -1;
7619         error = sysctl_handle_int(oidp, &result, 0, req);
7620
7621         if (error || !req->newptr)
7622                 return (error);
7623
7624         if (result == 1) {
7625                 sc = (struct bce_softc *)arg1;
7626                 bce_dump_bc_state(sc);
7627         }
7628
7629         return error;
7630 }
7631
7632
7633 /****************************************************************************/
7634 /* Provides a sysctl interface to allow dumping the RX chain.               */
7635 /*                                                                          */
7636 /* Returns:                                                                 */
7637 /*   0 for success, positive value for failure.                             */
7638 /****************************************************************************/
7639 static int
7640 bce_sysctl_dump_rx_chain(SYSCTL_HANDLER_ARGS)
7641 {
7642         int error;
7643         int result;
7644         struct bce_softc *sc;
7645
7646         result = -1;
7647         error = sysctl_handle_int(oidp, &result, 0, req);
7648
7649         if (error || !req->newptr)
7650                 return (error);
7651
7652         if (result == 1) {
7653                 sc = (struct bce_softc *)arg1;
7654                 bce_dump_rx_chain(sc, 0, TOTAL_RX_BD);
7655         }
7656
7657         return error;
7658 }
7659
7660
7661 /****************************************************************************/
7662 /* Provides a sysctl interface to allow dumping the TX chain.               */
7663 /*                                                                          */
7664 /* Returns:                                                                 */
7665 /*   0 for success, positive value for failure.                             */
7666 /****************************************************************************/
7667 static int
7668 bce_sysctl_dump_tx_chain(SYSCTL_HANDLER_ARGS)
7669 {
7670         int error;
7671         int result;
7672         struct bce_softc *sc;
7673
7674         result = -1;
7675         error = sysctl_handle_int(oidp, &result, 0, req);
7676
7677         if (error || !req->newptr)
7678                 return (error);
7679
7680         if (result == 1) {
7681                 sc = (struct bce_softc *)arg1;
7682                 bce_dump_tx_chain(sc, 0, USABLE_TX_BD);
7683         }
7684
7685         return error;
7686 }
7687
7688
7689 #ifdef ZERO_COPY_SOCKETS
7690 /****************************************************************************/
7691 /* Provides a sysctl interface to allow dumping the page chain.             */
7692 /*                                                                          */
7693 /* Returns:                                                                 */
7694 /*   0 for success, positive value for failure.                             */
7695 /****************************************************************************/
7696 static int
7697 bce_sysctl_dump_pg_chain(SYSCTL_HANDLER_ARGS)
7698 {
7699         int error;
7700         int result;
7701         struct bce_softc *sc;
7702
7703         result = -1;
7704         error = sysctl_handle_int(oidp, &result, 0, req);
7705
7706         if (error || !req->newptr)
7707                 return (error);
7708
7709         if (result == 1) {
7710                 sc = (struct bce_softc *)arg1;
7711                 bce_dump_pg_chain(sc, 0, TOTAL_PG_BD);
7712         }
7713
7714         return error;
7715 }
7716 #endif
7717
7718 /****************************************************************************/
7719 /* Provides a sysctl interface to allow reading arbitrary NVRAM offsets in  */
7720 /* the device.  DO NOT ENABLE ON PRODUCTION SYSTEMS!                        */
7721 /*                                                                          */
7722 /* Returns:                                                                 */
7723 /*   0 for success, positive value for failure.                             */
7724 /****************************************************************************/
7725 static int
7726 bce_sysctl_nvram_read(SYSCTL_HANDLER_ARGS)
7727 {
7728         struct bce_softc *sc = (struct bce_softc *)arg1;
7729         int error;
7730         u32 result;
7731         u32 val[1];
7732         u8 *data = (u8 *) val;
7733
7734         result = -1;
7735         error = sysctl_handle_int(oidp, &result, 0, req);
7736         if (error || (req->newptr == NULL))
7737                 return (error);
7738
7739         bce_nvram_read(sc, result, data, 4);
7740         BCE_PRINTF("offset 0x%08X = 0x%08X\n", result, bce_be32toh(val[0]));
7741
7742         return (error);
7743 }
7744
7745
7746 /****************************************************************************/
7747 /* Provides a sysctl interface to allow reading arbitrary registers in the  */
7748 /* device.  DO NOT ENABLE ON PRODUCTION SYSTEMS!                            */
7749 /*                                                                          */
7750 /* Returns:                                                                 */
7751 /*   0 for success, positive value for failure.                             */
7752 /****************************************************************************/
7753 static int
7754 bce_sysctl_reg_read(SYSCTL_HANDLER_ARGS)
7755 {
7756         struct bce_softc *sc = (struct bce_softc *)arg1;
7757         int error;
7758         u32 val, result;
7759
7760         result = -1;
7761         error = sysctl_handle_int(oidp, &result, 0, req);
7762         if (error || (req->newptr == NULL))
7763                 return (error);
7764
7765         /* Make sure the register is accessible. */
7766         if (result < 0x8000) {
7767                 val = REG_RD(sc, result);
7768                 BCE_PRINTF("reg 0x%08X = 0x%08X\n", result, val);
7769         } else if (result < 0x0280000) {
7770                 val = REG_RD_IND(sc, result);
7771                 BCE_PRINTF("reg 0x%08X = 0x%08X\n", result, val);
7772         }
7773
7774         return (error);
7775 }
7776
7777
7778 /****************************************************************************/
7779 /* Provides a sysctl interface to allow reading arbitrary PHY registers in  */
7780 /* the device.  DO NOT ENABLE ON PRODUCTION SYSTEMS!                        */
7781 /*                                                                          */
7782 /* Returns:                                                                 */
7783 /*   0 for success, positive value for failure.                             */
7784 /****************************************************************************/
7785 static int
7786 bce_sysctl_phy_read(SYSCTL_HANDLER_ARGS)
7787 {
7788         struct bce_softc *sc;
7789         device_t dev;
7790         int error, result;
7791         u16 val;
7792
7793         result = -1;
7794         error = sysctl_handle_int(oidp, &result, 0, req);
7795         if (error || (req->newptr == NULL))
7796                 return (error);
7797
7798         /* Make sure the register is accessible. */
7799         if (result < 0x20) {
7800                 sc = (struct bce_softc *)arg1;
7801                 dev = sc->bce_dev;
7802                 val = bce_miibus_read_reg(dev, sc->bce_phy_addr, result);
7803                 BCE_PRINTF("phy 0x%02X = 0x%04X\n", result, val);
7804         }
7805         return (error);
7806 }
7807
7808
7809 /****************************************************************************/
7810 /* Provides a sysctl interface to allow reading a CID.                      */
7811 /*                                                                          */
7812 /* Returns:                                                                 */
7813 /*   0 for success, positive value for failure.                             */
7814 /****************************************************************************/
7815 static int
7816 bce_sysctl_dump_ctx(SYSCTL_HANDLER_ARGS)
7817 {
7818         struct bce_softc *sc;
7819         int error;
7820         u16 result;
7821
7822         result = -1;
7823         error = sysctl_handle_int(oidp, &result, 0, req);
7824         if (error || (req->newptr == NULL))
7825                 return (error);
7826
7827         /* Make sure the register is accessible. */
7828         if (result <= TX_CID) {
7829                 sc = (struct bce_softc *)arg1;
7830                 bce_dump_ctx(sc, result);
7831         }
7832
7833         return (error);
7834 }
7835
7836
7837  /****************************************************************************/
7838 /* Provides a sysctl interface to forcing the driver to dump state and      */
7839 /* enter the debugger.  DO NOT ENABLE ON PRODUCTION SYSTEMS!                */
7840 /*                                                                          */
7841 /* Returns:                                                                 */
7842 /*   0 for success, positive value for failure.                             */
7843 /****************************************************************************/
7844 static int
7845 bce_sysctl_breakpoint(SYSCTL_HANDLER_ARGS)
7846 {
7847         int error;
7848         int result;
7849         struct bce_softc *sc;
7850
7851         result = -1;
7852         error = sysctl_handle_int(oidp, &result, 0, req);
7853
7854         if (error || !req->newptr)
7855                 return (error);
7856
7857         if (result == 1) {
7858                 sc = (struct bce_softc *)arg1;
7859                 bce_breakpoint(sc);
7860         }
7861
7862         return error;
7863 }
7864 #endif
7865
7866
7867 /****************************************************************************/
7868 /* Adds any sysctl parameters for tuning or debugging purposes.             */
7869 /*                                                                          */
7870 /* Returns:                                                                 */
7871 /*   0 for success, positive value for failure.                             */
7872 /****************************************************************************/
7873 static void
7874 bce_add_sysctls(struct bce_softc *sc)
7875 {
7876         struct sysctl_ctx_list *ctx;
7877         struct sysctl_oid_list *children;
7878
7879         DBENTER(BCE_VERBOSE_MISC);
7880
7881         ctx = device_get_sysctl_ctx(sc->bce_dev);
7882         children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->bce_dev));
7883
7884 #ifdef BCE_DEBUG
7885         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7886                 "l2fhdr_error_sim_control",
7887                 CTLFLAG_RW, &l2fhdr_error_sim_control,
7888                 0, "Debug control to force l2fhdr errors");
7889
7890         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7891                 "l2fhdr_error_sim_count",
7892                 CTLFLAG_RD, &sc->l2fhdr_error_sim_count,
7893                 0, "Number of simulated l2_fhdr errors");
7894 #endif
7895
7896         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7897                 "l2fhdr_error_count",
7898                 CTLFLAG_RD, &sc->l2fhdr_error_count,
7899                 0, "Number of l2_fhdr errors");
7900
7901 #ifdef BCE_DEBUG
7902         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7903                 "mbuf_alloc_failed_sim_control",
7904                 CTLFLAG_RW, &mbuf_alloc_failed_sim_control,
7905                 0, "Debug control to force mbuf allocation failures");
7906
7907         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7908                 "mbuf_alloc_failed_sim_count",
7909                 CTLFLAG_RD, &sc->mbuf_alloc_failed_sim_count,
7910                 0, "Number of simulated mbuf cluster allocation failures");
7911 #endif
7912
7913         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7914                 "mbuf_alloc_failed_count",
7915                 CTLFLAG_RD, &sc->mbuf_alloc_failed_count,
7916                 0, "Number of mbuf allocation failures");
7917
7918         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7919                 "fragmented_mbuf_count",
7920                 CTLFLAG_RD, &sc->fragmented_mbuf_count,
7921                 0, "Number of fragmented mbufs");
7922
7923 #ifdef BCE_DEBUG
7924         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7925                 "dma_map_addr_failed_sim_control",
7926                 CTLFLAG_RW, &dma_map_addr_failed_sim_control,
7927                 0, "Debug control to force DMA mapping failures");
7928
7929         /* ToDo: Figure out how to update this value in bce_dma_map_addr(). */
7930         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7931                 "dma_map_addr_failed_sim_count",
7932                 CTLFLAG_RD, &sc->dma_map_addr_failed_sim_count,
7933                 0, "Number of simulated DMA mapping failures");
7934
7935 #endif
7936
7937         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7938                 "dma_map_addr_rx_failed_count",
7939                 CTLFLAG_RD, &sc->dma_map_addr_rx_failed_count,
7940                 0, "Number of RX DMA mapping failures");
7941
7942         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7943                 "dma_map_addr_tx_failed_count",
7944                 CTLFLAG_RD, &sc->dma_map_addr_tx_failed_count,
7945                 0, "Number of TX DMA mapping failures");
7946
7947 #ifdef BCE_DEBUG
7948         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7949                 "unexpected_attention_sim_control",
7950                 CTLFLAG_RW, &unexpected_attention_sim_control,
7951                 0, "Debug control to simulate unexpected attentions");
7952
7953         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7954                 "unexpected_attention_sim_count",
7955                 CTLFLAG_RW, &sc->unexpected_attention_sim_count,
7956                 0, "Number of simulated unexpected attentions");
7957 #endif
7958
7959         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7960                 "unexpected_attention_count",
7961                 CTLFLAG_RW, &sc->unexpected_attention_count,
7962                 0, "Number of unexpected attentions");
7963
7964 #ifdef BCE_DEBUG
7965         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7966                 "debug_bootcode_running_failure",
7967                 CTLFLAG_RW, &bootcode_running_failure_sim_control,
7968                 0, "Debug control to force bootcode running failures");
7969
7970         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7971                 "rx_low_watermark",
7972                 CTLFLAG_RD, &sc->rx_low_watermark,
7973                 0, "Lowest level of free rx_bd's");
7974
7975         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7976                 "rx_empty_count",
7977                 CTLFLAG_RD, &sc->rx_empty_count,
7978                 0, "Number of times the RX chain was empty");
7979
7980         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7981                 "tx_hi_watermark",
7982                 CTLFLAG_RD, &sc->tx_hi_watermark,
7983                 0, "Highest level of used tx_bd's");
7984
7985         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7986                 "tx_full_count",
7987                 CTLFLAG_RD, &sc->tx_full_count,
7988                 0, "Number of times the TX chain was full");
7989
7990         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7991                 "requested_tso_frames",
7992                 CTLFLAG_RD, &sc->requested_tso_frames,
7993                 0, "Number of TSO frames received");
7994
7995         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
7996                 "rx_interrupts",
7997                 CTLFLAG_RD, &sc->rx_interrupts,
7998                 0, "Number of RX interrupts");
7999
8000         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8001                 "tx_interrupts",
8002                 CTLFLAG_RD, &sc->tx_interrupts,
8003                 0, "Number of TX interrupts");
8004
8005         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
8006                 "rx_intr_time",
8007                 CTLFLAG_RD, &sc->rx_intr_time,
8008                 "RX interrupt time");
8009
8010         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
8011                 "tx_intr_time",
8012                 CTLFLAG_RD, &sc->tx_intr_time,
8013                 "TX interrupt time");
8014 #endif
8015
8016         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
8017                 "stat_IfHcInOctets",
8018                 CTLFLAG_RD, &sc->stat_IfHCInOctets,
8019                 "Bytes received");
8020
8021         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
8022                 "stat_IfHCInBadOctets",
8023                 CTLFLAG_RD, &sc->stat_IfHCInBadOctets,
8024                 "Bad bytes received");
8025
8026         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
8027                 "stat_IfHCOutOctets",
8028                 CTLFLAG_RD, &sc->stat_IfHCOutOctets,
8029                 "Bytes sent");
8030
8031         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
8032                 "stat_IfHCOutBadOctets",
8033                 CTLFLAG_RD, &sc->stat_IfHCOutBadOctets,
8034                 "Bad bytes sent");
8035
8036         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
8037                 "stat_IfHCInUcastPkts",
8038                 CTLFLAG_RD, &sc->stat_IfHCInUcastPkts,
8039                 "Unicast packets received");
8040
8041         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
8042                 "stat_IfHCInMulticastPkts",
8043                 CTLFLAG_RD, &sc->stat_IfHCInMulticastPkts,
8044                 "Multicast packets received");
8045
8046         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
8047                 "stat_IfHCInBroadcastPkts",
8048                 CTLFLAG_RD, &sc->stat_IfHCInBroadcastPkts,
8049                 "Broadcast packets received");
8050
8051         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
8052                 "stat_IfHCOutUcastPkts",
8053                 CTLFLAG_RD, &sc->stat_IfHCOutUcastPkts,
8054                 "Unicast packets sent");
8055
8056         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
8057                 "stat_IfHCOutMulticastPkts",
8058                 CTLFLAG_RD, &sc->stat_IfHCOutMulticastPkts,
8059                 "Multicast packets sent");
8060
8061         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
8062                 "stat_IfHCOutBroadcastPkts",
8063                 CTLFLAG_RD, &sc->stat_IfHCOutBroadcastPkts,
8064                 "Broadcast packets sent");
8065
8066         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8067                 "stat_emac_tx_stat_dot3statsinternalmactransmiterrors",
8068                 CTLFLAG_RD, &sc->stat_emac_tx_stat_dot3statsinternalmactransmiterrors,
8069                 0, "Internal MAC transmit errors");
8070
8071         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8072                 "stat_Dot3StatsCarrierSenseErrors",
8073                 CTLFLAG_RD, &sc->stat_Dot3StatsCarrierSenseErrors,
8074                 0, "Carrier sense errors");
8075
8076         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8077                 "stat_Dot3StatsFCSErrors",
8078                 CTLFLAG_RD, &sc->stat_Dot3StatsFCSErrors,
8079                 0, "Frame check sequence errors");
8080
8081         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8082                 "stat_Dot3StatsAlignmentErrors",
8083                 CTLFLAG_RD, &sc->stat_Dot3StatsAlignmentErrors,
8084                 0, "Alignment errors");
8085
8086         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8087                 "stat_Dot3StatsSingleCollisionFrames",
8088                 CTLFLAG_RD, &sc->stat_Dot3StatsSingleCollisionFrames,
8089                 0, "Single Collision Frames");
8090
8091         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8092                 "stat_Dot3StatsMultipleCollisionFrames",
8093                 CTLFLAG_RD, &sc->stat_Dot3StatsMultipleCollisionFrames,
8094                 0, "Multiple Collision Frames");
8095
8096         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8097                 "stat_Dot3StatsDeferredTransmissions",
8098                 CTLFLAG_RD, &sc->stat_Dot3StatsDeferredTransmissions,
8099                 0, "Deferred Transmissions");
8100
8101         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8102                 "stat_Dot3StatsExcessiveCollisions",
8103                 CTLFLAG_RD, &sc->stat_Dot3StatsExcessiveCollisions,
8104                 0, "Excessive Collisions");
8105
8106         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8107                 "stat_Dot3StatsLateCollisions",
8108                 CTLFLAG_RD, &sc->stat_Dot3StatsLateCollisions,
8109                 0, "Late Collisions");
8110
8111         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8112                 "stat_EtherStatsCollisions",
8113                 CTLFLAG_RD, &sc->stat_EtherStatsCollisions,
8114                 0, "Collisions");
8115
8116         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8117                 "stat_EtherStatsFragments",
8118                 CTLFLAG_RD, &sc->stat_EtherStatsFragments,
8119                 0, "Fragments");
8120
8121         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8122                 "stat_EtherStatsJabbers",
8123                 CTLFLAG_RD, &sc->stat_EtherStatsJabbers,
8124                 0, "Jabbers");
8125
8126         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8127                 "stat_EtherStatsUndersizePkts",
8128                 CTLFLAG_RD, &sc->stat_EtherStatsUndersizePkts,
8129                 0, "Undersize packets");
8130
8131         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8132                 "stat_EtherStatsOversizePkts",
8133                 CTLFLAG_RD, &sc->stat_EtherStatsOversizePkts,
8134                 0, "stat_EtherStatsOversizePkts");
8135
8136         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8137                 "stat_EtherStatsPktsRx64Octets",
8138                 CTLFLAG_RD, &sc->stat_EtherStatsPktsRx64Octets,
8139                 0, "Bytes received in 64 byte packets");
8140
8141         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8142                 "stat_EtherStatsPktsRx65Octetsto127Octets",
8143                 CTLFLAG_RD, &sc->stat_EtherStatsPktsRx65Octetsto127Octets,
8144                 0, "Bytes received in 65 to 127 byte packets");
8145
8146         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8147                 "stat_EtherStatsPktsRx128Octetsto255Octets",
8148                 CTLFLAG_RD, &sc->stat_EtherStatsPktsRx128Octetsto255Octets,
8149                 0, "Bytes received in 128 to 255 byte packets");
8150
8151         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8152                 "stat_EtherStatsPktsRx256Octetsto511Octets",
8153                 CTLFLAG_RD, &sc->stat_EtherStatsPktsRx256Octetsto511Octets,
8154                 0, "Bytes received in 256 to 511 byte packets");
8155
8156         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8157                 "stat_EtherStatsPktsRx512Octetsto1023Octets",
8158                 CTLFLAG_RD, &sc->stat_EtherStatsPktsRx512Octetsto1023Octets,
8159                 0, "Bytes received in 512 to 1023 byte packets");
8160
8161         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8162                 "stat_EtherStatsPktsRx1024Octetsto1522Octets",
8163                 CTLFLAG_RD, &sc->stat_EtherStatsPktsRx1024Octetsto1522Octets,
8164                 0, "Bytes received in 1024 t0 1522 byte packets");
8165
8166         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8167                 "stat_EtherStatsPktsRx1523Octetsto9022Octets",
8168                 CTLFLAG_RD, &sc->stat_EtherStatsPktsRx1523Octetsto9022Octets,
8169                 0, "Bytes received in 1523 to 9022 byte packets");
8170
8171         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8172                 "stat_EtherStatsPktsTx64Octets",
8173                 CTLFLAG_RD, &sc->stat_EtherStatsPktsTx64Octets,
8174                 0, "Bytes sent in 64 byte packets");
8175
8176         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8177                 "stat_EtherStatsPktsTx65Octetsto127Octets",
8178                 CTLFLAG_RD, &sc->stat_EtherStatsPktsTx65Octetsto127Octets,
8179                 0, "Bytes sent in 65 to 127 byte packets");
8180
8181         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8182                 "stat_EtherStatsPktsTx128Octetsto255Octets",
8183                 CTLFLAG_RD, &sc->stat_EtherStatsPktsTx128Octetsto255Octets,
8184                 0, "Bytes sent in 128 to 255 byte packets");
8185
8186         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8187                 "stat_EtherStatsPktsTx256Octetsto511Octets",
8188                 CTLFLAG_RD, &sc->stat_EtherStatsPktsTx256Octetsto511Octets,
8189                 0, "Bytes sent in 256 to 511 byte packets");
8190
8191         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8192                 "stat_EtherStatsPktsTx512Octetsto1023Octets",
8193                 CTLFLAG_RD, &sc->stat_EtherStatsPktsTx512Octetsto1023Octets,
8194                 0, "Bytes sent in 512 to 1023 byte packets");
8195
8196         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8197                 "stat_EtherStatsPktsTx1024Octetsto1522Octets",
8198                 CTLFLAG_RD, &sc->stat_EtherStatsPktsTx1024Octetsto1522Octets,
8199                 0, "Bytes sent in 1024 to 1522 byte packets");
8200
8201         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8202                 "stat_EtherStatsPktsTx1523Octetsto9022Octets",
8203                 CTLFLAG_RD, &sc->stat_EtherStatsPktsTx1523Octetsto9022Octets,
8204                 0, "Bytes sent in 1523 to 9022 byte packets");
8205
8206         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8207                 "stat_XonPauseFramesReceived",
8208                 CTLFLAG_RD, &sc->stat_XonPauseFramesReceived,
8209                 0, "XON pause frames receved");
8210
8211         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8212                 "stat_XoffPauseFramesReceived",
8213                 CTLFLAG_RD, &sc->stat_XoffPauseFramesReceived,
8214                 0, "XOFF pause frames received");
8215
8216         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8217                 "stat_OutXonSent",
8218                 CTLFLAG_RD, &sc->stat_OutXonSent,
8219                 0, "XON pause frames sent");
8220
8221         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8222                 "stat_OutXoffSent",
8223                 CTLFLAG_RD, &sc->stat_OutXoffSent,
8224                 0, "XOFF pause frames sent");
8225
8226         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8227                 "stat_FlowControlDone",
8228                 CTLFLAG_RD, &sc->stat_FlowControlDone,
8229                 0, "Flow control done");
8230
8231         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8232                 "stat_MacControlFramesReceived",
8233                 CTLFLAG_RD, &sc->stat_MacControlFramesReceived,
8234                 0, "MAC control frames received");
8235
8236         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8237                 "stat_XoffStateEntered",
8238                 CTLFLAG_RD, &sc->stat_XoffStateEntered,
8239                 0, "XOFF state entered");
8240
8241         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8242                 "stat_IfInFramesL2FilterDiscards",
8243                 CTLFLAG_RD, &sc->stat_IfInFramesL2FilterDiscards,
8244                 0, "Received L2 packets discarded");
8245
8246         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8247                 "stat_IfInRuleCheckerDiscards",
8248                 CTLFLAG_RD, &sc->stat_IfInRuleCheckerDiscards,
8249                 0, "Received packets discarded by rule");
8250
8251         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8252                 "stat_IfInFTQDiscards",
8253                 CTLFLAG_RD, &sc->stat_IfInFTQDiscards,
8254                 0, "Received packet FTQ discards");
8255
8256         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8257                 "stat_IfInMBUFDiscards",
8258                 CTLFLAG_RD, &sc->stat_IfInMBUFDiscards,
8259                 0, "Received packets discarded due to lack of controller buffer memory");
8260
8261         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8262                 "stat_IfInRuleCheckerP4Hit",
8263                 CTLFLAG_RD, &sc->stat_IfInRuleCheckerP4Hit,
8264                 0, "Received packets rule checker hits");
8265
8266         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8267                 "stat_CatchupInRuleCheckerDiscards",
8268                 CTLFLAG_RD, &sc->stat_CatchupInRuleCheckerDiscards,
8269                 0, "Received packets discarded in Catchup path");
8270
8271         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8272                 "stat_CatchupInFTQDiscards",
8273                 CTLFLAG_RD, &sc->stat_CatchupInFTQDiscards,
8274                 0, "Received packets discarded in FTQ in Catchup path");
8275
8276         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8277                 "stat_CatchupInMBUFDiscards",
8278                 CTLFLAG_RD, &sc->stat_CatchupInMBUFDiscards,
8279                 0, "Received packets discarded in controller buffer memory in Catchup path");
8280
8281         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8282                 "stat_CatchupInRuleCheckerP4Hit",
8283                 CTLFLAG_RD, &sc->stat_CatchupInRuleCheckerP4Hit,
8284                 0, "Received packets rule checker hits in Catchup path");
8285
8286         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
8287                 "com_no_buffers",
8288                 CTLFLAG_RD, &sc->com_no_buffers,
8289                 0, "Valid packets received but no RX buffers available");
8290
8291 #ifdef BCE_DEBUG
8292         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
8293                 "driver_state", CTLTYPE_INT | CTLFLAG_RW,
8294                 (void *)sc, 0,
8295                 bce_sysctl_driver_state, "I", "Drive state information");
8296
8297         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
8298                 "hw_state", CTLTYPE_INT | CTLFLAG_RW,
8299                 (void *)sc, 0,
8300                 bce_sysctl_hw_state, "I", "Hardware state information");
8301
8302         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
8303                 "bc_state", CTLTYPE_INT | CTLFLAG_RW,
8304                 (void *)sc, 0,
8305                 bce_sysctl_bc_state, "I", "Bootcode state information");
8306
8307         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
8308                 "dump_rx_chain", CTLTYPE_INT | CTLFLAG_RW,
8309                 (void *)sc, 0,
8310                 bce_sysctl_dump_rx_chain, "I", "Dump rx_bd chain");
8311
8312         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
8313                 "dump_tx_chain", CTLTYPE_INT | CTLFLAG_RW,
8314                 (void *)sc, 0,
8315                 bce_sysctl_dump_tx_chain, "I", "Dump tx_bd chain");
8316
8317 #ifdef ZERO_COPY_SOCKETS
8318         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
8319                 "dump_pg_chain", CTLTYPE_INT | CTLFLAG_RW,
8320                 (void *)sc, 0,
8321                 bce_sysctl_dump_pg_chain, "I", "Dump page chain");
8322 #endif
8323         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
8324                 "dump_ctx", CTLTYPE_INT | CTLFLAG_RW,
8325                 (void *)sc, 0,
8326                 bce_sysctl_dump_ctx, "I", "Dump context memory");
8327
8328         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
8329                 "breakpoint", CTLTYPE_INT | CTLFLAG_RW,
8330                 (void *)sc, 0,
8331                 bce_sysctl_breakpoint, "I", "Driver breakpoint");
8332
8333         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
8334                 "reg_read", CTLTYPE_INT | CTLFLAG_RW,
8335                 (void *)sc, 0,
8336                 bce_sysctl_reg_read, "I", "Register read");
8337
8338         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
8339                 "nvram_read", CTLTYPE_INT | CTLFLAG_RW,
8340                 (void *)sc, 0,
8341                 bce_sysctl_nvram_read, "I", "NVRAM read");
8342
8343         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
8344                 "phy_read", CTLTYPE_INT | CTLFLAG_RW,
8345                 (void *)sc, 0,
8346                 bce_sysctl_phy_read, "I", "PHY register read");
8347
8348 #endif
8349
8350         DBEXIT(BCE_VERBOSE_MISC);
8351 }
8352
8353
8354 /****************************************************************************/
8355 /* BCE Debug Routines                                                       */
8356 /****************************************************************************/
8357 #ifdef BCE_DEBUG
8358
8359 /****************************************************************************/
8360 /* Freezes the controller to allow for a cohesive state dump.               */
8361 /*                                                                          */
8362 /* Returns:                                                                 */
8363 /*   Nothing.                                                               */
8364 /****************************************************************************/
8365 static void
8366 bce_freeze_controller(struct bce_softc *sc)
8367 {
8368         u32 val;
8369         val = REG_RD(sc, BCE_MISC_COMMAND);
8370         val |= BCE_MISC_COMMAND_DISABLE_ALL;
8371         REG_WR(sc, BCE_MISC_COMMAND, val);
8372 }
8373
8374
8375 /****************************************************************************/
8376 /* Unfreezes the controller after a freeze operation.  This may not always  */
8377 /* work and the controller will require a reset!                            */
8378 /*                                                                          */
8379 /* Returns:                                                                 */
8380 /*   Nothing.                                                               */
8381 /****************************************************************************/
8382 static void
8383 bce_unfreeze_controller(struct bce_softc *sc)
8384 {
8385         u32 val;
8386         val = REG_RD(sc, BCE_MISC_COMMAND);
8387         val |= BCE_MISC_COMMAND_ENABLE_ALL;
8388         REG_WR(sc, BCE_MISC_COMMAND, val);
8389 }
8390
8391
8392 /****************************************************************************/
8393 /* Prints out Ethernet frame information from an mbuf.                      */
8394 /*                                                                          */
8395 /* Partially decode an Ethernet frame to look at some important headers.    */
8396 /*                                                                          */
8397 /* Returns:                                                                 */
8398 /*   Nothing.                                                               */
8399 /****************************************************************************/
8400 static void
8401 bce_dump_enet(struct bce_softc *sc, struct mbuf *m)
8402 {
8403         struct ether_vlan_header *eh;
8404         u16 etype;
8405         int ehlen;
8406         struct ip *ip;
8407         struct tcphdr *th;
8408         struct udphdr *uh;
8409         struct arphdr *ah;
8410
8411                 BCE_PRINTF(
8412                         "-----------------------------"
8413                         " Frame Decode "
8414                         "-----------------------------\n");
8415
8416         eh = mtod(m, struct ether_vlan_header *);
8417
8418         /* Handle VLAN encapsulation if present. */
8419         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
8420                 etype = ntohs(eh->evl_proto);
8421                 ehlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
8422         } else {
8423                 etype = ntohs(eh->evl_encap_proto);
8424                 ehlen = ETHER_HDR_LEN;
8425         }
8426
8427         /* ToDo: Add VLAN output. */
8428         BCE_PRINTF("enet: dest = %6D, src = %6D, type = 0x%04X, hlen = %d\n",
8429                 eh->evl_dhost, ":", eh->evl_shost, ":", etype, ehlen);
8430
8431         switch (etype) {
8432                 case ETHERTYPE_IP:
8433                         ip = (struct ip *)(m->m_data + ehlen);
8434                         BCE_PRINTF("--ip: dest = 0x%08X , src = 0x%08X, len = %d bytes, "
8435                                 "protocol = 0x%02X, xsum = 0x%04X\n",
8436                                 ntohl(ip->ip_dst.s_addr), ntohl(ip->ip_src.s_addr),
8437                                 ntohs(ip->ip_len), ip->ip_p, ntohs(ip->ip_sum));
8438
8439                         switch (ip->ip_p) {
8440                                 case IPPROTO_TCP:
8441                                         th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2));
8442                                         BCE_PRINTF("-tcp: dest = %d, src = %d, hlen = %d bytes, "
8443                                                 "flags = 0x%b, csum = 0x%04X\n",
8444                                                 ntohs(th->th_dport), ntohs(th->th_sport), (th->th_off << 2),
8445                                                 th->th_flags, "\20\10CWR\07ECE\06URG\05ACK\04PSH\03RST\02SYN\01FIN",
8446                                                 ntohs(th->th_sum));
8447                                         break;
8448                                 case IPPROTO_UDP:
8449                             uh = (struct udphdr *)((caddr_t)ip + (ip->ip_hl << 2));
8450                                         BCE_PRINTF("-udp: dest = %d, src = %d, len = %d bytes, "
8451                                                 "csum = 0x%04X\n", ntohs(uh->uh_dport), ntohs(uh->uh_sport),
8452                                                 ntohs(uh->uh_ulen), ntohs(uh->uh_sum));
8453                                         break;
8454                                 case IPPROTO_ICMP:
8455                                         BCE_PRINTF("icmp:\n");
8456                                         break;
8457                                 default:
8458                                         BCE_PRINTF("----: Other IP protocol.\n");
8459                         }
8460                         break;
8461                 case ETHERTYPE_IPV6:
8462                         BCE_PRINTF("ipv6: No decode supported.\n");
8463                         break;
8464                 case ETHERTYPE_ARP:
8465                         BCE_PRINTF("-arp: ");
8466                         ah = (struct arphdr *) (m->m_data + ehlen);
8467                         switch (ntohs(ah->ar_op)) {
8468                                 case ARPOP_REVREQUEST:
8469                                         printf("reverse ARP request\n");
8470                                         break;
8471                                 case ARPOP_REVREPLY:
8472                                         printf("reverse ARP reply\n");
8473                                         break;
8474                                 case ARPOP_REQUEST:
8475                                         printf("ARP request\n");
8476                                         break;
8477                                 case ARPOP_REPLY:
8478                                         printf("ARP reply\n");
8479                                         break;
8480                                 default:
8481                                         printf("other ARP operation\n");
8482                         }
8483                         break;
8484                 default:
8485                         BCE_PRINTF("----: Other protocol.\n");
8486         }
8487
8488         BCE_PRINTF(
8489                 "-----------------------------"
8490                 "--------------"
8491                 "-----------------------------\n");
8492 }
8493
8494
8495 /****************************************************************************/
8496 /* Prints out information about an mbuf.                                    */
8497 /*                                                                          */
8498 /* Returns:                                                                 */
8499 /*   Nothing.                                                               */
8500 /****************************************************************************/
8501 static __attribute__ ((noinline)) void
8502 bce_dump_mbuf(struct bce_softc *sc, struct mbuf *m)
8503 {
8504         struct mbuf *mp = m;
8505
8506         if (m == NULL) {
8507                 BCE_PRINTF("mbuf: null pointer\n");
8508                 return;
8509         }
8510
8511         while (mp) {
8512                 BCE_PRINTF("mbuf: %p, m_len = %d, m_flags = 0x%b, m_data = %p\n",
8513                         mp, mp->m_len, mp->m_flags,
8514                         "\20\1M_EXT\2M_PKTHDR\3M_EOR\4M_RDONLY",
8515                         mp->m_data);
8516
8517                 if (mp->m_flags & M_PKTHDR) {
8518                         BCE_PRINTF("- m_pkthdr: len = %d, flags = 0x%b, csum_flags = %b\n",
8519                                 mp->m_pkthdr.len, mp->m_flags,
8520                                 "\20\12M_BCAST\13M_MCAST\14M_FRAG\15M_FIRSTFRAG"
8521                                 "\16M_LASTFRAG\21M_VLANTAG\22M_PROMISC\23M_NOFREE",
8522                                 mp->m_pkthdr.csum_flags,
8523                                 "\20\1CSUM_IP\2CSUM_TCP\3CSUM_UDP\4CSUM_IP_FRAGS"
8524                                 "\5CSUM_FRAGMENT\6CSUM_TSO\11CSUM_IP_CHECKED"
8525                                 "\12CSUM_IP_VALID\13CSUM_DATA_VALID\14CSUM_PSEUDO_HDR");
8526                 }
8527
8528                 if (mp->m_flags & M_EXT) {
8529                         BCE_PRINTF("- m_ext: %p, ext_size = %d, type = ",
8530                                 mp->m_ext.ext_buf, mp->m_ext.ext_size);
8531                         switch (mp->m_ext.ext_type) {
8532                                 case EXT_CLUSTER:    printf("EXT_CLUSTER\n"); break;
8533                                 case EXT_SFBUF:      printf("EXT_SFBUF\n"); break;
8534                                 case EXT_JUMBO9:     printf("EXT_JUMBO9\n"); break;
8535                                 case EXT_JUMBO16:    printf("EXT_JUMBO16\n"); break;
8536                                 case EXT_PACKET:     printf("EXT_PACKET\n"); break;
8537                                 case EXT_MBUF:       printf("EXT_MBUF\n"); break;
8538                                 case EXT_NET_DRV:    printf("EXT_NET_DRV\n"); break;
8539                                 case EXT_MOD_TYPE:   printf("EXT_MDD_TYPE\n"); break;
8540                                 case EXT_DISPOSABLE: printf("EXT_DISPOSABLE\n"); break;
8541                                 case EXT_EXTREF:     printf("EXT_EXTREF\n"); break;
8542                                 default:             printf("UNKNOWN\n");
8543                         }
8544                 }
8545
8546                 mp = mp->m_next;
8547         }
8548 }
8549
8550
8551 /****************************************************************************/
8552 /* Prints out the mbufs in the TX mbuf chain.                               */
8553 /*                                                                          */
8554 /* Returns:                                                                 */
8555 /*   Nothing.                                                               */
8556 /****************************************************************************/
8557 static __attribute__ ((noinline)) void
8558 bce_dump_tx_mbuf_chain(struct bce_softc *sc, u16 chain_prod, int count)
8559 {
8560         struct mbuf *m;
8561
8562         BCE_PRINTF(
8563                 "----------------------------"
8564                 "  tx mbuf data  "
8565                 "----------------------------\n");
8566
8567         for (int i = 0; i < count; i++) {
8568                 m = sc->tx_mbuf_ptr[chain_prod];
8569                 BCE_PRINTF("txmbuf[0x%04X]\n", chain_prod);
8570                 bce_dump_mbuf(sc, m);
8571                 chain_prod = TX_CHAIN_IDX(NEXT_TX_BD(chain_prod));
8572         }
8573
8574         BCE_PRINTF(
8575                 "----------------------------"
8576                 "----------------"
8577                 "----------------------------\n");
8578 }
8579
8580
8581 /****************************************************************************/
8582 /* Prints out the mbufs in the RX mbuf chain.                               */
8583 /*                                                                          */
8584 /* Returns:                                                                 */
8585 /*   Nothing.                                                               */
8586 /****************************************************************************/
8587 static __attribute__ ((noinline)) void
8588 bce_dump_rx_mbuf_chain(struct bce_softc *sc, u16 chain_prod, int count)
8589 {
8590         struct mbuf *m;
8591
8592         BCE_PRINTF(
8593                 "----------------------------"
8594                 "  rx mbuf data  "
8595                 "----------------------------\n");
8596
8597         for (int i = 0; i < count; i++) {
8598                 m = sc->rx_mbuf_ptr[chain_prod];
8599                 BCE_PRINTF("rxmbuf[0x%04X]\n", chain_prod);
8600                 bce_dump_mbuf(sc, m);
8601                 chain_prod = RX_CHAIN_IDX(NEXT_RX_BD(chain_prod));
8602         }
8603
8604
8605         BCE_PRINTF(
8606                 "----------------------------"
8607                 "----------------"
8608                 "----------------------------\n");
8609 }
8610
8611
8612 #ifdef ZERO_COPY_SOCKETS
8613 /****************************************************************************/
8614 /* Prints out the mbufs in the mbuf page chain.                             */
8615 /*                                                                          */
8616 /* Returns:                                                                 */
8617 /*   Nothing.                                                               */
8618 /****************************************************************************/
8619 static __attribute__ ((noinline)) void
8620 bce_dump_pg_mbuf_chain(struct bce_softc *sc, u16 chain_prod, int count)
8621 {
8622         struct mbuf *m;
8623
8624         BCE_PRINTF(
8625                 "----------------------------"
8626                 "  pg mbuf data  "
8627                 "----------------------------\n");
8628
8629         for (int i = 0; i < count; i++) {
8630                 m = sc->pg_mbuf_ptr[chain_prod];
8631                 BCE_PRINTF("pgmbuf[0x%04X]\n", chain_prod);
8632                 bce_dump_mbuf(sc, m);
8633                 chain_prod = PG_CHAIN_IDX(NEXT_PG_BD(chain_prod));
8634         }
8635
8636
8637         BCE_PRINTF(
8638                 "----------------------------"
8639                 "----------------"
8640                 "----------------------------\n");
8641 }
8642 #endif
8643
8644
8645 /****************************************************************************/
8646 /* Prints out a tx_bd structure.                                            */
8647 /*                                                                          */
8648 /* Returns:                                                                 */
8649 /*   Nothing.                                                               */
8650 /****************************************************************************/
8651 static __attribute__ ((noinline)) void
8652 bce_dump_txbd(struct bce_softc *sc, int idx, struct tx_bd *txbd)
8653 {
8654         if (idx > MAX_TX_BD)
8655                 /* Index out of range. */
8656                 BCE_PRINTF("tx_bd[0x%04X]: Invalid tx_bd index!\n", idx);
8657         else if ((idx & USABLE_TX_BD_PER_PAGE) == USABLE_TX_BD_PER_PAGE)
8658                 /* TX Chain page pointer. */
8659                 BCE_PRINTF("tx_bd[0x%04X]: haddr = 0x%08X:%08X, chain page pointer\n",
8660                         idx, txbd->tx_bd_haddr_hi, txbd->tx_bd_haddr_lo);
8661         else {
8662                         /* Normal tx_bd entry. */
8663                         BCE_PRINTF("tx_bd[0x%04X]: haddr = 0x%08X:%08X, nbytes = 0x%08X, "
8664                                 "vlan tag= 0x%04X, flags = 0x%04X (", idx,
8665                                 txbd->tx_bd_haddr_hi, txbd->tx_bd_haddr_lo,
8666                                 txbd->tx_bd_mss_nbytes, txbd->tx_bd_vlan_tag,
8667                                 txbd->tx_bd_flags);
8668
8669                         if (txbd->tx_bd_flags & TX_BD_FLAGS_CONN_FAULT)
8670                                 printf(" CONN_FAULT");
8671
8672                         if (txbd->tx_bd_flags & TX_BD_FLAGS_TCP_UDP_CKSUM)
8673                                 printf(" TCP_UDP_CKSUM");
8674
8675                         if (txbd->tx_bd_flags & TX_BD_FLAGS_IP_CKSUM)
8676                                 printf(" IP_CKSUM");
8677
8678                         if (txbd->tx_bd_flags & TX_BD_FLAGS_VLAN_TAG)
8679                                 printf("  VLAN");
8680
8681                         if (txbd->tx_bd_flags & TX_BD_FLAGS_COAL_NOW)
8682                                 printf(" COAL_NOW");
8683
8684                         if (txbd->tx_bd_flags & TX_BD_FLAGS_DONT_GEN_CRC)
8685                                 printf(" DONT_GEN_CRC");
8686
8687                         if (txbd->tx_bd_flags & TX_BD_FLAGS_START)
8688                                 printf(" START");
8689
8690                         if (txbd->tx_bd_flags & TX_BD_FLAGS_END)
8691                                 printf(" END");
8692
8693                         if (txbd->tx_bd_flags & TX_BD_FLAGS_SW_LSO)
8694                                 printf(" LSO");
8695
8696                         if (txbd->tx_bd_flags & TX_BD_FLAGS_SW_OPTION_WORD)
8697                                 printf(" OPTION_WORD");
8698
8699                         if (txbd->tx_bd_flags & TX_BD_FLAGS_SW_FLAGS)
8700                                 printf(" FLAGS");
8701
8702                         if (txbd->tx_bd_flags & TX_BD_FLAGS_SW_SNAP)
8703                                 printf(" SNAP");
8704
8705                         printf(" )\n");
8706                 }
8707
8708 }
8709
8710
8711 /****************************************************************************/
8712 /* Prints out a rx_bd structure.                                            */
8713 /*                                                                          */
8714 /* Returns:                                                                 */
8715 /*   Nothing.                                                               */
8716 /****************************************************************************/
8717 static __attribute__ ((noinline)) void
8718 bce_dump_rxbd(struct bce_softc *sc, int idx, struct rx_bd *rxbd)
8719 {
8720         if (idx > MAX_RX_BD)
8721                 /* Index out of range. */
8722                 BCE_PRINTF("rx_bd[0x%04X]: Invalid rx_bd index!\n", idx);
8723         else if ((idx & USABLE_RX_BD_PER_PAGE) == USABLE_RX_BD_PER_PAGE)
8724                 /* RX Chain page pointer. */
8725                 BCE_PRINTF("rx_bd[0x%04X]: haddr = 0x%08X:%08X, chain page pointer\n",
8726                         idx, rxbd->rx_bd_haddr_hi, rxbd->rx_bd_haddr_lo);
8727         else
8728                 /* Normal rx_bd entry. */
8729                 BCE_PRINTF("rx_bd[0x%04X]: haddr = 0x%08X:%08X, nbytes = 0x%08X, "
8730                         "flags = 0x%08X\n", idx,
8731                         rxbd->rx_bd_haddr_hi, rxbd->rx_bd_haddr_lo,
8732                         rxbd->rx_bd_len, rxbd->rx_bd_flags);
8733 }
8734
8735
8736 #ifdef ZERO_COPY_SOCKETS
8737 /****************************************************************************/
8738 /* Prints out a rx_bd structure in the page chain.                          */
8739 /*                                                                          */
8740 /* Returns:                                                                 */
8741 /*   Nothing.                                                               */
8742 /****************************************************************************/
8743 static __attribute__ ((noinline)) void
8744 bce_dump_pgbd(struct bce_softc *sc, int idx, struct rx_bd *pgbd)
8745 {
8746         if (idx > MAX_PG_BD)
8747                 /* Index out of range. */
8748                 BCE_PRINTF("pg_bd[0x%04X]: Invalid pg_bd index!\n", idx);
8749         else if ((idx & USABLE_PG_BD_PER_PAGE) == USABLE_PG_BD_PER_PAGE)
8750                 /* Page Chain page pointer. */
8751                 BCE_PRINTF("px_bd[0x%04X]: haddr = 0x%08X:%08X, chain page pointer\n",
8752                         idx, pgbd->rx_bd_haddr_hi, pgbd->rx_bd_haddr_lo);
8753         else
8754                 /* Normal rx_bd entry. */
8755                 BCE_PRINTF("pg_bd[0x%04X]: haddr = 0x%08X:%08X, nbytes = 0x%08X, "
8756                         "flags = 0x%08X\n", idx,
8757                         pgbd->rx_bd_haddr_hi, pgbd->rx_bd_haddr_lo,
8758                         pgbd->rx_bd_len, pgbd->rx_bd_flags);
8759 }
8760 #endif
8761
8762
8763 /****************************************************************************/
8764 /* Prints out a l2_fhdr structure.                                          */
8765 /*                                                                          */
8766 /* Returns:                                                                 */
8767 /*   Nothing.                                                               */
8768 /****************************************************************************/
8769 static __attribute__ ((noinline)) void
8770 bce_dump_l2fhdr(struct bce_softc *sc, int idx, struct l2_fhdr *l2fhdr)
8771 {
8772         BCE_PRINTF("l2_fhdr[0x%04X]: status = 0x%b, "
8773                 "pkt_len = %d, vlan = 0x%04x, ip_xsum/hdr_len = 0x%04X, "
8774                 "tcp_udp_xsum = 0x%04X\n", idx,
8775                 l2fhdr->l2_fhdr_status, BCE_L2FHDR_PRINTFB,
8776                 l2fhdr->l2_fhdr_pkt_len, l2fhdr->l2_fhdr_vlan_tag,
8777                 l2fhdr->l2_fhdr_ip_xsum, l2fhdr->l2_fhdr_tcp_udp_xsum);
8778 }
8779
8780
8781 /****************************************************************************/
8782 /* Prints out context memory info.  (Only useful for CID 0 to 16.)          */
8783 /*                                                                          */
8784 /* Returns:                                                                 */
8785 /*   Nothing.                                                               */
8786 /****************************************************************************/
8787 static __attribute__ ((noinline)) void
8788 bce_dump_ctx(struct bce_softc *sc, u16 cid)
8789 {
8790         if (cid <= TX_CID) {
8791                 BCE_PRINTF(
8792                         "----------------------------"
8793                         "    CTX Data    "
8794                         "----------------------------\n");
8795
8796                 BCE_PRINTF("     0x%04X - (CID) Context ID\n", cid);
8797
8798                 if (cid == RX_CID) {
8799                         BCE_PRINTF(" 0x%08X - (L2CTX_RX_HOST_BDIDX) host rx "
8800                                 "producer index\n",
8801                                 CTX_RD(sc, GET_CID_ADDR(cid), BCE_L2CTX_RX_HOST_BDIDX));
8802                         BCE_PRINTF(" 0x%08X - (L2CTX_RX_HOST_BSEQ) host byte sequence\n",
8803                                 CTX_RD(sc, GET_CID_ADDR(cid), BCE_L2CTX_RX_HOST_BSEQ));
8804                         BCE_PRINTF(" 0x%08X - (L2CTX_RX_NX_BSEQ) h/w byte sequence\n",
8805                                 CTX_RD(sc, GET_CID_ADDR(cid), BCE_L2CTX_RX_NX_BSEQ));
8806                         BCE_PRINTF(" 0x%08X - (L2CTX_RX_NX_BDHADDR_HI) h/w buffer "
8807                                 "descriptor address\n",
8808                                 CTX_RD(sc, GET_CID_ADDR(cid), BCE_L2CTX_RX_NX_BDHADDR_HI));
8809                         BCE_PRINTF(" 0x%08X - (L2CTX_RX_NX_BDHADDR_LO) h/w buffer "
8810                                 "descriptor address\n",
8811                                 CTX_RD(sc, GET_CID_ADDR(cid), BCE_L2CTX_RX_NX_BDHADDR_LO));
8812                         BCE_PRINTF(" 0x%08X - (L2CTX_RX_NX_BDIDX) h/w rx consumer index\n",
8813                                 CTX_RD(sc, GET_CID_ADDR(cid), BCE_L2CTX_RX_NX_BDIDX));
8814                         BCE_PRINTF(" 0x%08X - (L2CTX_RX_HOST_PG_BDIDX) host page "
8815                                 "producer index\n",
8816                                 CTX_RD(sc, GET_CID_ADDR(cid), BCE_L2CTX_RX_HOST_PG_BDIDX));
8817                         BCE_PRINTF(" 0x%08X - (L2CTX_RX_PG_BUF_SIZE) host rx_bd/page "
8818                                 "buffer size\n",
8819                                 CTX_RD(sc, GET_CID_ADDR(cid), BCE_L2CTX_RX_PG_BUF_SIZE));
8820                         BCE_PRINTF(" 0x%08X - (L2CTX_RX_NX_PG_BDHADDR_HI) h/w page "
8821                                 "chain address\n",
8822                                 CTX_RD(sc, GET_CID_ADDR(cid), BCE_L2CTX_RX_NX_PG_BDHADDR_HI));
8823                         BCE_PRINTF(" 0x%08X - (L2CTX_RX_NX_PG_BDHADDR_LO) h/w page "
8824                                 "chain address\n",
8825                                 CTX_RD(sc, GET_CID_ADDR(cid), BCE_L2CTX_RX_NX_PG_BDHADDR_LO));
8826                         BCE_PRINTF(" 0x%08X - (L2CTX_RX_NX_PG_BDIDX) h/w page "
8827                                 "consumer index\n",
8828                                 CTX_RD(sc, GET_CID_ADDR(cid), BCE_L2CTX_RX_NX_PG_BDIDX));
8829                 } else if (cid == TX_CID) {
8830                         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
8831                                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
8832                                 BCE_PRINTF(" 0x%08X - (L2CTX_TX_TYPE_XI) ctx type\n",
8833                                         CTX_RD(sc, GET_CID_ADDR(cid), BCE_L2CTX_TX_TYPE_XI));
8834                                 BCE_PRINTF(" 0x%08X - (L2CTX_CMD_TX_TYPE_XI) ctx cmd\n",
8835                                         CTX_RD(sc, GET_CID_ADDR(cid), BCE_L2CTX_TX_CMD_TYPE_XI));
8836                                 BCE_PRINTF(" 0x%08X - (L2CTX_TX_TBDR_BDHADDR_HI_XI) h/w buffer "
8837                                         "descriptor address\n", CTX_RD(sc,
8838                                         GET_CID_ADDR(cid), BCE_L2CTX_TX_TBDR_BHADDR_HI_XI));
8839                                 BCE_PRINTF(" 0x%08X - (L2CTX_TX_TBDR_BHADDR_LO_XI) h/w buffer "
8840                                         "descriptor address\n", CTX_RD(sc,
8841                                         GET_CID_ADDR(cid), BCE_L2CTX_TX_TBDR_BHADDR_LO_XI));
8842                                 BCE_PRINTF(" 0x%08X - (L2CTX_TX_HOST_BIDX_XI) host producer "
8843                                         "index\n", CTX_RD(sc, GET_CID_ADDR(cid),
8844                                         BCE_L2CTX_TX_HOST_BIDX_XI));
8845                                 BCE_PRINTF(" 0x%08X - (L2CTX_TX_HOST_BSEQ_XI) host byte "
8846                                         "sequence\n", CTX_RD(sc, GET_CID_ADDR(cid),
8847                                         BCE_L2CTX_TX_HOST_BSEQ_XI));
8848                         } else {
8849                                 BCE_PRINTF(" 0x%08X - (L2CTX_TX_TYPE) ctx type\n",
8850                                         CTX_RD(sc, GET_CID_ADDR(cid), BCE_L2CTX_TX_TYPE));
8851                                 BCE_PRINTF(" 0x%08X - (L2CTX_TX_CMD_TYPE) ctx cmd\n",
8852                                         CTX_RD(sc, GET_CID_ADDR(cid), BCE_L2CTX_TX_CMD_TYPE));
8853                                 BCE_PRINTF(" 0x%08X - (L2CTX_TX_TBDR_BDHADDR_HI) h/w buffer "
8854                                         "descriptor address\n", CTX_RD(sc, GET_CID_ADDR(cid),
8855                                         BCE_L2CTX_TX_TBDR_BHADDR_HI));
8856                                 BCE_PRINTF(" 0x%08X - (L2CTX_TX_TBDR_BHADDR_LO) h/w buffer "
8857                                         "descriptor address\n", CTX_RD(sc, GET_CID_ADDR(cid),
8858                                         BCE_L2CTX_TX_TBDR_BHADDR_LO));
8859                                 BCE_PRINTF(" 0x%08X - (L2CTX_TX_HOST_BIDX) host producer "
8860                                         "index\n", CTX_RD(sc, GET_CID_ADDR(cid),
8861                                         BCE_L2CTX_TX_HOST_BIDX));
8862                                 BCE_PRINTF(" 0x%08X - (L2CTX_TX_HOST_BSEQ) host byte "
8863                                         "sequence\n", CTX_RD(sc, GET_CID_ADDR(cid),
8864                                         BCE_L2CTX_TX_HOST_BSEQ));
8865                         }
8866                 } else
8867                         BCE_PRINTF(" Unknown CID\n");
8868
8869                 BCE_PRINTF(
8870                         "----------------------------"
8871                         "    Raw CTX     "
8872                         "----------------------------\n");
8873
8874                 for (int i = 0x0; i < 0x300; i += 0x10) {
8875                         BCE_PRINTF("0x%04X: 0x%08X 0x%08X 0x%08X 0x%08X\n", i,
8876                                 CTX_RD(sc, GET_CID_ADDR(cid), i),
8877                                 CTX_RD(sc, GET_CID_ADDR(cid), i + 0x4),
8878                                 CTX_RD(sc, GET_CID_ADDR(cid), i + 0x8),
8879                                 CTX_RD(sc, GET_CID_ADDR(cid), i + 0xc));
8880                 }
8881
8882
8883                 BCE_PRINTF(
8884                         "----------------------------"
8885                         "----------------"
8886                         "----------------------------\n");
8887         }
8888 }
8889
8890
8891 /****************************************************************************/
8892 /* Prints out the FTQ data.                                                 */
8893 /*                                                                          */
8894 /* Returns:                                                                */
8895 /*   Nothing.                                                               */
8896 /****************************************************************************/
8897 static __attribute__ ((noinline)) void
8898 bce_dump_ftqs(struct bce_softc *sc)
8899 {
8900         u32 cmd, ctl, cur_depth, max_depth, valid_cnt, val;
8901
8902         BCE_PRINTF(
8903                 "----------------------------"
8904                 "    FTQ Data    "
8905                 "----------------------------\n");
8906
8907         BCE_PRINTF("   FTQ    Command    Control   Depth_Now  Max_Depth  Valid_Cnt \n");
8908         BCE_PRINTF(" ------- ---------- ---------- ---------- ---------- ----------\n");
8909
8910         /* Setup the generic statistic counters for the FTQ valid count. */
8911         val = (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_RV2PPQ_VALID_CNT << 24) |
8912                 (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_RXPCQ_VALID_CNT  << 16) |
8913                 (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_RXPQ_VALID_CNT   <<  8) |
8914                 (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_RLUPQ_VALID_CNT);
8915         REG_WR(sc, BCE_HC_STAT_GEN_SEL_0, val);
8916
8917         val = (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_TSCHQ_VALID_CNT  << 24) |
8918                 (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_RDMAQ_VALID_CNT  << 16) |
8919                 (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_RV2PTQ_VALID_CNT <<  8) |
8920                 (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_RV2PMQ_VALID_CNT);
8921         REG_WR(sc, BCE_HC_STAT_GEN_SEL_1, val);
8922
8923         val = (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_TPATQ_VALID_CNT  << 24) |
8924                 (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_TDMAQ_VALID_CNT  << 16) |
8925                 (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_TXPQ_VALID_CNT   <<  8) |
8926                 (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_TBDRQ_VALID_CNT);
8927         REG_WR(sc, BCE_HC_STAT_GEN_SEL_2, val);
8928
8929         val = (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_COMQ_VALID_CNT   << 24) |
8930                 (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_COMTQ_VALID_CNT  << 16) |
8931                 (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_COMXQ_VALID_CNT  <<  8) |
8932                 (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_TASQ_VALID_CNT);
8933         REG_WR(sc, BCE_HC_STAT_GEN_SEL_3, val);
8934
8935         /* Input queue to the Receive Lookup state machine */
8936         cmd = REG_RD(sc, BCE_RLUP_FTQ_CMD);
8937         ctl = REG_RD(sc, BCE_RLUP_FTQ_CTL);
8938         cur_depth = (ctl & BCE_RLUP_FTQ_CTL_CUR_DEPTH) >> 22;
8939         max_depth = (ctl & BCE_RLUP_FTQ_CTL_MAX_DEPTH) >> 12;
8940         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT0);
8941         BCE_PRINTF(" RLUP    0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
8942                 cmd, ctl, cur_depth, max_depth, valid_cnt);
8943
8944         /* Input queue to the Receive Processor */
8945         cmd = REG_RD_IND(sc, BCE_RXP_FTQ_CMD);
8946         ctl = REG_RD_IND(sc, BCE_RXP_FTQ_CTL);
8947         cur_depth = (ctl & BCE_RXP_FTQ_CTL_CUR_DEPTH) >> 22;
8948         max_depth = (ctl & BCE_RXP_FTQ_CTL_MAX_DEPTH) >> 12;
8949         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT1);
8950         BCE_PRINTF(" RXP     0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
8951                 cmd, ctl, cur_depth, max_depth, valid_cnt);
8952
8953         /* Input queue to the Recevie Processor */
8954         cmd = REG_RD_IND(sc, BCE_RXP_CFTQ_CMD);
8955         ctl = REG_RD_IND(sc, BCE_RXP_CFTQ_CTL);
8956         cur_depth = (ctl & BCE_RXP_CFTQ_CTL_CUR_DEPTH) >> 22;
8957         max_depth = (ctl & BCE_RXP_CFTQ_CTL_MAX_DEPTH) >> 12;
8958         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT2);
8959         BCE_PRINTF(" RXPC    0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
8960                 cmd, ctl, cur_depth, max_depth, valid_cnt);
8961
8962         /* Input queue to the Receive Virtual to Physical state machine */
8963         cmd = REG_RD(sc, BCE_RV2P_PFTQ_CMD);
8964         ctl = REG_RD(sc, BCE_RV2P_PFTQ_CTL);
8965         cur_depth = (ctl & BCE_RV2P_PFTQ_CTL_CUR_DEPTH) >> 22;
8966         max_depth = (ctl & BCE_RV2P_PFTQ_CTL_MAX_DEPTH) >> 12;
8967         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT3);
8968         BCE_PRINTF(" RV2PP   0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
8969                 cmd, ctl, cur_depth, max_depth, valid_cnt);
8970
8971         /* Input queue to the Recevie Virtual to Physical state machine */
8972         cmd = REG_RD(sc, BCE_RV2P_MFTQ_CMD);
8973         ctl = REG_RD(sc, BCE_RV2P_MFTQ_CTL);
8974         cur_depth = (ctl & BCE_RV2P_MFTQ_CTL_CUR_DEPTH) >> 22;
8975         max_depth = (ctl & BCE_RV2P_MFTQ_CTL_MAX_DEPTH) >> 12;
8976         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT4);
8977         BCE_PRINTF(" RV2PM   0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
8978                 cmd, ctl, cur_depth, max_depth, valid_cnt);
8979
8980         /* Input queue to the Receive Virtual to Physical state machine */
8981         cmd = REG_RD(sc, BCE_RV2P_TFTQ_CMD);
8982         ctl = REG_RD(sc, BCE_RV2P_TFTQ_CTL);
8983         cur_depth = (ctl & BCE_RV2P_TFTQ_CTL_CUR_DEPTH) >> 22;
8984         max_depth = (ctl & BCE_RV2P_TFTQ_CTL_MAX_DEPTH) >> 12;
8985         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT5);
8986         BCE_PRINTF(" RV2PT   0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
8987                 cmd, ctl, cur_depth, max_depth, valid_cnt);
8988
8989         /* Input queue to the Receive DMA state machine */
8990         cmd = REG_RD(sc, BCE_RDMA_FTQ_CMD);
8991         ctl = REG_RD(sc, BCE_RDMA_FTQ_CTL);
8992         cur_depth = (ctl & BCE_RDMA_FTQ_CTL_CUR_DEPTH) >> 22;
8993         max_depth = (ctl & BCE_RDMA_FTQ_CTL_MAX_DEPTH) >> 12;
8994         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT6);
8995         BCE_PRINTF(" RDMA    0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
8996                 cmd, ctl, cur_depth, max_depth, valid_cnt);
8997
8998         /* Input queue to the Transmit Scheduler state machine */
8999         cmd = REG_RD(sc, BCE_TSCH_FTQ_CMD);
9000         ctl = REG_RD(sc, BCE_TSCH_FTQ_CTL);
9001         cur_depth = (ctl & BCE_TSCH_FTQ_CTL_CUR_DEPTH) >> 22;
9002         max_depth = (ctl & BCE_TSCH_FTQ_CTL_MAX_DEPTH) >> 12;
9003         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT7);
9004         BCE_PRINTF(" TSCH    0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
9005                 cmd, ctl, cur_depth, max_depth, valid_cnt);
9006
9007         /* Input queue to the Transmit Buffer Descriptor state machine */
9008         cmd = REG_RD(sc, BCE_TBDR_FTQ_CMD);
9009         ctl = REG_RD(sc, BCE_TBDR_FTQ_CTL);
9010         cur_depth = (ctl & BCE_TBDR_FTQ_CTL_CUR_DEPTH) >> 22;
9011         max_depth = (ctl & BCE_TBDR_FTQ_CTL_MAX_DEPTH) >> 12;
9012         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT8);
9013         BCE_PRINTF(" TBDR    0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
9014                 cmd, ctl, cur_depth, max_depth, valid_cnt);
9015
9016         /* Input queue to the Transmit Processor */
9017         cmd = REG_RD_IND(sc, BCE_TXP_FTQ_CMD);
9018         ctl = REG_RD_IND(sc, BCE_TXP_FTQ_CTL);
9019         cur_depth = (ctl & BCE_TXP_FTQ_CTL_CUR_DEPTH) >> 22;
9020         max_depth = (ctl & BCE_TXP_FTQ_CTL_MAX_DEPTH) >> 12;
9021         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT9);
9022         BCE_PRINTF(" TXP     0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
9023                 cmd, ctl, cur_depth, max_depth, valid_cnt);
9024
9025         /* Input queue to the Transmit DMA state machine */
9026         cmd = REG_RD(sc, BCE_TDMA_FTQ_CMD);
9027         ctl = REG_RD(sc, BCE_TDMA_FTQ_CTL);
9028         cur_depth = (ctl & BCE_TDMA_FTQ_CTL_CUR_DEPTH) >> 22;
9029         max_depth = (ctl & BCE_TDMA_FTQ_CTL_MAX_DEPTH) >> 12;
9030         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT10);
9031         BCE_PRINTF(" TDMA    0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
9032                 cmd, ctl, cur_depth, max_depth, valid_cnt);
9033
9034         /* Input queue to the Transmit Patch-Up Processor */
9035         cmd = REG_RD_IND(sc, BCE_TPAT_FTQ_CMD);
9036         ctl = REG_RD_IND(sc, BCE_TPAT_FTQ_CTL);
9037         cur_depth = (ctl & BCE_TPAT_FTQ_CTL_CUR_DEPTH) >> 22;
9038         max_depth = (ctl & BCE_TPAT_FTQ_CTL_MAX_DEPTH) >> 12;
9039         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT11);
9040         BCE_PRINTF(" TPAT    0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
9041                 cmd, ctl, cur_depth, max_depth, valid_cnt);
9042
9043         /* Input queue to the Transmit Assembler state machine */
9044         cmd = REG_RD_IND(sc, BCE_TAS_FTQ_CMD);
9045         ctl = REG_RD_IND(sc, BCE_TAS_FTQ_CTL);
9046         cur_depth = (ctl & BCE_TAS_FTQ_CTL_CUR_DEPTH) >> 22;
9047         max_depth = (ctl & BCE_TAS_FTQ_CTL_MAX_DEPTH) >> 12;
9048         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT12);
9049         BCE_PRINTF(" TAS     0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
9050                 cmd, ctl, cur_depth, max_depth, valid_cnt);
9051
9052         /* Input queue to the Completion Processor */
9053         cmd = REG_RD_IND(sc, BCE_COM_COMXQ_FTQ_CMD);
9054         ctl = REG_RD_IND(sc, BCE_COM_COMXQ_FTQ_CTL);
9055         cur_depth = (ctl & BCE_COM_COMXQ_FTQ_CTL_CUR_DEPTH) >> 22;
9056         max_depth = (ctl & BCE_COM_COMXQ_FTQ_CTL_MAX_DEPTH) >> 12;
9057         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT13);
9058         BCE_PRINTF(" COMX    0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
9059                 cmd, ctl, cur_depth, max_depth, valid_cnt);
9060
9061         /* Input queue to the Completion Processor */
9062         cmd = REG_RD_IND(sc, BCE_COM_COMTQ_FTQ_CMD);
9063         ctl = REG_RD_IND(sc, BCE_COM_COMTQ_FTQ_CTL);
9064         cur_depth = (ctl & BCE_COM_COMTQ_FTQ_CTL_CUR_DEPTH) >> 22;
9065         max_depth = (ctl & BCE_COM_COMTQ_FTQ_CTL_MAX_DEPTH) >> 12;
9066         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT14);
9067         BCE_PRINTF(" COMT    0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
9068                 cmd, ctl, cur_depth, max_depth, valid_cnt);
9069
9070         /* Input queue to the Completion Processor */
9071         cmd = REG_RD_IND(sc, BCE_COM_COMQ_FTQ_CMD);
9072         ctl = REG_RD_IND(sc, BCE_COM_COMQ_FTQ_CTL);
9073         cur_depth = (ctl & BCE_COM_COMQ_FTQ_CTL_CUR_DEPTH) >> 22;
9074         max_depth = (ctl & BCE_COM_COMQ_FTQ_CTL_MAX_DEPTH) >> 12;
9075         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT15);
9076         BCE_PRINTF(" COMX    0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
9077                 cmd, ctl, cur_depth, max_depth, valid_cnt);
9078
9079         /* Setup the generic statistic counters for the FTQ valid count. */
9080         val = (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_CSQ_VALID_CNT  << 16) |
9081                 (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_CPQ_VALID_CNT  <<  8) |
9082                 (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_MGMQ_VALID_CNT);
9083
9084         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709)     ||
9085                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716))
9086                 val = val | (BCE_HC_STAT_GEN_SEL_0_GEN_SEL_0_RV2PCSQ_VALID_CNT_XI << 24);
9087                 REG_WR(sc, BCE_HC_STAT_GEN_SEL_0, val);
9088
9089         /* Input queue to the Management Control Processor */
9090         cmd = REG_RD_IND(sc, BCE_MCP_MCPQ_FTQ_CMD);
9091         ctl = REG_RD_IND(sc, BCE_MCP_MCPQ_FTQ_CTL);
9092         cur_depth = (ctl & BCE_MCP_MCPQ_FTQ_CTL_CUR_DEPTH) >> 22;
9093         max_depth = (ctl & BCE_MCP_MCPQ_FTQ_CTL_MAX_DEPTH) >> 12;
9094         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT0);
9095         BCE_PRINTF(" MCP     0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
9096                 cmd, ctl, cur_depth, max_depth, valid_cnt);
9097
9098         /* Input queue to the Command Processor */
9099         cmd = REG_RD_IND(sc, BCE_CP_CPQ_FTQ_CMD);
9100         ctl = REG_RD_IND(sc, BCE_CP_CPQ_FTQ_CTL);
9101         cur_depth = (ctl & BCE_CP_CPQ_FTQ_CTL_CUR_DEPTH) >> 22;
9102         max_depth = (ctl & BCE_CP_CPQ_FTQ_CTL_MAX_DEPTH) >> 12;
9103         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT1);
9104         BCE_PRINTF(" CP      0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
9105                 cmd, ctl, cur_depth, max_depth, valid_cnt);
9106
9107         /* Input queue to the Completion Scheduler state machine */
9108         cmd = REG_RD(sc, BCE_CSCH_CH_FTQ_CMD);
9109         ctl = REG_RD(sc, BCE_CSCH_CH_FTQ_CTL);
9110         cur_depth = (ctl & BCE_CSCH_CH_FTQ_CTL_CUR_DEPTH) >> 22;
9111         max_depth = (ctl & BCE_CSCH_CH_FTQ_CTL_MAX_DEPTH) >> 12;
9112         valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT2);
9113         BCE_PRINTF(" CS      0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
9114                 cmd, ctl, cur_depth, max_depth, valid_cnt);
9115
9116         if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) ||
9117                 (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716)) {
9118                 /* Input queue to the Receive Virtual to Physical Command Scheduler */
9119                 cmd = REG_RD(sc, BCE_RV2PCSR_FTQ_CMD);
9120                 ctl = REG_RD(sc, BCE_RV2PCSR_FTQ_CTL);
9121                 cur_depth = (ctl & 0xFFC00000) >> 22;
9122                 max_depth = (ctl & 0x003FF000) >> 12;
9123                 valid_cnt = REG_RD(sc, BCE_HC_STAT_GEN_STAT3);
9124                 BCE_PRINTF(" RV2PCSR 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
9125                         cmd, ctl, cur_depth, max_depth, valid_cnt);
9126         }
9127
9128         BCE_PRINTF(
9129                 "----------------------------"
9130                 "----------------"
9131                 "----------------------------\n");
9132 }
9133
9134
9135 /****************************************************************************/
9136 /* Prints out the TX chain.                                                 */
9137 /*                                                                          */
9138 /* Returns:                                                                 */
9139 /*   Nothing.                                                               */
9140 /****************************************************************************/
9141 static __attribute__ ((noinline)) void
9142 bce_dump_tx_chain(struct bce_softc *sc, u16 tx_prod, int count)
9143 {
9144         struct tx_bd *txbd;
9145
9146         /* First some info about the tx_bd chain structure. */
9147         BCE_PRINTF(
9148                 "----------------------------"
9149                 "  tx_bd  chain  "
9150                 "----------------------------\n");
9151
9152         BCE_PRINTF("page size      = 0x%08X, tx chain pages        = 0x%08X\n",
9153                 (u32) BCM_PAGE_SIZE, (u32) TX_PAGES);
9154
9155         BCE_PRINTF("tx_bd per page = 0x%08X, usable tx_bd per page = 0x%08X\n",
9156                 (u32) TOTAL_TX_BD_PER_PAGE, (u32) USABLE_TX_BD_PER_PAGE);
9157
9158         BCE_PRINTF("total tx_bd    = 0x%08X\n", (u32) TOTAL_TX_BD);
9159
9160         BCE_PRINTF(
9161                 "----------------------------"
9162                 "   tx_bd data   "
9163                 "----------------------------\n");
9164
9165         /* Now print out the tx_bd's themselves. */
9166         for (int i = 0; i < count; i++) {
9167                 txbd = &sc->tx_bd_chain[TX_PAGE(tx_prod)][TX_IDX(tx_prod)];
9168                 bce_dump_txbd(sc, tx_prod, txbd);
9169                 tx_prod = NEXT_TX_BD(tx_prod);
9170         }
9171
9172         BCE_PRINTF(
9173                 "----------------------------"
9174                 "----------------"
9175                 "----------------------------\n");
9176 }
9177
9178
9179 /****************************************************************************/
9180 /* Prints out the RX chain.                                                 */
9181 /*                                                                          */
9182 /* Returns:                                                                 */
9183 /*   Nothing.                                                               */
9184 /****************************************************************************/
9185 static __attribute__ ((noinline)) void
9186 bce_dump_rx_chain(struct bce_softc *sc, u16 rx_prod, int count)
9187 {
9188         struct rx_bd *rxbd;
9189
9190         /* First some info about the rx_bd chain structure. */
9191         BCE_PRINTF(
9192                 "----------------------------"
9193                 "  rx_bd  chain  "
9194                 "----------------------------\n");
9195
9196         BCE_PRINTF("page size      = 0x%08X, rx chain pages        = 0x%08X\n",
9197                 (u32) BCM_PAGE_SIZE, (u32) RX_PAGES);
9198
9199         BCE_PRINTF("rx_bd per page = 0x%08X, usable rx_bd per page = 0x%08X\n",
9200                 (u32) TOTAL_RX_BD_PER_PAGE, (u32) USABLE_RX_BD_PER_PAGE);
9201
9202         BCE_PRINTF("total rx_bd    = 0x%08X\n", (u32) TOTAL_RX_BD);
9203
9204         BCE_PRINTF(
9205                 "----------------------------"
9206                 "   rx_bd data   "
9207                 "----------------------------\n");
9208
9209         /* Now print out the rx_bd's themselves. */
9210         for (int i = 0; i < count; i++) {
9211                 rxbd = &sc->rx_bd_chain[RX_PAGE(rx_prod)][RX_IDX(rx_prod)];
9212                 bce_dump_rxbd(sc, rx_prod, rxbd);
9213                 rx_prod = RX_CHAIN_IDX(rx_prod + 1);
9214         }
9215
9216         BCE_PRINTF(
9217                 "----------------------------"
9218                 "----------------"
9219                 "----------------------------\n");
9220 }
9221
9222
9223 #ifdef ZERO_COPY_SOCKETS
9224 /****************************************************************************/
9225 /* Prints out the page chain.                                               */
9226 /*                                                                          */
9227 /* Returns:                                                                 */
9228 /*   Nothing.                                                               */
9229 /****************************************************************************/
9230 static __attribute__ ((noinline)) void
9231 bce_dump_pg_chain(struct bce_softc *sc, u16 pg_prod, int count)
9232 {
9233         struct rx_bd *pgbd;
9234
9235         /* First some info about the page chain structure. */
9236         BCE_PRINTF(
9237                 "----------------------------"
9238                 "   page chain   "
9239                 "----------------------------\n");
9240
9241         BCE_PRINTF("page size      = 0x%08X, pg chain pages        = 0x%08X\n",
9242                 (u32) BCM_PAGE_SIZE, (u32) PG_PAGES);
9243
9244         BCE_PRINTF("rx_bd per page = 0x%08X, usable rx_bd per page = 0x%08X\n",
9245                 (u32) TOTAL_PG_BD_PER_PAGE, (u32) USABLE_PG_BD_PER_PAGE);
9246
9247         BCE_PRINTF("total rx_bd    = 0x%08X, max_pg_bd             = 0x%08X\n",
9248                 (u32) TOTAL_PG_BD, (u32) MAX_PG_BD);
9249
9250         BCE_PRINTF(
9251                 "----------------------------"
9252                 "   page data    "
9253                 "----------------------------\n");
9254
9255         /* Now print out the rx_bd's themselves. */
9256         for (int i = 0; i < count; i++) {
9257                 pgbd = &sc->pg_bd_chain[PG_PAGE(pg_prod)][PG_IDX(pg_prod)];
9258                 bce_dump_pgbd(sc, pg_prod, pgbd);
9259                 pg_prod = PG_CHAIN_IDX(pg_prod + 1);
9260         }
9261
9262         BCE_PRINTF(
9263                 "----------------------------"
9264                 "----------------"
9265                 "----------------------------\n");
9266 }
9267 #endif
9268
9269
9270 /****************************************************************************/
9271 /* Prints out the status block from host memory.                            */
9272 /*                                                                          */
9273 /* Returns:                                                                 */
9274 /*   Nothing.                                                               */
9275 /****************************************************************************/
9276 static __attribute__ ((noinline)) void
9277 bce_dump_status_block(struct bce_softc *sc)
9278 {
9279         struct status_block *sblk;
9280
9281         sblk = sc->status_block;
9282
9283         BCE_PRINTF(
9284                 "----------------------------"
9285                 "  Status Block  "
9286                 "----------------------------\n");
9287
9288         BCE_PRINTF("    0x%08X - attn_bits\n",
9289                 sblk->status_attn_bits);
9290
9291         BCE_PRINTF("    0x%08X - attn_bits_ack\n",
9292                 sblk->status_attn_bits_ack);
9293
9294         BCE_PRINTF("0x%04X(0x%04X) - rx_cons0\n",
9295                 sblk->status_rx_quick_consumer_index0,
9296                 (u16) RX_CHAIN_IDX(sblk->status_rx_quick_consumer_index0));
9297
9298         BCE_PRINTF("0x%04X(0x%04X) - tx_cons0\n",
9299                 sblk->status_tx_quick_consumer_index0,
9300                 (u16) TX_CHAIN_IDX(sblk->status_tx_quick_consumer_index0));
9301
9302         BCE_PRINTF("        0x%04X - status_idx\n", sblk->status_idx);
9303
9304         /* Theses indices are not used for normal L2 drivers. */
9305         if (sblk->status_rx_quick_consumer_index1)
9306                 BCE_PRINTF("0x%04X(0x%04X) - rx_cons1\n",
9307                         sblk->status_rx_quick_consumer_index1,
9308                         (u16) RX_CHAIN_IDX(sblk->status_rx_quick_consumer_index1));
9309
9310         if (sblk->status_tx_quick_consumer_index1)
9311                 BCE_PRINTF("0x%04X(0x%04X) - tx_cons1\n",
9312                         sblk->status_tx_quick_consumer_index1,
9313                         (u16) TX_CHAIN_IDX(sblk->status_tx_quick_consumer_index1));
9314
9315         if (sblk->status_rx_quick_consumer_index2)
9316                 BCE_PRINTF("0x%04X(0x%04X)- rx_cons2\n",
9317                         sblk->status_rx_quick_consumer_index2,
9318                         (u16) RX_CHAIN_IDX(sblk->status_rx_quick_consumer_index2));
9319
9320         if (sblk->status_tx_quick_consumer_index2)
9321                 BCE_PRINTF("0x%04X(0x%04X) - tx_cons2\n",
9322                         sblk->status_tx_quick_consumer_index2,
9323                         (u16) TX_CHAIN_IDX(sblk->status_tx_quick_consumer_index2));
9324
9325         if (sblk->status_rx_quick_consumer_index3)
9326                 BCE_PRINTF("0x%04X(0x%04X) - rx_cons3\n",
9327                         sblk->status_rx_quick_consumer_index3,
9328                         (u16) RX_CHAIN_IDX(sblk->status_rx_quick_consumer_index3));
9329
9330         if (sblk->status_tx_quick_consumer_index3)
9331                 BCE_PRINTF("0x%04X(0x%04X) - tx_cons3\n",
9332                         sblk->status_tx_quick_consumer_index3,
9333                         (u16) TX_CHAIN_IDX(sblk->status_tx_quick_consumer_index3));
9334
9335         if (sblk->status_rx_quick_consumer_index4 ||
9336                 sblk->status_rx_quick_consumer_index5)
9337                 BCE_PRINTF("rx_cons4  = 0x%08X, rx_cons5      = 0x%08X\n",
9338                         sblk->status_rx_quick_consumer_index4,
9339                         sblk->status_rx_quick_consumer_index5);
9340
9341         if (sblk->status_rx_quick_consumer_index6 ||
9342                 sblk->status_rx_quick_consumer_index7)
9343                 BCE_PRINTF("rx_cons6  = 0x%08X, rx_cons7      = 0x%08X\n",
9344                         sblk->status_rx_quick_consumer_index6,
9345                         sblk->status_rx_quick_consumer_index7);
9346
9347         if (sblk->status_rx_quick_consumer_index8 ||
9348                 sblk->status_rx_quick_consumer_index9)
9349                 BCE_PRINTF("rx_cons8  = 0x%08X, rx_cons9      = 0x%08X\n",
9350                         sblk->status_rx_quick_consumer_index8,
9351                         sblk->status_rx_quick_consumer_index9);
9352
9353         if (sblk->status_rx_quick_consumer_index10 ||
9354                 sblk->status_rx_quick_consumer_index11)
9355                 BCE_PRINTF("rx_cons10 = 0x%08X, rx_cons11     = 0x%08X\n",
9356                         sblk->status_rx_quick_consumer_index10,
9357                         sblk->status_rx_quick_consumer_index11);
9358
9359         if (sblk->status_rx_quick_consumer_index12 ||
9360                 sblk->status_rx_quick_consumer_index13)
9361                 BCE_PRINTF("rx_cons12 = 0x%08X, rx_cons13     = 0x%08X\n",
9362                         sblk->status_rx_quick_consumer_index12,
9363                         sblk->status_rx_quick_consumer_index13);
9364
9365         if (sblk->status_rx_quick_consumer_index14 ||
9366                 sblk->status_rx_quick_consumer_index15)
9367                 BCE_PRINTF("rx_cons14 = 0x%08X, rx_cons15     = 0x%08X\n",
9368                         sblk->status_rx_quick_consumer_index14,
9369                         sblk->status_rx_quick_consumer_index15);
9370
9371         if (sblk->status_completion_producer_index ||
9372                 sblk->status_cmd_consumer_index)
9373                 BCE_PRINTF("com_prod  = 0x%08X, cmd_cons      = 0x%08X\n",
9374                         sblk->status_completion_producer_index,
9375                         sblk->status_cmd_consumer_index);
9376
9377         BCE_PRINTF(
9378                 "----------------------------"
9379                 "----------------"
9380                 "----------------------------\n");
9381 }
9382
9383
9384 /****************************************************************************/
9385 /* Prints out the statistics block from host memory.                        */
9386 /*                                                                          */
9387 /* Returns:                                                                 */
9388 /*   Nothing.                                                               */
9389 /****************************************************************************/
9390 static __attribute__ ((noinline)) void
9391 bce_dump_stats_block(struct bce_softc *sc)
9392 {
9393         struct statistics_block *sblk;
9394
9395         sblk = sc->stats_block;
9396
9397         BCE_PRINTF(
9398                 "---------------"
9399                 " Stats Block  (All Stats Not Shown Are 0) "
9400                 "---------------\n");
9401
9402         if (sblk->stat_IfHCInOctets_hi
9403                 || sblk->stat_IfHCInOctets_lo)
9404                 BCE_PRINTF("0x%08X:%08X : "
9405                         "IfHcInOctets\n",
9406                         sblk->stat_IfHCInOctets_hi,
9407                         sblk->stat_IfHCInOctets_lo);
9408
9409         if (sblk->stat_IfHCInBadOctets_hi
9410                 || sblk->stat_IfHCInBadOctets_lo)
9411                 BCE_PRINTF("0x%08X:%08X : "
9412                         "IfHcInBadOctets\n",
9413                         sblk->stat_IfHCInBadOctets_hi,
9414                         sblk->stat_IfHCInBadOctets_lo);
9415
9416         if (sblk->stat_IfHCOutOctets_hi
9417                 || sblk->stat_IfHCOutOctets_lo)
9418                 BCE_PRINTF("0x%08X:%08X : "
9419                         "IfHcOutOctets\n",
9420                         sblk->stat_IfHCOutOctets_hi,
9421                         sblk->stat_IfHCOutOctets_lo);
9422
9423         if (sblk->stat_IfHCOutBadOctets_hi
9424                 || sblk->stat_IfHCOutBadOctets_lo)
9425                 BCE_PRINTF("0x%08X:%08X : "
9426                         "IfHcOutBadOctets\n",
9427                         sblk->stat_IfHCOutBadOctets_hi,
9428                         sblk->stat_IfHCOutBadOctets_lo);
9429
9430         if (sblk->stat_IfHCInUcastPkts_hi
9431                 || sblk->stat_IfHCInUcastPkts_lo)
9432                 BCE_PRINTF("0x%08X:%08X : "
9433                         "IfHcInUcastPkts\n",
9434                         sblk->stat_IfHCInUcastPkts_hi,
9435                         sblk->stat_IfHCInUcastPkts_lo);
9436
9437         if (sblk->stat_IfHCInBroadcastPkts_hi
9438                 || sblk->stat_IfHCInBroadcastPkts_lo)
9439                 BCE_PRINTF("0x%08X:%08X : "
9440                         "IfHcInBroadcastPkts\n",
9441                         sblk->stat_IfHCInBroadcastPkts_hi,
9442                         sblk->stat_IfHCInBroadcastPkts_lo);
9443
9444         if (sblk->stat_IfHCInMulticastPkts_hi
9445                 || sblk->stat_IfHCInMulticastPkts_lo)
9446                 BCE_PRINTF("0x%08X:%08X : "
9447                         "IfHcInMulticastPkts\n",
9448                         sblk->stat_IfHCInMulticastPkts_hi,
9449                         sblk->stat_IfHCInMulticastPkts_lo);
9450
9451         if (sblk->stat_IfHCOutUcastPkts_hi
9452                 || sblk->stat_IfHCOutUcastPkts_lo)
9453                 BCE_PRINTF("0x%08X:%08X : "
9454                         "IfHcOutUcastPkts\n",
9455                         sblk->stat_IfHCOutUcastPkts_hi,
9456                         sblk->stat_IfHCOutUcastPkts_lo);
9457
9458         if (sblk->stat_IfHCOutBroadcastPkts_hi
9459                 || sblk->stat_IfHCOutBroadcastPkts_lo)
9460                 BCE_PRINTF("0x%08X:%08X : "
9461                         "IfHcOutBroadcastPkts\n",
9462                         sblk->stat_IfHCOutBroadcastPkts_hi,
9463                         sblk->stat_IfHCOutBroadcastPkts_lo);
9464
9465         if (sblk->stat_IfHCOutMulticastPkts_hi
9466                 || sblk->stat_IfHCOutMulticastPkts_lo)
9467                 BCE_PRINTF("0x%08X:%08X : "
9468                         "IfHcOutMulticastPkts\n",
9469                         sblk->stat_IfHCOutMulticastPkts_hi,
9470                         sblk->stat_IfHCOutMulticastPkts_lo);
9471
9472         if (sblk->stat_emac_tx_stat_dot3statsinternalmactransmiterrors)
9473                 BCE_PRINTF("         0x%08X : "
9474                         "emac_tx_stat_dot3statsinternalmactransmiterrors\n",
9475                         sblk->stat_emac_tx_stat_dot3statsinternalmactransmiterrors);
9476
9477         if (sblk->stat_Dot3StatsCarrierSenseErrors)
9478                 BCE_PRINTF("         0x%08X : Dot3StatsCarrierSenseErrors\n",
9479                         sblk->stat_Dot3StatsCarrierSenseErrors);
9480
9481         if (sblk->stat_Dot3StatsFCSErrors)
9482                 BCE_PRINTF("         0x%08X : Dot3StatsFCSErrors\n",
9483                         sblk->stat_Dot3StatsFCSErrors);
9484
9485         if (sblk->stat_Dot3StatsAlignmentErrors)
9486                 BCE_PRINTF("         0x%08X : Dot3StatsAlignmentErrors\n",
9487                         sblk->stat_Dot3StatsAlignmentErrors);
9488
9489         if (sblk->stat_Dot3StatsSingleCollisionFrames)
9490                 BCE_PRINTF("         0x%08X : Dot3StatsSingleCollisionFrames\n",
9491                         sblk->stat_Dot3StatsSingleCollisionFrames);
9492
9493         if (sblk->stat_Dot3StatsMultipleCollisionFrames)
9494                 BCE_PRINTF("         0x%08X : Dot3StatsMultipleCollisionFrames\n",
9495                         sblk->stat_Dot3StatsMultipleCollisionFrames);
9496
9497         if (sblk->stat_Dot3StatsDeferredTransmissions)
9498                 BCE_PRINTF("         0x%08X : Dot3StatsDeferredTransmissions\n",
9499                         sblk->stat_Dot3StatsDeferredTransmissions);
9500
9501         if (sblk->stat_Dot3StatsExcessiveCollisions)
9502                 BCE_PRINTF("         0x%08X : Dot3StatsExcessiveCollisions\n",
9503                         sblk->stat_Dot3StatsExcessiveCollisions);
9504
9505         if (sblk->stat_Dot3StatsLateCollisions)
9506                 BCE_PRINTF("         0x%08X : Dot3StatsLateCollisions\n",
9507                         sblk->stat_Dot3StatsLateCollisions);
9508
9509         if (sblk->stat_EtherStatsCollisions)
9510                 BCE_PRINTF("         0x%08X : EtherStatsCollisions\n",
9511                         sblk->stat_EtherStatsCollisions);
9512
9513         if (sblk->stat_EtherStatsFragments)
9514                 BCE_PRINTF("         0x%08X : EtherStatsFragments\n",
9515                         sblk->stat_EtherStatsFragments);
9516
9517         if (sblk->stat_EtherStatsJabbers)
9518                 BCE_PRINTF("         0x%08X : EtherStatsJabbers\n",
9519                         sblk->stat_EtherStatsJabbers);
9520
9521         if (sblk->stat_EtherStatsUndersizePkts)
9522                 BCE_PRINTF("         0x%08X : EtherStatsUndersizePkts\n",
9523                         sblk->stat_EtherStatsUndersizePkts);
9524
9525         if (sblk->stat_EtherStatsOversizePkts)
9526                 BCE_PRINTF("         0x%08X : EtherStatsOverrsizePkts\n",
9527                         sblk->stat_EtherStatsOversizePkts);
9528
9529         if (sblk->stat_EtherStatsPktsRx64Octets)
9530                 BCE_PRINTF("         0x%08X : EtherStatsPktsRx64Octets\n",
9531                         sblk->stat_EtherStatsPktsRx64Octets);
9532
9533         if (sblk->stat_EtherStatsPktsRx65Octetsto127Octets)
9534                 BCE_PRINTF("         0x%08X : EtherStatsPktsRx65Octetsto127Octets\n",
9535                         sblk->stat_EtherStatsPktsRx65Octetsto127Octets);
9536
9537         if (sblk->stat_EtherStatsPktsRx128Octetsto255Octets)
9538                 BCE_PRINTF("         0x%08X : EtherStatsPktsRx128Octetsto255Octets\n",
9539                         sblk->stat_EtherStatsPktsRx128Octetsto255Octets);
9540
9541         if (sblk->stat_EtherStatsPktsRx256Octetsto511Octets)
9542                 BCE_PRINTF("         0x%08X : EtherStatsPktsRx256Octetsto511Octets\n",
9543                         sblk->stat_EtherStatsPktsRx256Octetsto511Octets);
9544
9545         if (sblk->stat_EtherStatsPktsRx512Octetsto1023Octets)
9546                 BCE_PRINTF("         0x%08X : EtherStatsPktsRx512Octetsto1023Octets\n",
9547                         sblk->stat_EtherStatsPktsRx512Octetsto1023Octets);
9548
9549         if (sblk->stat_EtherStatsPktsRx1024Octetsto1522Octets)
9550                 BCE_PRINTF("         0x%08X : EtherStatsPktsRx1024Octetsto1522Octets\n",
9551                         sblk->stat_EtherStatsPktsRx1024Octetsto1522Octets);
9552
9553         if (sblk->stat_EtherStatsPktsRx1523Octetsto9022Octets)
9554                 BCE_PRINTF("         0x%08X : EtherStatsPktsRx1523Octetsto9022Octets\n",
9555                         sblk->stat_EtherStatsPktsRx1523Octetsto9022Octets);
9556
9557         if (sblk->stat_EtherStatsPktsTx64Octets)
9558                 BCE_PRINTF("         0x%08X : EtherStatsPktsTx64Octets\n",
9559                         sblk->stat_EtherStatsPktsTx64Octets);
9560
9561         if (sblk->stat_EtherStatsPktsTx65Octetsto127Octets)
9562                 BCE_PRINTF("         0x%08X : EtherStatsPktsTx65Octetsto127Octets\n",
9563                         sblk->stat_EtherStatsPktsTx65Octetsto127Octets);
9564
9565         if (sblk->stat_EtherStatsPktsTx128Octetsto255Octets)
9566                 BCE_PRINTF("         0x%08X : EtherStatsPktsTx128Octetsto255Octets\n",
9567                         sblk->stat_EtherStatsPktsTx128Octetsto255Octets);
9568
9569         if (sblk->stat_EtherStatsPktsTx256Octetsto511Octets)
9570                 BCE_PRINTF("         0x%08X : EtherStatsPktsTx256Octetsto511Octets\n",
9571                         sblk->stat_EtherStatsPktsTx256Octetsto511Octets);
9572
9573         if (sblk->stat_EtherStatsPktsTx512Octetsto1023Octets)
9574                 BCE_PRINTF("         0x%08X : EtherStatsPktsTx512Octetsto1023Octets\n",
9575                         sblk->stat_EtherStatsPktsTx512Octetsto1023Octets);
9576
9577         if (sblk->stat_EtherStatsPktsTx1024Octetsto1522Octets)
9578                 BCE_PRINTF("         0x%08X : EtherStatsPktsTx1024Octetsto1522Octets\n",
9579                         sblk->stat_EtherStatsPktsTx1024Octetsto1522Octets);
9580
9581         if (sblk->stat_EtherStatsPktsTx1523Octetsto9022Octets)
9582                 BCE_PRINTF("         0x%08X : EtherStatsPktsTx1523Octetsto9022Octets\n",
9583                         sblk->stat_EtherStatsPktsTx1523Octetsto9022Octets);
9584
9585         if (sblk->stat_XonPauseFramesReceived)
9586                 BCE_PRINTF("         0x%08X : XonPauseFramesReceived\n",
9587                         sblk->stat_XonPauseFramesReceived);
9588
9589         if (sblk->stat_XoffPauseFramesReceived)
9590            BCE_PRINTF("          0x%08X : XoffPauseFramesReceived\n",
9591                         sblk->stat_XoffPauseFramesReceived);
9592
9593         if (sblk->stat_OutXonSent)
9594                 BCE_PRINTF("         0x%08X : OutXonSent\n",
9595                         sblk->stat_OutXonSent);
9596
9597         if (sblk->stat_OutXoffSent)
9598                 BCE_PRINTF("         0x%08X : OutXoffSent\n",
9599                         sblk->stat_OutXoffSent);
9600
9601         if (sblk->stat_FlowControlDone)
9602                 BCE_PRINTF("         0x%08X : FlowControlDone\n",
9603                         sblk->stat_FlowControlDone);
9604
9605         if (sblk->stat_MacControlFramesReceived)
9606                 BCE_PRINTF("         0x%08X : MacControlFramesReceived\n",
9607                         sblk->stat_MacControlFramesReceived);
9608
9609         if (sblk->stat_XoffStateEntered)
9610                 BCE_PRINTF("         0x%08X : XoffStateEntered\n",
9611                         sblk->stat_XoffStateEntered);
9612
9613         if (sblk->stat_IfInFramesL2FilterDiscards)
9614                 BCE_PRINTF("         0x%08X : IfInFramesL2FilterDiscards\n",
9615                         sblk->stat_IfInFramesL2FilterDiscards);
9616
9617         if (sblk->stat_IfInRuleCheckerDiscards)
9618                 BCE_PRINTF("         0x%08X : IfInRuleCheckerDiscards\n",
9619                         sblk->stat_IfInRuleCheckerDiscards);
9620
9621         if (sblk->stat_IfInFTQDiscards)
9622                 BCE_PRINTF("         0x%08X : IfInFTQDiscards\n",
9623                         sblk->stat_IfInFTQDiscards);
9624
9625         if (sblk->stat_IfInMBUFDiscards)
9626                 BCE_PRINTF("         0x%08X : IfInMBUFDiscards\n",
9627                         sblk->stat_IfInMBUFDiscards);
9628
9629         if (sblk->stat_IfInRuleCheckerP4Hit)
9630                 BCE_PRINTF("         0x%08X : IfInRuleCheckerP4Hit\n",
9631                         sblk->stat_IfInRuleCheckerP4Hit);
9632
9633         if (sblk->stat_CatchupInRuleCheckerDiscards)
9634                 BCE_PRINTF("         0x%08X : CatchupInRuleCheckerDiscards\n",
9635                         sblk->stat_CatchupInRuleCheckerDiscards);
9636
9637         if (sblk->stat_CatchupInFTQDiscards)
9638                 BCE_PRINTF("         0x%08X : CatchupInFTQDiscards\n",
9639                         sblk->stat_CatchupInFTQDiscards);
9640
9641         if (sblk->stat_CatchupInMBUFDiscards)
9642                 BCE_PRINTF("         0x%08X : CatchupInMBUFDiscards\n",
9643                         sblk->stat_CatchupInMBUFDiscards);
9644
9645         if (sblk->stat_CatchupInRuleCheckerP4Hit)
9646                 BCE_PRINTF("         0x%08X : CatchupInRuleCheckerP4Hit\n",
9647                         sblk->stat_CatchupInRuleCheckerP4Hit);
9648
9649         BCE_PRINTF(
9650                 "----------------------------"
9651                 "----------------"
9652                 "----------------------------\n");
9653 }
9654
9655
9656 /****************************************************************************/
9657 /* Prints out a summary of the driver state.                                */
9658 /*                                                                          */
9659 /* Returns:                                                                 */
9660 /*   Nothing.                                                               */
9661 /****************************************************************************/
9662 static __attribute__ ((noinline)) void
9663 bce_dump_driver_state(struct bce_softc *sc)
9664 {
9665         u32 val_hi, val_lo;
9666
9667         BCE_PRINTF(
9668                 "-----------------------------"
9669                 " Driver State "
9670                 "-----------------------------\n");
9671
9672         val_hi = BCE_ADDR_HI(sc);
9673         val_lo = BCE_ADDR_LO(sc);
9674         BCE_PRINTF("0x%08X:%08X - (sc) driver softc structure virtual address\n",
9675                 val_hi, val_lo);
9676
9677         val_hi = BCE_ADDR_HI(sc->bce_vhandle);
9678         val_lo = BCE_ADDR_LO(sc->bce_vhandle);
9679         BCE_PRINTF("0x%08X:%08X - (sc->bce_vhandle) PCI BAR virtual address\n",
9680                 val_hi, val_lo);
9681
9682         val_hi = BCE_ADDR_HI(sc->status_block);
9683         val_lo = BCE_ADDR_LO(sc->status_block);
9684         BCE_PRINTF("0x%08X:%08X - (sc->status_block) status block virtual address\n",
9685                 val_hi, val_lo);
9686
9687         val_hi = BCE_ADDR_HI(sc->stats_block);
9688         val_lo = BCE_ADDR_LO(sc->stats_block);
9689         BCE_PRINTF("0x%08X:%08X - (sc->stats_block) statistics block virtual address\n",
9690                 val_hi, val_lo);
9691
9692         val_hi = BCE_ADDR_HI(sc->tx_bd_chain);
9693         val_lo = BCE_ADDR_LO(sc->tx_bd_chain);
9694         BCE_PRINTF(
9695                 "0x%08X:%08X - (sc->tx_bd_chain) tx_bd chain virtual adddress\n",
9696                 val_hi, val_lo);
9697
9698         val_hi = BCE_ADDR_HI(sc->rx_bd_chain);
9699         val_lo = BCE_ADDR_LO(sc->rx_bd_chain);
9700         BCE_PRINTF(
9701                 "0x%08X:%08X - (sc->rx_bd_chain) rx_bd chain virtual address\n",
9702                 val_hi, val_lo);
9703
9704 #ifdef ZERO_COPY_SOCKETS
9705         val_hi = BCE_ADDR_HI(sc->pg_bd_chain);
9706         val_lo = BCE_ADDR_LO(sc->pg_bd_chain);
9707         BCE_PRINTF(
9708                 "0x%08X:%08X - (sc->pg_bd_chain) page chain virtual address\n",
9709                 val_hi, val_lo);
9710 #endif
9711
9712         val_hi = BCE_ADDR_HI(sc->tx_mbuf_ptr);
9713         val_lo = BCE_ADDR_LO(sc->tx_mbuf_ptr);
9714         BCE_PRINTF(
9715                 "0x%08X:%08X - (sc->tx_mbuf_ptr) tx mbuf chain virtual address\n",
9716                 val_hi, val_lo);
9717
9718         val_hi = BCE_ADDR_HI(sc->rx_mbuf_ptr);
9719         val_lo = BCE_ADDR_LO(sc->rx_mbuf_ptr);
9720         BCE_PRINTF(
9721                 "0x%08X:%08X - (sc->rx_mbuf_ptr) rx mbuf chain virtual address\n",
9722                 val_hi, val_lo);
9723
9724 #ifdef ZERO_COPY_SOCKETS
9725         val_hi = BCE_ADDR_HI(sc->pg_mbuf_ptr);
9726         val_lo = BCE_ADDR_LO(sc->pg_mbuf_ptr);
9727         BCE_PRINTF(
9728                 "0x%08X:%08X - (sc->pg_mbuf_ptr) page mbuf chain virtual address\n",
9729                 val_hi, val_lo);
9730 #endif
9731
9732         BCE_PRINTF("         0x%08X - (sc->interrupts_generated) h/w intrs\n",
9733                 sc->interrupts_generated);
9734
9735         BCE_PRINTF("         0x%08X - (sc->rx_interrupts) rx interrupts handled\n",
9736                 sc->rx_interrupts);
9737
9738         BCE_PRINTF("         0x%08X - (sc->tx_interrupts) tx interrupts handled\n",
9739                 sc->tx_interrupts);
9740
9741         BCE_PRINTF("         0x%08X - (sc->last_status_idx) status block index\n",
9742                 sc->last_status_idx);
9743
9744         BCE_PRINTF("     0x%04X(0x%04X) - (sc->tx_prod) tx producer index\n",
9745                 sc->tx_prod, (u16) TX_CHAIN_IDX(sc->tx_prod));
9746
9747         BCE_PRINTF("     0x%04X(0x%04X) - (sc->tx_cons) tx consumer index\n",
9748                 sc->tx_cons, (u16) TX_CHAIN_IDX(sc->tx_cons));
9749
9750         BCE_PRINTF("         0x%08X - (sc->tx_prod_bseq) tx producer bseq index\n",
9751                 sc->tx_prod_bseq);
9752
9753         BCE_PRINTF("         0x%08X - (sc->debug_tx_mbuf_alloc) tx mbufs allocated\n",
9754                 sc->debug_tx_mbuf_alloc);
9755
9756         BCE_PRINTF("         0x%08X - (sc->used_tx_bd) used tx_bd's\n",
9757                 sc->used_tx_bd);
9758
9759         BCE_PRINTF("0x%08X/%08X - (sc->tx_hi_watermark) tx hi watermark\n",
9760                 sc->tx_hi_watermark, sc->max_tx_bd);
9761
9762         BCE_PRINTF("     0x%04X(0x%04X) - (sc->rx_prod) rx producer index\n",
9763                 sc->rx_prod, (u16) RX_CHAIN_IDX(sc->rx_prod));
9764
9765         BCE_PRINTF("     0x%04X(0x%04X) - (sc->rx_cons) rx consumer index\n",
9766                 sc->rx_cons, (u16) RX_CHAIN_IDX(sc->rx_cons));
9767
9768         BCE_PRINTF("         0x%08X - (sc->rx_prod_bseq) rx producer bseq index\n",
9769                 sc->rx_prod_bseq);
9770
9771         BCE_PRINTF("         0x%08X - (sc->debug_rx_mbuf_alloc) rx mbufs allocated\n",
9772                 sc->debug_rx_mbuf_alloc);
9773
9774         BCE_PRINTF("         0x%08X - (sc->free_rx_bd) free rx_bd's\n",
9775                 sc->free_rx_bd);
9776
9777 #ifdef ZERO_COPY_SOCKETS
9778         BCE_PRINTF("     0x%04X(0x%04X) - (sc->pg_prod) page producer index\n",
9779                 sc->pg_prod, (u16) PG_CHAIN_IDX(sc->pg_prod));
9780
9781         BCE_PRINTF("     0x%04X(0x%04X) - (sc->pg_cons) page consumer index\n",
9782                 sc->pg_cons, (u16) PG_CHAIN_IDX(sc->pg_cons));
9783
9784         BCE_PRINTF("         0x%08X - (sc->debug_pg_mbuf_alloc) page mbufs allocated\n",
9785                 sc->debug_pg_mbuf_alloc);
9786
9787         BCE_PRINTF("         0x%08X - (sc->free_pg_bd) free page rx_bd's\n",
9788                 sc->free_pg_bd);
9789
9790         BCE_PRINTF("0x%08X/%08X - (sc->pg_low_watermark) page low watermark\n",
9791                 sc->pg_low_watermark, sc->max_pg_bd);
9792 #endif
9793
9794         BCE_PRINTF("         0x%08X - (sc->mbuf_alloc_failed_count) "
9795                 "mbuf alloc failures\n",
9796                 sc->mbuf_alloc_failed_count);
9797
9798         BCE_PRINTF("         0x%08X - (sc->bce_flags) bce mac flags\n",
9799                 sc->bce_flags);
9800
9801         BCE_PRINTF("         0x%08X - (sc->bce_phy_flags) bce phy flags\n",
9802                 sc->bce_phy_flags);
9803
9804         BCE_PRINTF(
9805                 "----------------------------"
9806                 "----------------"
9807                 "----------------------------\n");
9808 }
9809
9810
9811 /****************************************************************************/
9812 /* Prints out the hardware state through a summary of important register,   */
9813 /* followed by a complete register dump.                                    */
9814 /*                                                                          */
9815 /* Returns:                                                                 */
9816 /*   Nothing.                                                               */
9817 /****************************************************************************/
9818 static __attribute__ ((noinline)) void
9819 bce_dump_hw_state(struct bce_softc *sc)
9820 {
9821         u32 val;
9822
9823         BCE_PRINTF(
9824                 "----------------------------"
9825                 " Hardware State "
9826                 "----------------------------\n");
9827
9828         BCE_PRINTF("0x%08X - bootcode version\n", sc->bce_bc_ver);
9829
9830         val = REG_RD(sc, BCE_MISC_ENABLE_STATUS_BITS);
9831         BCE_PRINTF("0x%08X - (0x%06X) misc_enable_status_bits\n",
9832                 val, BCE_MISC_ENABLE_STATUS_BITS);
9833
9834         val = REG_RD(sc, BCE_DMA_STATUS);
9835         BCE_PRINTF("0x%08X - (0x%06X) dma_status\n", val, BCE_DMA_STATUS);
9836
9837         val = REG_RD(sc, BCE_CTX_STATUS);
9838         BCE_PRINTF("0x%08X - (0x%06X) ctx_status\n", val, BCE_CTX_STATUS);
9839
9840         val = REG_RD(sc, BCE_EMAC_STATUS);
9841         BCE_PRINTF("0x%08X - (0x%06X) emac_status\n", val, BCE_EMAC_STATUS);
9842
9843         val = REG_RD(sc, BCE_RPM_STATUS);
9844         BCE_PRINTF("0x%08X - (0x%06X) rpm_status\n", val, BCE_RPM_STATUS);
9845
9846         val = REG_RD(sc, 0x2004);
9847         BCE_PRINTF("0x%08X - (0x%06X) rlup_status\n", val, 0x2004);
9848
9849         val = REG_RD(sc, BCE_RV2P_STATUS);
9850         BCE_PRINTF("0x%08X - (0x%06X) rv2p_status\n", val, BCE_RV2P_STATUS);
9851
9852         val = REG_RD(sc, 0x2c04);
9853         BCE_PRINTF("0x%08X - (0x%06X) rdma_status\n", val, 0x2c04);
9854
9855         val = REG_RD(sc, BCE_TBDR_STATUS);
9856         BCE_PRINTF("0x%08X - (0x%06X) tbdr_status\n", val, BCE_TBDR_STATUS);
9857
9858         val = REG_RD(sc, BCE_TDMA_STATUS);
9859         BCE_PRINTF("0x%08X - (0x%06X) tdma_status\n", val, BCE_TDMA_STATUS);
9860
9861         val = REG_RD(sc, BCE_HC_STATUS);
9862         BCE_PRINTF("0x%08X - (0x%06X) hc_status\n", val, BCE_HC_STATUS);
9863
9864         val = REG_RD_IND(sc, BCE_TXP_CPU_STATE);
9865         BCE_PRINTF("0x%08X - (0x%06X) txp_cpu_state\n", val, BCE_TXP_CPU_STATE);
9866
9867         val = REG_RD_IND(sc, BCE_TPAT_CPU_STATE);
9868         BCE_PRINTF("0x%08X - (0x%06X) tpat_cpu_state\n", val, BCE_TPAT_CPU_STATE);
9869
9870         val = REG_RD_IND(sc, BCE_RXP_CPU_STATE);
9871         BCE_PRINTF("0x%08X - (0x%06X) rxp_cpu_state\n", val, BCE_RXP_CPU_STATE);
9872
9873         val = REG_RD_IND(sc, BCE_COM_CPU_STATE);
9874         BCE_PRINTF("0x%08X - (0x%06X) com_cpu_state\n", val, BCE_COM_CPU_STATE);
9875
9876         val = REG_RD_IND(sc, BCE_MCP_CPU_STATE);
9877         BCE_PRINTF("0x%08X - (0x%06X) mcp_cpu_state\n", val, BCE_MCP_CPU_STATE);
9878
9879         val = REG_RD_IND(sc, BCE_CP_CPU_STATE);
9880         BCE_PRINTF("0x%08X - (0x%06X) cp_cpu_state\n", val, BCE_CP_CPU_STATE);
9881
9882         BCE_PRINTF(
9883                 "----------------------------"
9884                 "----------------"
9885                 "----------------------------\n");
9886
9887         BCE_PRINTF(
9888                 "----------------------------"
9889                 " Register  Dump "
9890                 "----------------------------\n");
9891
9892         for (int i = 0x400; i < 0x8000; i += 0x10) {
9893                 BCE_PRINTF("0x%04X: 0x%08X 0x%08X 0x%08X 0x%08X\n",
9894                         i, REG_RD(sc, i), REG_RD(sc, i + 0x4),
9895                         REG_RD(sc, i + 0x8), REG_RD(sc, i + 0xC));
9896         }
9897
9898         BCE_PRINTF(
9899                 "----------------------------"
9900                 "----------------"
9901                 "----------------------------\n");
9902 }
9903
9904
9905 /****************************************************************************/
9906 /* Prints out the mailbox queue registers.                                  */
9907 /*                                                                          */
9908 /* Returns:                                                                 */
9909 /*   Nothing.                                                               */
9910 /****************************************************************************/
9911 static __attribute__ ((noinline)) void
9912 bce_dump_mq_regs(struct bce_softc *sc)
9913 {
9914         BCE_PRINTF(
9915                 "----------------------------"
9916                 "    MQ Regs     "
9917                 "----------------------------\n");
9918
9919         BCE_PRINTF(
9920                 "----------------------------"
9921                 "----------------"
9922                 "----------------------------\n");
9923
9924         for (int i = 0x3c00; i < 0x4000; i += 0x10) {
9925                 BCE_PRINTF("0x%04X: 0x%08X 0x%08X 0x%08X 0x%08X\n",
9926                         i, REG_RD(sc, i), REG_RD(sc, i + 0x4),
9927                         REG_RD(sc, i + 0x8), REG_RD(sc, i + 0xC));
9928         }
9929
9930         BCE_PRINTF(
9931                 "----------------------------"
9932                 "----------------"
9933                 "----------------------------\n");
9934 }
9935
9936
9937 /****************************************************************************/
9938 /* Prints out the bootcode state.                                           */
9939 /*                                                                          */
9940 /* Returns:                                                                 */
9941 /*   Nothing.                                                               */
9942 /****************************************************************************/
9943 static __attribute__ ((noinline)) void
9944 bce_dump_bc_state(struct bce_softc *sc)
9945 {
9946         u32 val;
9947
9948         BCE_PRINTF(
9949                 "----------------------------"
9950                 " Bootcode State "
9951                 "----------------------------\n");
9952
9953         BCE_PRINTF("0x%08X - bootcode version\n", sc->bce_bc_ver);
9954
9955         val = REG_RD_IND(sc, sc->bce_shmem_base + BCE_BC_RESET_TYPE);
9956         BCE_PRINTF("0x%08X - (0x%06X) reset_type\n",
9957                 val, BCE_BC_RESET_TYPE);
9958
9959         val = REG_RD_IND(sc, sc->bce_shmem_base + BCE_BC_STATE);
9960         BCE_PRINTF("0x%08X - (0x%06X) state\n",
9961                 val, BCE_BC_STATE);
9962
9963         val = REG_RD_IND(sc, sc->bce_shmem_base + BCE_BC_CONDITION);
9964         BCE_PRINTF("0x%08X - (0x%06X) condition\n",
9965                 val, BCE_BC_CONDITION);
9966
9967         val = REG_RD_IND(sc, sc->bce_shmem_base + BCE_BC_STATE_DEBUG_CMD);
9968         BCE_PRINTF("0x%08X - (0x%06X) debug_cmd\n",
9969                 val, BCE_BC_STATE_DEBUG_CMD);
9970
9971         BCE_PRINTF(
9972                 "----------------------------"
9973                 "----------------"
9974                 "----------------------------\n");
9975 }
9976
9977
9978 /****************************************************************************/
9979 /* Prints out the TXP processor state.                                      */
9980 /*                                                                          */
9981 /* Returns:                                                                 */
9982 /*   Nothing.                                                               */
9983 /****************************************************************************/
9984 static __attribute__ ((noinline)) void
9985 bce_dump_txp_state(struct bce_softc *sc, int regs)
9986 {
9987         u32 val;
9988         u32 fw_version[3];
9989
9990         BCE_PRINTF(
9991                 "----------------------------"
9992                 "   TXP  State   "
9993                 "----------------------------\n");
9994
9995         for (int i = 0; i < 3; i++)
9996                 fw_version[i] = htonl(REG_RD_IND(sc,
9997                         (BCE_TXP_SCRATCH + 0x10 + i * 4)));
9998         BCE_PRINTF("Firmware version - %s\n", (char *) fw_version);
9999
10000         val = REG_RD_IND(sc, BCE_TXP_CPU_MODE);
10001         BCE_PRINTF("0x%08X - (0x%06X) txp_cpu_mode\n", val, BCE_TXP_CPU_MODE);
10002
10003         val = REG_RD_IND(sc, BCE_TXP_CPU_STATE);
10004         BCE_PRINTF("0x%08X - (0x%06X) txp_cpu_state\n", val, BCE_TXP_CPU_STATE);
10005
10006         val = REG_RD_IND(sc, BCE_TXP_CPU_EVENT_MASK);
10007         BCE_PRINTF("0x%08X - (0x%06X) txp_cpu_event_mask\n", val,
10008                 BCE_TXP_CPU_EVENT_MASK);
10009
10010         if (regs) {
10011                 BCE_PRINTF(
10012                         "----------------------------"
10013                         " Register  Dump "
10014                         "----------------------------\n");
10015
10016                 for (int i = BCE_TXP_CPU_MODE; i < 0x68000; i += 0x10) {
10017                         /* Skip the big blank spaces */
10018                         if (i < 0x454000 && i > 0x5ffff)
10019                                 BCE_PRINTF("0x%04X: 0x%08X 0x%08X 0x%08X 0x%08X\n",
10020                                         i, REG_RD_IND(sc, i), REG_RD_IND(sc, i + 0x4),
10021                                         REG_RD_IND(sc, i + 0x8), REG_RD_IND(sc, i + 0xC));
10022                 }
10023         }
10024
10025         BCE_PRINTF(
10026                 "----------------------------"
10027                 "----------------"
10028                 "----------------------------\n");
10029 }
10030
10031
10032 /****************************************************************************/
10033 /* Prints out the RXP processor state.                                      */
10034 /*                                                                          */
10035 /* Returns:                                                                 */
10036 /*   Nothing.                                                               */
10037 /****************************************************************************/
10038 static __attribute__ ((noinline)) void
10039 bce_dump_rxp_state(struct bce_softc *sc, int regs)
10040 {
10041         u32 val;
10042         u32 fw_version[3];
10043
10044         BCE_PRINTF(
10045                 "----------------------------"
10046                 "   RXP  State   "
10047                 "----------------------------\n");
10048
10049         for (int i = 0; i < 3; i++)
10050                 fw_version[i] = htonl(REG_RD_IND(sc,
10051                         (BCE_RXP_SCRATCH + 0x10 + i * 4)));
10052         BCE_PRINTF("Firmware version - %s\n", (char *) fw_version);
10053
10054         val = REG_RD_IND(sc, BCE_RXP_CPU_MODE);
10055         BCE_PRINTF("0x%08X - (0x%06X) rxp_cpu_mode\n", val, BCE_RXP_CPU_MODE);
10056
10057         val = REG_RD_IND(sc, BCE_RXP_CPU_STATE);
10058         BCE_PRINTF("0x%08X - (0x%06X) rxp_cpu_state\n", val, BCE_RXP_CPU_STATE);
10059
10060         val = REG_RD_IND(sc, BCE_RXP_CPU_EVENT_MASK);
10061         BCE_PRINTF("0x%08X - (0x%06X) rxp_cpu_event_mask\n", val,
10062                 BCE_RXP_CPU_EVENT_MASK);
10063
10064         if (regs) {
10065                 BCE_PRINTF(
10066                         "----------------------------"
10067                         " Register  Dump "
10068                         "----------------------------\n");
10069
10070                 for (int i = BCE_RXP_CPU_MODE; i < 0xe8fff; i += 0x10) {
10071                         /* Skip the big blank sapces */
10072                         if (i < 0xc5400 && i > 0xdffff)
10073                                 BCE_PRINTF("0x%04X: 0x%08X 0x%08X 0x%08X 0x%08X\n",
10074                                         i, REG_RD_IND(sc, i), REG_RD_IND(sc, i + 0x4),
10075                                         REG_RD_IND(sc, i + 0x8), REG_RD_IND(sc, i + 0xC));
10076                 }
10077         }
10078
10079         BCE_PRINTF(
10080                 "----------------------------"
10081                 "----------------"
10082                 "----------------------------\n");
10083 }
10084
10085
10086 /****************************************************************************/
10087 /* Prints out the TPAT processor state.                                     */
10088 /*                                                                          */
10089 /* Returns:                                                                 */
10090 /*   Nothing.                                                               */
10091 /****************************************************************************/
10092 static __attribute__ ((noinline)) void
10093 bce_dump_tpat_state(struct bce_softc *sc, int regs)
10094 {
10095         u32 val;
10096         u32 fw_version[3];
10097
10098         BCE_PRINTF(
10099                 "----------------------------"
10100                 "   TPAT State   "
10101                 "----------------------------\n");
10102
10103         for (int i = 0; i < 3; i++)
10104                 fw_version[i] = htonl(REG_RD_IND(sc,
10105                         (BCE_TPAT_SCRATCH + 0x410 + i * 4)));
10106         BCE_PRINTF("Firmware version - %s\n", (char *) fw_version);
10107
10108         val = REG_RD_IND(sc, BCE_TPAT_CPU_MODE);
10109         BCE_PRINTF("0x%08X - (0x%06X) tpat_cpu_mode\n", val, BCE_TPAT_CPU_MODE);
10110
10111         val = REG_RD_IND(sc, BCE_TPAT_CPU_STATE);
10112         BCE_PRINTF("0x%08X - (0x%06X) tpat_cpu_state\n", val, BCE_TPAT_CPU_STATE);
10113
10114         val = REG_RD_IND(sc, BCE_TPAT_CPU_EVENT_MASK);
10115         BCE_PRINTF("0x%08X - (0x%06X) tpat_cpu_event_mask\n", val,
10116                 BCE_TPAT_CPU_EVENT_MASK);
10117
10118         if (regs) {
10119                 BCE_PRINTF(
10120                         "----------------------------"
10121                         " Register  Dump "
10122                         "----------------------------\n");
10123
10124                 for (int i = BCE_TPAT_CPU_MODE; i < 0xa3fff; i += 0x10) {
10125                         /* Skip the big blank spaces */
10126                         if (i < 0x854000 && i > 0x9ffff)
10127                                 BCE_PRINTF("0x%04X: 0x%08X 0x%08X 0x%08X 0x%08X\n",
10128                                         i, REG_RD_IND(sc, i), REG_RD_IND(sc, i + 0x4),
10129                                         REG_RD_IND(sc, i + 0x8), REG_RD_IND(sc, i + 0xC));
10130                 }
10131         }
10132
10133         BCE_PRINTF(
10134                 "----------------------------"
10135                 "----------------"
10136                 "----------------------------\n");
10137 }
10138
10139
10140 /****************************************************************************/
10141 /* Prints out the Command Procesor (CP) state.                              */
10142 /*                                                                          */
10143 /* Returns:                                                                 */
10144 /*   Nothing.                                                               */
10145 /****************************************************************************/
10146 static __attribute__ ((noinline)) void
10147 bce_dump_cp_state(struct bce_softc *sc, int regs)
10148 {
10149         u32 val;
10150         u32 fw_version[3];
10151
10152         BCE_PRINTF(
10153                 "----------------------------"
10154                 "    CP State    "
10155                 "----------------------------\n");
10156
10157         for (int i = 0; i < 3; i++)
10158                 fw_version[i] = htonl(REG_RD_IND(sc,
10159                         (BCE_CP_SCRATCH + 0x10 + i * 4)));
10160         BCE_PRINTF("Firmware version - %s\n", (char *) fw_version);
10161
10162         val = REG_RD_IND(sc, BCE_CP_CPU_MODE);
10163         BCE_PRINTF("0x%08X - (0x%06X) cp_cpu_mode\n", val, BCE_CP_CPU_MODE);
10164
10165         val = REG_RD_IND(sc, BCE_CP_CPU_STATE);
10166         BCE_PRINTF("0x%08X - (0x%06X) cp_cpu_state\n", val, BCE_CP_CPU_STATE);
10167
10168         val = REG_RD_IND(sc, BCE_CP_CPU_EVENT_MASK);
10169         BCE_PRINTF("0x%08X - (0x%06X) cp_cpu_event_mask\n", val,
10170                 BCE_CP_CPU_EVENT_MASK);
10171
10172         if (regs) {
10173                 BCE_PRINTF(
10174                         "----------------------------"
10175                         " Register  Dump "
10176                         "----------------------------\n");
10177
10178                 for (int i = BCE_CP_CPU_MODE; i < 0x1aa000; i += 0x10) {
10179                         /* Skip the big blank spaces */
10180                         if (i < 0x185400 && i > 0x19ffff)
10181                                 BCE_PRINTF("0x%04X: 0x%08X 0x%08X 0x%08X 0x%08X\n",
10182                                         i, REG_RD_IND(sc, i), REG_RD_IND(sc, i + 0x4),
10183                                         REG_RD_IND(sc, i + 0x8), REG_RD_IND(sc, i + 0xC));
10184                 }
10185         }
10186
10187         BCE_PRINTF(
10188                 "----------------------------"
10189                 "----------------"
10190                 "----------------------------\n");
10191 }
10192
10193
10194 /****************************************************************************/
10195 /* Prints out the Completion Procesor (COM) state.                          */
10196 /*                                                                          */
10197 /* Returns:                                                                 */
10198 /*   Nothing.                                                               */
10199 /****************************************************************************/
10200 static __attribute__ ((noinline)) void
10201 bce_dump_com_state(struct bce_softc *sc, int regs)
10202 {
10203         u32 val;
10204         u32 fw_version[3];
10205
10206         BCE_PRINTF(
10207                 "----------------------------"
10208                 "   COM State    "
10209                 "----------------------------\n");
10210
10211         for (int i = 0; i < 3; i++)
10212                 fw_version[i] = htonl(REG_RD_IND(sc,
10213                         (BCE_COM_SCRATCH + 0x10 + i * 4)));
10214         BCE_PRINTF("Firmware version - %s\n", (char *) fw_version);
10215
10216         val = REG_RD_IND(sc, BCE_COM_CPU_MODE);
10217         BCE_PRINTF("0x%08X - (0x%06X) com_cpu_mode\n", val, BCE_COM_CPU_MODE);
10218
10219         val = REG_RD_IND(sc, BCE_COM_CPU_STATE);
10220         BCE_PRINTF("0x%08X - (0x%06X) com_cpu_state\n", val, BCE_COM_CPU_STATE);
10221
10222         val = REG_RD_IND(sc, BCE_COM_CPU_EVENT_MASK);
10223         BCE_PRINTF("0x%08X - (0x%06X) com_cpu_event_mask\n", val,
10224                 BCE_COM_CPU_EVENT_MASK);
10225
10226         if (regs) {
10227                 BCE_PRINTF(
10228                         "----------------------------"
10229                         " Register  Dump "
10230                         "----------------------------\n");
10231
10232                 for (int i = BCE_COM_CPU_MODE; i < 0x1053e8; i += 0x10) {
10233                         BCE_PRINTF("0x%04X: 0x%08X 0x%08X 0x%08X 0x%08X\n",
10234                                 i, REG_RD_IND(sc, i), REG_RD_IND(sc, i + 0x4),
10235                                 REG_RD_IND(sc, i + 0x8), REG_RD_IND(sc, i + 0xC));
10236                 }
10237         }
10238
10239         BCE_PRINTF(
10240                 "----------------------------"
10241                 "----------------"
10242                 "----------------------------\n");
10243 }
10244
10245
10246 /****************************************************************************/
10247 /* Prints out the driver state and then enters the debugger.                */
10248 /*                                                                          */
10249 /* Returns:                                                                 */
10250 /*   Nothing.                                                               */
10251 /****************************************************************************/
10252 static void
10253 bce_breakpoint(struct bce_softc *sc)
10254 {
10255
10256         /*
10257          * Unreachable code to silence compiler warnings
10258          * about unused functions.
10259          */
10260         if (0) {
10261                 bce_freeze_controller(sc);
10262                 bce_unfreeze_controller(sc);
10263                 bce_dump_enet(sc, NULL);
10264                 bce_dump_txbd(sc, 0, NULL);
10265                 bce_dump_rxbd(sc, 0, NULL);
10266                 bce_dump_tx_mbuf_chain(sc, 0, USABLE_TX_BD);
10267                 bce_dump_rx_mbuf_chain(sc, 0, USABLE_RX_BD);
10268                 bce_dump_l2fhdr(sc, 0, NULL);
10269                 bce_dump_ctx(sc, RX_CID);
10270                 bce_dump_ftqs(sc);
10271                 bce_dump_tx_chain(sc, 0, USABLE_TX_BD);
10272                 bce_dump_rx_chain(sc, 0, USABLE_RX_BD);
10273                 bce_dump_status_block(sc);
10274                 bce_dump_stats_block(sc);
10275                 bce_dump_driver_state(sc);
10276                 bce_dump_hw_state(sc);
10277                 bce_dump_bc_state(sc);
10278                 bce_dump_txp_state(sc, 0);
10279                 bce_dump_rxp_state(sc, 0);
10280                 bce_dump_tpat_state(sc, 0);
10281                 bce_dump_cp_state(sc, 0);
10282                 bce_dump_com_state(sc, 0);
10283 #ifdef ZERO_COPY_SOCKETS
10284                 bce_dump_pgbd(sc, 0, NULL);
10285                 bce_dump_pg_mbuf_chain(sc, 0, USABLE_PG_BD);
10286                 bce_dump_pg_chain(sc, 0, USABLE_PG_BD);
10287 #endif
10288         }
10289
10290         bce_dump_status_block(sc);
10291         bce_dump_driver_state(sc);
10292
10293         /* Call the debugger. */
10294         breakpoint();
10295
10296         return;
10297 }
10298 #endif
10299