sys/dev/bge/if_bge.c

   1 /*-
   2  * Copyright (c) 2001 Wind River Systems
   3  * Copyright (c) 1997, 1998, 1999, 2001
   4  *      Bill Paul <wpaul@windriver.com>.  All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  * 3. All advertising materials mentioning features or use of this software
  15  *    must display the following acknowledgement:
  16  *      This product includes software developed by Bill Paul.
  17  * 4. Neither the name of the author nor the names of any co-contributors
  18  *    may be used to endorse or promote products derived from this software
  19  *    without specific prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND
  22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  24  * ARE DISCLAIMED.  IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD
  25  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  26  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  27  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  28  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  29  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  30  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  31  * THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/cdefs.h>
  35 __FBSDID("$FreeBSD$");
  36
  37 /*
  38  * Broadcom BCM57xx(x)/BCM590x NetXtreme and NetLink family Ethernet driver
  39  *
  40  * The Broadcom BCM5700 is based on technology originally developed by
  41  * Alteon Networks as part of the Tigon I and Tigon II Gigabit Ethernet
  42  * MAC chips. The BCM5700, sometimes referred to as the Tigon III, has
  43  * two on-board MIPS R4000 CPUs and can have as much as 16MB of external
  44  * SSRAM. The BCM5700 supports TCP, UDP and IP checksum offload, jumbo
  45  * frames, highly configurable RX filtering, and 16 RX and TX queues
  46  * (which, along with RX filter rules, can be used for QOS applications).
  47  * Other features, such as TCP segmentation, may be available as part
  48  * of value-added firmware updates. Unlike the Tigon I and Tigon II,
  49  * firmware images can be stored in hardware and need not be compiled
  50  * into the driver.
  51  *
  52  * The BCM5700 supports the PCI v2.2 and PCI-X v1.0 standards, and will
  53  * function in a 32-bit/64-bit 33/66Mhz bus, or a 64-bit/133Mhz bus.
  54  *
  55  * The BCM5701 is a single-chip solution incorporating both the BCM5700
  56  * MAC and a BCM5401 10/100/1000 PHY. Unlike the BCM5700, the BCM5701
  57  * does not support external SSRAM.
  58  *
  59  * Broadcom also produces a variation of the BCM5700 under the "Altima"
  60  * brand name, which is functionally similar but lacks PCI-X support.
  61  *
  62  * Without external SSRAM, you can only have at most 4 TX rings,
  63  * and the use of the mini RX ring is disabled. This seems to imply
  64  * that these features are simply not available on the BCM5701. As a
  65  * result, this driver does not implement any support for the mini RX
  66  * ring.
  67  */
  68
  69 #ifdef HAVE_KERNEL_OPTION_HEADERS
  70 #include "opt_device_polling.h"
  71 #endif
  72
  73 #include <sys/param.h>
  74 #include <sys/endian.h>
  75 #include <sys/systm.h>
  76 #include <sys/sockio.h>
  77 #include <sys/mbuf.h>
  78 #include <sys/malloc.h>
  79 #include <sys/kernel.h>
  80 #include <sys/module.h>
  81 #include <sys/socket.h>
  82 #include <sys/sysctl.h>
  83 #include <sys/taskqueue.h>
  84
  85 #include <net/if.h>
  86 #include <net/if_arp.h>
  87 #include <net/ethernet.h>
  88 #include <net/if_dl.h>
  89 #include <net/if_media.h>
  90
  91 #include <net/bpf.h>
  92
  93 #include <net/if_types.h>
  94 #include <net/if_vlan_var.h>
  95
  96 #include <netinet/in_systm.h>
  97 #include <netinet/in.h>
  98 #include <netinet/ip.h>
  99 #include <netinet/tcp.h>
 100
 101 #include <machine/bus.h>
 102 #include <machine/resource.h>
 103 #include <sys/bus.h>
 104 #include <sys/rman.h>
 105
 106 #include <dev/mii/mii.h>
 107 #include <dev/mii/miivar.h>
 108 #include "miidevs.h"
 109 #include <dev/mii/brgphyreg.h>
 110
 111 #ifdef __sparc64__
 112 #include <dev/ofw/ofw_bus.h>
 113 #include <dev/ofw/openfirm.h>
 114 #include <machine/ofw_machdep.h>
 115 #include <machine/ver.h>
 116 #endif
 117
 118 #include <dev/pci/pcireg.h>
 119 #include <dev/pci/pcivar.h>
 120
 121 #include <dev/bge/if_bgereg.h>
 122
 123 #define BGE_CSUM_FEATURES       (CSUM_IP | CSUM_TCP)
 124 #define ETHER_MIN_NOPAD         (ETHER_MIN_LEN - ETHER_CRC_LEN) /* i.e., 60 */
 125
 126 MODULE_DEPEND(bge, pci, 1, 1, 1);
 127 MODULE_DEPEND(bge, ether, 1, 1, 1);
 128 MODULE_DEPEND(bge, miibus, 1, 1, 1);
 129
 130 /* "device miibus" required.  See GENERIC if you get errors here. */
 131 #include "miibus_if.h"
 132
 133 /*
 134  * Various supported device vendors/types and their names. Note: the
 135  * spec seems to indicate that the hardware still has Alteon's vendor
 136  * ID burned into it, though it will always be overriden by the vendor
 137  * ID in the EEPROM. Just to be safe, we cover all possibilities.
 138  */
 139 static const struct bge_type {
 140         uint16_t        bge_vid;
 141         uint16_t        bge_did;
 142 } bge_devs[] = {
 143         { ALTEON_VENDORID,      ALTEON_DEVICEID_BCM5700 },
 144         { ALTEON_VENDORID,      ALTEON_DEVICEID_BCM5701 },
 145
 146         { ALTIMA_VENDORID,      ALTIMA_DEVICE_AC1000 },
 147         { ALTIMA_VENDORID,      ALTIMA_DEVICE_AC1002 },
 148         { ALTIMA_VENDORID,      ALTIMA_DEVICE_AC9100 },
 149
 150         { APPLE_VENDORID,       APPLE_DEVICE_BCM5701 },
 151
 152         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5700 },
 153         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5701 },
 154         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5702 },
 155         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5702_ALT },
 156         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5702X },
 157         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5703 },
 158         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5703_ALT },
 159         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5703X },
 160         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5704C },
 161         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5704S },
 162         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5704S_ALT },
 163         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5705 },
 164         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5705F },
 165         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5705K },
 166         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5705M },
 167         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5705M_ALT },
 168         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5714C },
 169         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5714S },
 170         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5715 },
 171         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5715S },
 172         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5717 },
 173         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5718 },
 174         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5719 },
 175         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5720 },
 176         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5721 },
 177         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5722 },
 178         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5723 },
 179         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5750 },
 180         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5750M },
 181         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5751 },
 182         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5751F },
 183         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5751M },
 184         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5752 },
 185         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5752M },
 186         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5753 },
 187         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5753F },
 188         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5753M },
 189         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5754 },
 190         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5754M },
 191         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5755 },
 192         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5755M },
 193         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5756 },
 194         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5761 },
 195         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5761E },
 196         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5761S },
 197         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5761SE },
 198         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5764 },
 199         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5780 },
 200         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5780S },
 201         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5781 },
 202         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5782 },
 203         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5784 },
 204         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5785F },
 205         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5785G },
 206         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5786 },
 207         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5787 },
 208         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5787F },
 209         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5787M },
 210         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5788 },
 211         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5789 },
 212         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5901 },
 213         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5901A2 },
 214         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5903M },
 215         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5906 },
 216         { BCOM_VENDORID,        BCOM_DEVICEID_BCM5906M },
 217         { BCOM_VENDORID,        BCOM_DEVICEID_BCM57760 },
 218         { BCOM_VENDORID,        BCOM_DEVICEID_BCM57761 },
 219         { BCOM_VENDORID,        BCOM_DEVICEID_BCM57762 },
 220         { BCOM_VENDORID,        BCOM_DEVICEID_BCM57765 },
 221         { BCOM_VENDORID,        BCOM_DEVICEID_BCM57766 },
 222         { BCOM_VENDORID,        BCOM_DEVICEID_BCM57780 },
 223         { BCOM_VENDORID,        BCOM_DEVICEID_BCM57781 },
 224         { BCOM_VENDORID,        BCOM_DEVICEID_BCM57785 },
 225         { BCOM_VENDORID,        BCOM_DEVICEID_BCM57788 },
 226         { BCOM_VENDORID,        BCOM_DEVICEID_BCM57790 },
 227         { BCOM_VENDORID,        BCOM_DEVICEID_BCM57791 },
 228         { BCOM_VENDORID,        BCOM_DEVICEID_BCM57795 },
 229
 230         { SK_VENDORID,          SK_DEVICEID_ALTIMA },
 231
 232         { TC_VENDORID,          TC_DEVICEID_3C996 },
 233
 234         { FJTSU_VENDORID,       FJTSU_DEVICEID_PW008GE4 },
 235         { FJTSU_VENDORID,       FJTSU_DEVICEID_PW008GE5 },
 236         { FJTSU_VENDORID,       FJTSU_DEVICEID_PP250450 },
 237
 238         { 0, 0 }
 239 };
 240
 241 static const struct bge_vendor {
 242         uint16_t        v_id;
 243         const char      *v_name;
 244 } bge_vendors[] = {
 245         { ALTEON_VENDORID,      "Alteon" },
 246         { ALTIMA_VENDORID,      "Altima" },
 247         { APPLE_VENDORID,       "Apple" },
 248         { BCOM_VENDORID,        "Broadcom" },
 249         { SK_VENDORID,          "SysKonnect" },
 250         { TC_VENDORID,          "3Com" },
 251         { FJTSU_VENDORID,       "Fujitsu" },
 252
 253         { 0, NULL }
 254 };
 255
 256 static const struct bge_revision {
 257         uint32_t        br_chipid;
 258         const char      *br_name;
 259 } bge_revisions[] = {
 260         { BGE_CHIPID_BCM5700_A0,        "BCM5700 A0" },
 261         { BGE_CHIPID_BCM5700_A1,        "BCM5700 A1" },
 262         { BGE_CHIPID_BCM5700_B0,        "BCM5700 B0" },
 263         { BGE_CHIPID_BCM5700_B1,        "BCM5700 B1" },
 264         { BGE_CHIPID_BCM5700_B2,        "BCM5700 B2" },
 265         { BGE_CHIPID_BCM5700_B3,        "BCM5700 B3" },
 266         { BGE_CHIPID_BCM5700_ALTIMA,    "BCM5700 Altima" },
 267         { BGE_CHIPID_BCM5700_C0,        "BCM5700 C0" },
 268         { BGE_CHIPID_BCM5701_A0,        "BCM5701 A0" },
 269         { BGE_CHIPID_BCM5701_B0,        "BCM5701 B0" },
 270         { BGE_CHIPID_BCM5701_B2,        "BCM5701 B2" },
 271         { BGE_CHIPID_BCM5701_B5,        "BCM5701 B5" },
 272         { BGE_CHIPID_BCM5703_A0,        "BCM5703 A0" },
 273         { BGE_CHIPID_BCM5703_A1,        "BCM5703 A1" },
 274         { BGE_CHIPID_BCM5703_A2,        "BCM5703 A2" },
 275         { BGE_CHIPID_BCM5703_A3,        "BCM5703 A3" },
 276         { BGE_CHIPID_BCM5703_B0,        "BCM5703 B0" },
 277         { BGE_CHIPID_BCM5704_A0,        "BCM5704 A0" },
 278         { BGE_CHIPID_BCM5704_A1,        "BCM5704 A1" },
 279         { BGE_CHIPID_BCM5704_A2,        "BCM5704 A2" },
 280         { BGE_CHIPID_BCM5704_A3,        "BCM5704 A3" },
 281         { BGE_CHIPID_BCM5704_B0,        "BCM5704 B0" },
 282         { BGE_CHIPID_BCM5705_A0,        "BCM5705 A0" },
 283         { BGE_CHIPID_BCM5705_A1,        "BCM5705 A1" },
 284         { BGE_CHIPID_BCM5705_A2,        "BCM5705 A2" },
 285         { BGE_CHIPID_BCM5705_A3,        "BCM5705 A3" },
 286         { BGE_CHIPID_BCM5750_A0,        "BCM5750 A0" },
 287         { BGE_CHIPID_BCM5750_A1,        "BCM5750 A1" },
 288         { BGE_CHIPID_BCM5750_A3,        "BCM5750 A3" },
 289         { BGE_CHIPID_BCM5750_B0,        "BCM5750 B0" },
 290         { BGE_CHIPID_BCM5750_B1,        "BCM5750 B1" },
 291         { BGE_CHIPID_BCM5750_C0,        "BCM5750 C0" },
 292         { BGE_CHIPID_BCM5750_C1,        "BCM5750 C1" },
 293         { BGE_CHIPID_BCM5750_C2,        "BCM5750 C2" },
 294         { BGE_CHIPID_BCM5714_A0,        "BCM5714 A0" },
 295         { BGE_CHIPID_BCM5752_A0,        "BCM5752 A0" },
 296         { BGE_CHIPID_BCM5752_A1,        "BCM5752 A1" },
 297         { BGE_CHIPID_BCM5752_A2,        "BCM5752 A2" },
 298         { BGE_CHIPID_BCM5714_B0,        "BCM5714 B0" },
 299         { BGE_CHIPID_BCM5714_B3,        "BCM5714 B3" },
 300         { BGE_CHIPID_BCM5715_A0,        "BCM5715 A0" },
 301         { BGE_CHIPID_BCM5715_A1,        "BCM5715 A1" },
 302         { BGE_CHIPID_BCM5715_A3,        "BCM5715 A3" },
 303         { BGE_CHIPID_BCM5717_A0,        "BCM5717 A0" },
 304         { BGE_CHIPID_BCM5717_B0,        "BCM5717 B0" },
 305         { BGE_CHIPID_BCM5719_A0,        "BCM5719 A0" },
 306         { BGE_CHIPID_BCM5720_A0,        "BCM5720 A0" },
 307         { BGE_CHIPID_BCM5755_A0,        "BCM5755 A0" },
 308         { BGE_CHIPID_BCM5755_A1,        "BCM5755 A1" },
 309         { BGE_CHIPID_BCM5755_A2,        "BCM5755 A2" },
 310         { BGE_CHIPID_BCM5722_A0,        "BCM5722 A0" },
 311         { BGE_CHIPID_BCM5761_A0,        "BCM5761 A0" },
 312         { BGE_CHIPID_BCM5761_A1,        "BCM5761 A1" },
 313         { BGE_CHIPID_BCM5784_A0,        "BCM5784 A0" },
 314         { BGE_CHIPID_BCM5784_A1,        "BCM5784 A1" },
 315         /* 5754 and 5787 share the same ASIC ID */
 316         { BGE_CHIPID_BCM5787_A0,        "BCM5754/5787 A0" },
 317         { BGE_CHIPID_BCM5787_A1,        "BCM5754/5787 A1" },
 318         { BGE_CHIPID_BCM5787_A2,        "BCM5754/5787 A2" },
 319         { BGE_CHIPID_BCM5906_A1,        "BCM5906 A1" },
 320         { BGE_CHIPID_BCM5906_A2,        "BCM5906 A2" },
 321         { BGE_CHIPID_BCM57765_A0,       "BCM57765 A0" },
 322         { BGE_CHIPID_BCM57765_B0,       "BCM57765 B0" },
 323         { BGE_CHIPID_BCM57780_A0,       "BCM57780 A0" },
 324         { BGE_CHIPID_BCM57780_A1,       "BCM57780 A1" },
 325
 326         { 0, NULL }
 327 };
 328
 329 /*
 330  * Some defaults for major revisions, so that newer steppings
 331  * that we don't know about have a shot at working.
 332  */
 333 static const struct bge_revision bge_majorrevs[] = {
 334         { BGE_ASICREV_BCM5700,          "unknown BCM5700" },
 335         { BGE_ASICREV_BCM5701,          "unknown BCM5701" },
 336         { BGE_ASICREV_BCM5703,          "unknown BCM5703" },
 337         { BGE_ASICREV_BCM5704,          "unknown BCM5704" },
 338         { BGE_ASICREV_BCM5705,          "unknown BCM5705" },
 339         { BGE_ASICREV_BCM5750,          "unknown BCM5750" },
 340         { BGE_ASICREV_BCM5714_A0,       "unknown BCM5714" },
 341         { BGE_ASICREV_BCM5752,          "unknown BCM5752" },
 342         { BGE_ASICREV_BCM5780,          "unknown BCM5780" },
 343         { BGE_ASICREV_BCM5714,          "unknown BCM5714" },
 344         { BGE_ASICREV_BCM5755,          "unknown BCM5755" },
 345         { BGE_ASICREV_BCM5761,          "unknown BCM5761" },
 346         { BGE_ASICREV_BCM5784,          "unknown BCM5784" },
 347         { BGE_ASICREV_BCM5785,          "unknown BCM5785" },
 348         /* 5754 and 5787 share the same ASIC ID */
 349         { BGE_ASICREV_BCM5787,          "unknown BCM5754/5787" },
 350         { BGE_ASICREV_BCM5906,          "unknown BCM5906" },
 351         { BGE_ASICREV_BCM57765,         "unknown BCM57765" },
 352         { BGE_ASICREV_BCM57766,         "unknown BCM57766" },
 353         { BGE_ASICREV_BCM57780,         "unknown BCM57780" },
 354         { BGE_ASICREV_BCM5717,          "unknown BCM5717" },
 355         { BGE_ASICREV_BCM5719,          "unknown BCM5719" },
 356         { BGE_ASICREV_BCM5720,          "unknown BCM5720" },
 357
 358         { 0, NULL }
 359 };
 360
 361 #define BGE_IS_JUMBO_CAPABLE(sc)        ((sc)->bge_flags & BGE_FLAG_JUMBO)
 362 #define BGE_IS_5700_FAMILY(sc)          ((sc)->bge_flags & BGE_FLAG_5700_FAMILY)
 363 #define BGE_IS_5705_PLUS(sc)            ((sc)->bge_flags & BGE_FLAG_5705_PLUS)
 364 #define BGE_IS_5714_FAMILY(sc)          ((sc)->bge_flags & BGE_FLAG_5714_FAMILY)
 365 #define BGE_IS_575X_PLUS(sc)            ((sc)->bge_flags & BGE_FLAG_575X_PLUS)
 366 #define BGE_IS_5755_PLUS(sc)            ((sc)->bge_flags & BGE_FLAG_5755_PLUS)
 367 #define BGE_IS_5717_PLUS(sc)            ((sc)->bge_flags & BGE_FLAG_5717_PLUS)
 368 #define BGE_IS_57765_PLUS(sc)           ((sc)->bge_flags & BGE_FLAG_57765_PLUS)
 369
 370 static uint32_t bge_chipid(device_t);
 371 static const struct bge_vendor * bge_lookup_vendor(uint16_t);
 372 static const struct bge_revision * bge_lookup_rev(uint32_t);
 373
 374 typedef int     (*bge_eaddr_fcn_t)(struct bge_softc *, uint8_t[]);
 375
 376 static int bge_probe(device_t);
 377 static int bge_attach(device_t);
 378 static int bge_detach(device_t);
 379 static int bge_suspend(device_t);
 380 static int bge_resume(device_t);
 381 static void bge_release_resources(struct bge_softc *);
 382 static void bge_dma_map_addr(void *, bus_dma_segment_t *, int, int);
 383 static int bge_dma_alloc(struct bge_softc *);
 384 static void bge_dma_free(struct bge_softc *);
 385 static int bge_dma_ring_alloc(struct bge_softc *, bus_size_t, bus_size_t,
 386     bus_dma_tag_t *, uint8_t **, bus_dmamap_t *, bus_addr_t *, const char *);
 387
 388 static void bge_devinfo(struct bge_softc *);
 389 static int bge_mbox_reorder(struct bge_softc *);
 390
 391 static int bge_get_eaddr_fw(struct bge_softc *sc, uint8_t ether_addr[]);
 392 static int bge_get_eaddr_mem(struct bge_softc *, uint8_t[]);
 393 static int bge_get_eaddr_nvram(struct bge_softc *, uint8_t[]);
 394 static int bge_get_eaddr_eeprom(struct bge_softc *, uint8_t[]);
 395 static int bge_get_eaddr(struct bge_softc *, uint8_t[]);
 396
 397 static void bge_txeof(struct bge_softc *, uint16_t);
 398 static void bge_rxcsum(struct bge_softc *, struct bge_rx_bd *, struct mbuf *);
 399 static int bge_rxeof(struct bge_softc *, uint16_t, int);
 400
 401 static void bge_asf_driver_up (struct bge_softc *);
 402 static void bge_tick(void *);
 403 static void bge_stats_clear_regs(struct bge_softc *);
 404 static void bge_stats_update(struct bge_softc *);
 405 static void bge_stats_update_regs(struct bge_softc *);
 406 static struct mbuf *bge_check_short_dma(struct mbuf *);
 407 static struct mbuf *bge_setup_tso(struct bge_softc *, struct mbuf *,
 408     uint16_t *, uint16_t *);
 409 static int bge_encap(struct bge_softc *, struct mbuf **, uint32_t *);
 410
 411 static void bge_intr(void *);
 412 static int bge_msi_intr(void *);
 413 static void bge_intr_task(void *, int);
 414 static void bge_start_locked(struct ifnet *);
 415 static void bge_start(struct ifnet *);
 416 static int bge_ioctl(struct ifnet *, u_long, caddr_t);
 417 static void bge_init_locked(struct bge_softc *);
 418 static void bge_init(void *);
 419 static void bge_stop_block(struct bge_softc *, bus_size_t, uint32_t);
 420 static void bge_stop(struct bge_softc *);
 421 static void bge_watchdog(struct bge_softc *);
 422 static int bge_shutdown(device_t);
 423 static int bge_ifmedia_upd_locked(struct ifnet *);
 424 static int bge_ifmedia_upd(struct ifnet *);
 425 static void bge_ifmedia_sts(struct ifnet *, struct ifmediareq *);
 426
 427 static uint8_t bge_nvram_getbyte(struct bge_softc *, int, uint8_t *);
 428 static int bge_read_nvram(struct bge_softc *, caddr_t, int, int);
 429
 430 static uint8_t bge_eeprom_getbyte(struct bge_softc *, int, uint8_t *);
 431 static int bge_read_eeprom(struct bge_softc *, caddr_t, int, int);
 432
 433 static void bge_setpromisc(struct bge_softc *);
 434 static void bge_setmulti(struct bge_softc *);
 435 static void bge_setvlan(struct bge_softc *);
 436
 437 static __inline void bge_rxreuse_std(struct bge_softc *, int);
 438 static __inline void bge_rxreuse_jumbo(struct bge_softc *, int);
 439 static int bge_newbuf_std(struct bge_softc *, int);
 440 static int bge_newbuf_jumbo(struct bge_softc *, int);
 441 static int bge_init_rx_ring_std(struct bge_softc *);
 442 static void bge_free_rx_ring_std(struct bge_softc *);
 443 static int bge_init_rx_ring_jumbo(struct bge_softc *);
 444 static void bge_free_rx_ring_jumbo(struct bge_softc *);
 445 static void bge_free_tx_ring(struct bge_softc *);
 446 static int bge_init_tx_ring(struct bge_softc *);
 447
 448 static int bge_chipinit(struct bge_softc *);
 449 static int bge_blockinit(struct bge_softc *);
 450 static uint32_t bge_dma_swap_options(struct bge_softc *);
 451
 452 static int bge_has_eaddr(struct bge_softc *);
 453 static uint32_t bge_readmem_ind(struct bge_softc *, int);
 454 static void bge_writemem_ind(struct bge_softc *, int, int);
 455 static void bge_writembx(struct bge_softc *, int, int);
 456 #ifdef notdef
 457 static uint32_t bge_readreg_ind(struct bge_softc *, int);
 458 #endif
 459 static void bge_writemem_direct(struct bge_softc *, int, int);
 460 static void bge_writereg_ind(struct bge_softc *, int, int);
 461
 462 static int bge_miibus_readreg(device_t, int, int);
 463 static int bge_miibus_writereg(device_t, int, int, int);
 464 static void bge_miibus_statchg(device_t);
 465 #ifdef DEVICE_POLLING
 466 static int bge_poll(struct ifnet *ifp, enum poll_cmd cmd, int count);
 467 #endif
 468
 469 #define BGE_RESET_SHUTDOWN      0
 470 #define BGE_RESET_START         1
 471 #define BGE_RESET_SUSPEND       2
 472 static void bge_sig_post_reset(struct bge_softc *, int);
 473 static void bge_sig_legacy(struct bge_softc *, int);
 474 static void bge_sig_pre_reset(struct bge_softc *, int);
 475 static void bge_stop_fw(struct bge_softc *);
 476 static int bge_reset(struct bge_softc *);
 477 static void bge_link_upd(struct bge_softc *);
 478
 479 static void bge_ape_lock_init(struct bge_softc *);
 480 static void bge_ape_read_fw_ver(struct bge_softc *);
 481 static int bge_ape_lock(struct bge_softc *, int);
 482 static void bge_ape_unlock(struct bge_softc *, int);
 483 static void bge_ape_send_event(struct bge_softc *, uint32_t);
 484 static void bge_ape_driver_state_change(struct bge_softc *, int);
 485
 486 /*
 487  * The BGE_REGISTER_DEBUG option is only for low-level debugging.  It may
 488  * leak information to untrusted users.  It is also known to cause alignment
 489  * traps on certain architectures.
 490  */
 491 #ifdef BGE_REGISTER_DEBUG
 492 static int bge_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
 493 static int bge_sysctl_reg_read(SYSCTL_HANDLER_ARGS);
 494 static int bge_sysctl_ape_read(SYSCTL_HANDLER_ARGS);
 495 static int bge_sysctl_mem_read(SYSCTL_HANDLER_ARGS);
 496 #endif
 497 static void bge_add_sysctls(struct bge_softc *);
 498 static void bge_add_sysctl_stats_regs(struct bge_softc *,
 499     struct sysctl_ctx_list *, struct sysctl_oid_list *);
 500 static void bge_add_sysctl_stats(struct bge_softc *, struct sysctl_ctx_list *,
 501     struct sysctl_oid_list *);
 502 static int bge_sysctl_stats(SYSCTL_HANDLER_ARGS);
 503
 504 static device_method_t bge_methods[] = {
 505         /* Device interface */
 506         DEVMETHOD(device_probe,         bge_probe),
 507         DEVMETHOD(device_attach,        bge_attach),
 508         DEVMETHOD(device_detach,        bge_detach),
 509         DEVMETHOD(device_shutdown,      bge_shutdown),
 510         DEVMETHOD(device_suspend,       bge_suspend),
 511         DEVMETHOD(device_resume,        bge_resume),
 512
 513         /* MII interface */
 514         DEVMETHOD(miibus_readreg,       bge_miibus_readreg),
 515         DEVMETHOD(miibus_writereg,      bge_miibus_writereg),
 516         DEVMETHOD(miibus_statchg,       bge_miibus_statchg),
 517
 518         DEVMETHOD_END
 519 };
 520
 521 static driver_t bge_driver = {
 522         "bge",
 523         bge_methods,
 524         sizeof(struct bge_softc)
 525 };
 526
 527 static devclass_t bge_devclass;
 528
 529 DRIVER_MODULE(bge, pci, bge_driver, bge_devclass, 0, 0);
 530 DRIVER_MODULE(miibus, bge, miibus_driver, miibus_devclass, 0, 0);
 531
 532 static int bge_allow_asf = 1;
 533
 534 TUNABLE_INT("hw.bge.allow_asf", &bge_allow_asf);
 535
 536 static SYSCTL_NODE(_hw, OID_AUTO, bge, CTLFLAG_RD, 0, "BGE driver parameters");
 537 SYSCTL_INT(_hw_bge, OID_AUTO, allow_asf, CTLFLAG_RD, &bge_allow_asf, 0,
 538         "Allow ASF mode if available");
 539
 540 #define SPARC64_BLADE_1500_MODEL        "SUNW,Sun-Blade-1500"
 541 #define SPARC64_BLADE_1500_PATH_BGE     "/pci@1f,700000/network@2"
 542 #define SPARC64_BLADE_2500_MODEL        "SUNW,Sun-Blade-2500"
 543 #define SPARC64_BLADE_2500_PATH_BGE     "/pci@1c,600000/network@3"
 544 #define SPARC64_OFW_SUBVENDOR           "subsystem-vendor-id"
 545
 546 static int
 547 bge_has_eaddr(struct bge_softc *sc)
 548 {
 549 #ifdef __sparc64__
 550         char buf[sizeof(SPARC64_BLADE_1500_PATH_BGE)];
 551         device_t dev;
 552         uint32_t subvendor;
 553
 554         dev = sc->bge_dev;
 555
 556         /*
 557          * The on-board BGEs found in sun4u machines aren't fitted with
 558          * an EEPROM which means that we have to obtain the MAC address
 559          * via OFW and that some tests will always fail.  We distinguish
 560          * such BGEs by the subvendor ID, which also has to be obtained
 561          * from OFW instead of the PCI configuration space as the latter
 562          * indicates Broadcom as the subvendor of the netboot interface.
 563          * For early Blade 1500 and 2500 we even have to check the OFW
 564          * device path as the subvendor ID always defaults to Broadcom
 565          * there.
 566          */
 567         if (OF_getprop(ofw_bus_get_node(dev), SPARC64_OFW_SUBVENDOR,
 568             &subvendor, sizeof(subvendor)) == sizeof(subvendor) &&
 569             (subvendor == FJTSU_VENDORID || subvendor == SUN_VENDORID))
 570                 return (0);
 571         memset(buf, 0, sizeof(buf));
 572         if (OF_package_to_path(ofw_bus_get_node(dev), buf, sizeof(buf)) > 0) {
 573                 if (strcmp(sparc64_model, SPARC64_BLADE_1500_MODEL) == 0 &&
 574                     strcmp(buf, SPARC64_BLADE_1500_PATH_BGE) == 0)
 575                         return (0);
 576                 if (strcmp(sparc64_model, SPARC64_BLADE_2500_MODEL) == 0 &&
 577                     strcmp(buf, SPARC64_BLADE_2500_PATH_BGE) == 0)
 578                         return (0);
 579         }
 580 #endif
 581         return (1);
 582 }
 583
 584 static uint32_t
 585 bge_readmem_ind(struct bge_softc *sc, int off)
 586 {
 587         device_t dev;
 588         uint32_t val;
 589
 590         if (sc->bge_asicrev == BGE_ASICREV_BCM5906 &&
 591             off >= BGE_STATS_BLOCK && off < BGE_SEND_RING_1_TO_4)
 592                 return (0);
 593
 594         dev = sc->bge_dev;
 595
 596         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, off, 4);
 597         val = pci_read_config(dev, BGE_PCI_MEMWIN_DATA, 4);
 598         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, 0, 4);
 599         return (val);
 600 }
 601
 602 static void
 603 bge_writemem_ind(struct bge_softc *sc, int off, int val)
 604 {
 605         device_t dev;
 606
 607         if (sc->bge_asicrev == BGE_ASICREV_BCM5906 &&
 608             off >= BGE_STATS_BLOCK && off < BGE_SEND_RING_1_TO_4)
 609                 return;
 610
 611         dev = sc->bge_dev;
 612
 613         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, off, 4);
 614         pci_write_config(dev, BGE_PCI_MEMWIN_DATA, val, 4);
 615         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, 0, 4);
 616 }
 617
 618 #ifdef notdef
 619 static uint32_t
 620 bge_readreg_ind(struct bge_softc *sc, int off)
 621 {
 622         device_t dev;
 623
 624         dev = sc->bge_dev;
 625
 626         pci_write_config(dev, BGE_PCI_REG_BASEADDR, off, 4);
 627         return (pci_read_config(dev, BGE_PCI_REG_DATA, 4));
 628 }
 629 #endif
 630
 631 static void
 632 bge_writereg_ind(struct bge_softc *sc, int off, int val)
 633 {
 634         device_t dev;
 635
 636         dev = sc->bge_dev;
 637
 638         pci_write_config(dev, BGE_PCI_REG_BASEADDR, off, 4);
 639         pci_write_config(dev, BGE_PCI_REG_DATA, val, 4);
 640 }
 641
 642 static void
 643 bge_writemem_direct(struct bge_softc *sc, int off, int val)
 644 {
 645         CSR_WRITE_4(sc, off, val);
 646 }
 647
 648 static void
 649 bge_writembx(struct bge_softc *sc, int off, int val)
 650 {
 651         if (sc->bge_asicrev == BGE_ASICREV_BCM5906)
 652                 off += BGE_LPMBX_IRQ0_HI - BGE_MBX_IRQ0_HI;
 653
 654         CSR_WRITE_4(sc, off, val);
 655         if ((sc->bge_flags & BGE_FLAG_MBOX_REORDER) != 0)
 656                 CSR_READ_4(sc, off);
 657 }
 658
 659 /*
 660  * Clear all stale locks and select the lock for this driver instance.
 661  */
 662 static void
 663 bge_ape_lock_init(struct bge_softc *sc)
 664 {
 665         uint32_t bit, regbase;
 666         int i;
 667
 668         if (sc->bge_asicrev == BGE_ASICREV_BCM5761)
 669                 regbase = BGE_APE_LOCK_GRANT;
 670         else
 671                 regbase = BGE_APE_PER_LOCK_GRANT;
 672
 673         /* Clear any stale locks. */
 674         for (i = BGE_APE_LOCK_PHY0; i <= BGE_APE_LOCK_GPIO; i++) {
 675                 switch (i) {
 676                 case BGE_APE_LOCK_PHY0:
 677                 case BGE_APE_LOCK_PHY1:
 678                 case BGE_APE_LOCK_PHY2:
 679                 case BGE_APE_LOCK_PHY3:
 680                         bit = BGE_APE_LOCK_GRANT_DRIVER0;
 681                         break;
 682                 default:
 683                         if (sc->bge_func_addr == 0)
 684                                 bit = BGE_APE_LOCK_GRANT_DRIVER0;
 685                         else
 686                                 bit = (1 << sc->bge_func_addr);
 687                 }
 688                 APE_WRITE_4(sc, regbase + 4 * i, bit);
 689         }
 690
 691         /* Select the PHY lock based on the device's function number. */
 692         switch (sc->bge_func_addr) {
 693         case 0:
 694                 sc->bge_phy_ape_lock = BGE_APE_LOCK_PHY0;
 695                 break;
 696         case 1:
 697                 sc->bge_phy_ape_lock = BGE_APE_LOCK_PHY1;
 698                 break;
 699         case 2:
 700                 sc->bge_phy_ape_lock = BGE_APE_LOCK_PHY2;
 701                 break;
 702         case 3:
 703                 sc->bge_phy_ape_lock = BGE_APE_LOCK_PHY3;
 704                 break;
 705         default:
 706                 device_printf(sc->bge_dev,
 707                     "PHY lock not supported on this function\n");
 708         }
 709 }
 710
 711 /*
 712  * Check for APE firmware, set flags, and print version info.
 713  */
 714 static void
 715 bge_ape_read_fw_ver(struct bge_softc *sc)
 716 {
 717         const char *fwtype;
 718         uint32_t apedata, features;
 719
 720         /* Check for a valid APE signature in shared memory. */
 721         apedata = APE_READ_4(sc, BGE_APE_SEG_SIG);
 722         if (apedata != BGE_APE_SEG_SIG_MAGIC) {
 723                 sc->bge_mfw_flags &= ~ BGE_MFW_ON_APE;
 724                 return;
 725         }
 726
 727         /* Check if APE firmware is running. */
 728         apedata = APE_READ_4(sc, BGE_APE_FW_STATUS);
 729         if ((apedata & BGE_APE_FW_STATUS_READY) == 0) {
 730                 device_printf(sc->bge_dev, "APE signature found "
 731                     "but FW status not ready! 0x%08x\n", apedata);
 732                 return;
 733         }
 734
 735         sc->bge_mfw_flags |= BGE_MFW_ON_APE;
 736
 737         /* Fetch the APE firwmare type and version. */
 738         apedata = APE_READ_4(sc, BGE_APE_FW_VERSION);
 739         features = APE_READ_4(sc, BGE_APE_FW_FEATURES);
 740         if ((features & BGE_APE_FW_FEATURE_NCSI) != 0) {
 741                 sc->bge_mfw_flags |= BGE_MFW_TYPE_NCSI;
 742                 fwtype = "NCSI";
 743         } else if ((features & BGE_APE_FW_FEATURE_DASH) != 0) {
 744                 sc->bge_mfw_flags |= BGE_MFW_TYPE_DASH;
 745                 fwtype = "DASH";
 746         } else
 747                 fwtype = "UNKN";
 748
 749         /* Print the APE firmware version. */
 750         device_printf(sc->bge_dev, "APE FW version: %s v%d.%d.%d.%d\n",
 751             fwtype,
 752             (apedata & BGE_APE_FW_VERSION_MAJMSK) >> BGE_APE_FW_VERSION_MAJSFT,
 753             (apedata & BGE_APE_FW_VERSION_MINMSK) >> BGE_APE_FW_VERSION_MINSFT,
 754             (apedata & BGE_APE_FW_VERSION_REVMSK) >> BGE_APE_FW_VERSION_REVSFT,
 755             (apedata & BGE_APE_FW_VERSION_BLDMSK));
 756 }
 757
 758 static int
 759 bge_ape_lock(struct bge_softc *sc, int locknum)
 760 {
 761         uint32_t bit, gnt, req, status;
 762         int i, off;
 763
 764         if ((sc->bge_mfw_flags & BGE_MFW_ON_APE) == 0)
 765                 return (0);
 766
 767         /* Lock request/grant registers have different bases. */
 768         if (sc->bge_asicrev == BGE_ASICREV_BCM5761) {
 769                 req = BGE_APE_LOCK_REQ;
 770                 gnt = BGE_APE_LOCK_GRANT;
 771         } else {
 772                 req = BGE_APE_PER_LOCK_REQ;
 773                 gnt = BGE_APE_PER_LOCK_GRANT;
 774         }
 775
 776         off = 4 * locknum;
 777
 778         switch (locknum) {
 779         case BGE_APE_LOCK_GPIO:
 780                 /* Lock required when using GPIO. */
 781                 if (sc->bge_asicrev == BGE_ASICREV_BCM5761)
 782                         return (0);
 783                 if (sc->bge_func_addr == 0)
 784                         bit = BGE_APE_LOCK_REQ_DRIVER0;
 785                 else
 786                         bit = (1 << sc->bge_func_addr);
 787                 break;
 788         case BGE_APE_LOCK_GRC:
 789                 /* Lock required to reset the device. */
 790                 if (sc->bge_func_addr == 0)
 791                         bit = BGE_APE_LOCK_REQ_DRIVER0;
 792                 else
 793                         bit = (1 << sc->bge_func_addr);
 794                 break;
 795         case BGE_APE_LOCK_MEM:
 796                 /* Lock required when accessing certain APE memory. */
 797                 if (sc->bge_func_addr == 0)
 798                         bit = BGE_APE_LOCK_REQ_DRIVER0;
 799                 else
 800                         bit = (1 << sc->bge_func_addr);
 801                 break;
 802         case BGE_APE_LOCK_PHY0:
 803         case BGE_APE_LOCK_PHY1:
 804         case BGE_APE_LOCK_PHY2:
 805         case BGE_APE_LOCK_PHY3:
 806                 /* Lock required when accessing PHYs. */
 807                 bit = BGE_APE_LOCK_REQ_DRIVER0;
 808                 break;
 809         default:
 810                 return (EINVAL);
 811         }
 812
 813         /* Request a lock. */
 814         APE_WRITE_4(sc, req + off, bit);
 815
 816         /* Wait up to 1 second to acquire lock. */
 817         for (i = 0; i < 20000; i++) {
 818                 status = APE_READ_4(sc, gnt + off);
 819                 if (status == bit)
 820                         break;
 821                 DELAY(50);
 822         }
 823
 824         /* Handle any errors. */
 825         if (status != bit) {
 826                 device_printf(sc->bge_dev, "APE lock %d request failed! "
 827                     "request = 0x%04x[0x%04x], status = 0x%04x[0x%04x]\n",
 828                     locknum, req + off, bit & 0xFFFF, gnt + off,
 829                     status & 0xFFFF);
 830                 /* Revoke the lock request. */
 831                 APE_WRITE_4(sc, gnt + off, bit);
 832                 return (EBUSY);
 833         }
 834
 835         return (0);
 836 }
 837
 838 static void
 839 bge_ape_unlock(struct bge_softc *sc, int locknum)
 840 {
 841         uint32_t bit, gnt;
 842         int off;
 843
 844         if ((sc->bge_mfw_flags & BGE_MFW_ON_APE) == 0)
 845                 return;
 846
 847         if (sc->bge_asicrev == BGE_ASICREV_BCM5761)
 848                 gnt = BGE_APE_LOCK_GRANT;
 849         else
 850                 gnt = BGE_APE_PER_LOCK_GRANT;
 851
 852         off = 4 * locknum;
 853
 854         switch (locknum) {
 855         case BGE_APE_LOCK_GPIO:
 856                 if (sc->bge_asicrev == BGE_ASICREV_BCM5761)
 857                         return;
 858                 if (sc->bge_func_addr == 0)
 859                         bit = BGE_APE_LOCK_GRANT_DRIVER0;
 860                 else
 861                         bit = (1 << sc->bge_func_addr);
 862                 break;
 863         case BGE_APE_LOCK_GRC:
 864                 if (sc->bge_func_addr == 0)
 865                         bit = BGE_APE_LOCK_GRANT_DRIVER0;
 866                 else
 867                         bit = (1 << sc->bge_func_addr);
 868                 break;
 869         case BGE_APE_LOCK_MEM:
 870                 if (sc->bge_func_addr == 0)
 871                         bit = BGE_APE_LOCK_GRANT_DRIVER0;
 872                 else
 873                         bit = (1 << sc->bge_func_addr);
 874                 break;
 875         case BGE_APE_LOCK_PHY0:
 876         case BGE_APE_LOCK_PHY1:
 877         case BGE_APE_LOCK_PHY2:
 878         case BGE_APE_LOCK_PHY3:
 879                 bit = BGE_APE_LOCK_GRANT_DRIVER0;
 880                 break;
 881         default:
 882                 return;
 883         }
 884
 885         APE_WRITE_4(sc, gnt + off, bit);
 886 }
 887
 888 /*
 889  * Send an event to the APE firmware.
 890  */
 891 static void
 892 bge_ape_send_event(struct bge_softc *sc, uint32_t event)
 893 {
 894         uint32_t apedata;
 895         int i;
 896
 897         /* NCSI does not support APE events. */
 898         if ((sc->bge_mfw_flags & BGE_MFW_ON_APE) == 0)
 899                 return;
 900
 901         /* Wait up to 1ms for APE to service previous event. */
 902         for (i = 10; i > 0; i--) {
 903                 if (bge_ape_lock(sc, BGE_APE_LOCK_MEM) != 0)
 904                         break;
 905                 apedata = APE_READ_4(sc, BGE_APE_EVENT_STATUS);
 906                 if ((apedata & BGE_APE_EVENT_STATUS_EVENT_PENDING) == 0) {
 907                         APE_WRITE_4(sc, BGE_APE_EVENT_STATUS, event |
 908                             BGE_APE_EVENT_STATUS_EVENT_PENDING);
 909                         bge_ape_unlock(sc, BGE_APE_LOCK_MEM);
 910                         APE_WRITE_4(sc, BGE_APE_EVENT, BGE_APE_EVENT_1);
 911                         break;
 912                 }
 913                 bge_ape_unlock(sc, BGE_APE_LOCK_MEM);
 914                 DELAY(100);
 915         }
 916         if (i == 0)
 917                 device_printf(sc->bge_dev, "APE event 0x%08x send timed out\n",
 918                     event);
 919 }
 920
 921 static void
 922 bge_ape_driver_state_change(struct bge_softc *sc, int kind)
 923 {
 924         uint32_t apedata, event;
 925
 926         if ((sc->bge_mfw_flags & BGE_MFW_ON_APE) == 0)
 927                 return;
 928
 929         switch (kind) {
 930         case BGE_RESET_START:
 931                 /* If this is the first load, clear the load counter. */
 932                 apedata = APE_READ_4(sc, BGE_APE_HOST_SEG_SIG);
 933                 if (apedata != BGE_APE_HOST_SEG_SIG_MAGIC)
 934                         APE_WRITE_4(sc, BGE_APE_HOST_INIT_COUNT, 0);
 935                 else {
 936                         apedata = APE_READ_4(sc, BGE_APE_HOST_INIT_COUNT);
 937                         APE_WRITE_4(sc, BGE_APE_HOST_INIT_COUNT, ++apedata);
 938                 }
 939                 APE_WRITE_4(sc, BGE_APE_HOST_SEG_SIG,
 940                     BGE_APE_HOST_SEG_SIG_MAGIC);
 941                 APE_WRITE_4(sc, BGE_APE_HOST_SEG_LEN,
 942                     BGE_APE_HOST_SEG_LEN_MAGIC);
 943
 944                 /* Add some version info if bge(4) supports it. */
 945                 APE_WRITE_4(sc, BGE_APE_HOST_DRIVER_ID,
 946                     BGE_APE_HOST_DRIVER_ID_MAGIC(1, 0));
 947                 APE_WRITE_4(sc, BGE_APE_HOST_BEHAVIOR,
 948                     BGE_APE_HOST_BEHAV_NO_PHYLOCK);
 949                 APE_WRITE_4(sc, BGE_APE_HOST_HEARTBEAT_INT_MS,
 950                     BGE_APE_HOST_HEARTBEAT_INT_DISABLE);
 951                 APE_WRITE_4(sc, BGE_APE_HOST_DRVR_STATE,
 952                     BGE_APE_HOST_DRVR_STATE_START);
 953                 event = BGE_APE_EVENT_STATUS_STATE_START;
 954                 break;
 955         case BGE_RESET_SHUTDOWN:
 956                 APE_WRITE_4(sc, BGE_APE_HOST_DRVR_STATE,
 957                     BGE_APE_HOST_DRVR_STATE_UNLOAD);
 958                 event = BGE_APE_EVENT_STATUS_STATE_UNLOAD;
 959                 break;
 960         case BGE_RESET_SUSPEND:
 961                 event = BGE_APE_EVENT_STATUS_STATE_SUSPEND;
 962                 break;
 963         default:
 964                 return;
 965         }
 966
 967         bge_ape_send_event(sc, event | BGE_APE_EVENT_STATUS_DRIVER_EVNT |
 968             BGE_APE_EVENT_STATUS_STATE_CHNGE);
 969 }
 970
 971 /*
 972  * Map a single buffer address.
 973  */
 974
 975 static void
 976 bge_dma_map_addr(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 977 {
 978         struct bge_dmamap_arg *ctx;
 979
 980         if (error)
 981                 return;
 982
 983         KASSERT(nseg == 1, ("%s: %d segments returned!", __func__, nseg));
 984
 985         ctx = arg;
 986         ctx->bge_busaddr = segs->ds_addr;
 987 }
 988
 989 static uint8_t
 990 bge_nvram_getbyte(struct bge_softc *sc, int addr, uint8_t *dest)
 991 {
 992         uint32_t access, byte = 0;
 993         int i;
 994
 995         /* Lock. */
 996         CSR_WRITE_4(sc, BGE_NVRAM_SWARB, BGE_NVRAMSWARB_SET1);
 997         for (i = 0; i < 8000; i++) {
 998                 if (CSR_READ_4(sc, BGE_NVRAM_SWARB) & BGE_NVRAMSWARB_GNT1)
 999                         break;
1000                 DELAY(20);
1001         }
1002         if (i == 8000)
1003                 return (1);
1004
1005         /* Enable access. */
1006         access = CSR_READ_4(sc, BGE_NVRAM_ACCESS);
1007         CSR_WRITE_4(sc, BGE_NVRAM_ACCESS, access | BGE_NVRAMACC_ENABLE);
1008
1009         CSR_WRITE_4(sc, BGE_NVRAM_ADDR, addr & 0xfffffffc);
1010         CSR_WRITE_4(sc, BGE_NVRAM_CMD, BGE_NVRAM_READCMD);
1011         for (i = 0; i < BGE_TIMEOUT * 10; i++) {
1012                 DELAY(10);
1013                 if (CSR_READ_4(sc, BGE_NVRAM_CMD) & BGE_NVRAMCMD_DONE) {
1014                         DELAY(10);
1015                         break;
1016                 }
1017         }
1018
1019         if (i == BGE_TIMEOUT * 10) {
1020                 if_printf(sc->bge_ifp, "nvram read timed out\n");
1021                 return (1);
1022         }
1023
1024         /* Get result. */
1025         byte = CSR_READ_4(sc, BGE_NVRAM_RDDATA);
1026
1027         *dest = (bswap32(byte) >> ((addr % 4) * 8)) & 0xFF;
1028
1029         /* Disable access. */
1030         CSR_WRITE_4(sc, BGE_NVRAM_ACCESS, access);
1031
1032         /* Unlock. */
1033         CSR_WRITE_4(sc, BGE_NVRAM_SWARB, BGE_NVRAMSWARB_CLR1);
1034         CSR_READ_4(sc, BGE_NVRAM_SWARB);
1035
1036         return (0);
1037 }
1038
1039 /*
1040  * Read a sequence of bytes from NVRAM.
1041  */
1042 static int
1043 bge_read_nvram(struct bge_softc *sc, caddr_t dest, int off, int cnt)
1044 {
1045         int err = 0, i;
1046         uint8_t byte = 0;
1047
1048         if (sc->bge_asicrev != BGE_ASICREV_BCM5906)
1049                 return (1);
1050
1051         for (i = 0; i < cnt; i++) {
1052                 err = bge_nvram_getbyte(sc, off + i, &byte);
1053                 if (err)
1054                         break;
1055                 *(dest + i) = byte;
1056         }
1057
1058         return (err ? 1 : 0);
1059 }
1060
1061 /*
1062  * Read a byte of data stored in the EEPROM at address 'addr.' The
1063  * BCM570x supports both the traditional bitbang interface and an
1064  * auto access interface for reading the EEPROM. We use the auto
1065  * access method.
1066  */
1067 static uint8_t
1068 bge_eeprom_getbyte(struct bge_softc *sc, int addr, uint8_t *dest)
1069 {
1070         int i;
1071         uint32_t byte = 0;
1072
1073         /*
1074          * Enable use of auto EEPROM access so we can avoid
1075          * having to use the bitbang method.
1076          */
1077         BGE_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_AUTO_EEPROM);
1078
1079         /* Reset the EEPROM, load the clock period. */
1080         CSR_WRITE_4(sc, BGE_EE_ADDR,
1081             BGE_EEADDR_RESET | BGE_EEHALFCLK(BGE_HALFCLK_384SCL));
1082         DELAY(20);
1083
1084         /* Issue the read EEPROM command. */
1085         CSR_WRITE_4(sc, BGE_EE_ADDR, BGE_EE_READCMD | addr);
1086
1087         /* Wait for completion */
1088         for(i = 0; i < BGE_TIMEOUT * 10; i++) {
1089                 DELAY(10);
1090                 if (CSR_READ_4(sc, BGE_EE_ADDR) & BGE_EEADDR_DONE)
1091                         break;
1092         }
1093
1094         if (i == BGE_TIMEOUT * 10) {
1095                 device_printf(sc->bge_dev, "EEPROM read timed out\n");
1096                 return (1);
1097         }
1098
1099         /* Get result. */
1100         byte = CSR_READ_4(sc, BGE_EE_DATA);
1101
1102         *dest = (byte >> ((addr % 4) * 8)) & 0xFF;
1103
1104         return (0);
1105 }
1106
1107 /*
1108  * Read a sequence of bytes from the EEPROM.
1109  */
1110 static int
1111 bge_read_eeprom(struct bge_softc *sc, caddr_t dest, int off, int cnt)
1112 {
1113         int i, error = 0;
1114         uint8_t byte = 0;
1115
1116         for (i = 0; i < cnt; i++) {
1117                 error = bge_eeprom_getbyte(sc, off + i, &byte);
1118                 if (error)
1119                         break;
1120                 *(dest + i) = byte;
1121         }
1122
1123         return (error ? 1 : 0);
1124 }
1125
1126 static int
1127 bge_miibus_readreg(device_t dev, int phy, int reg)
1128 {
1129         struct bge_softc *sc;
1130         uint32_t val;
1131         int i;
1132
1133         sc = device_get_softc(dev);
1134
1135         if (bge_ape_lock(sc, sc->bge_phy_ape_lock) != 0)
1136                 return (0);
1137
1138         /* Clear the autopoll bit if set, otherwise may trigger PCI errors. */
1139         if ((sc->bge_mi_mode & BGE_MIMODE_AUTOPOLL) != 0) {
1140                 CSR_WRITE_4(sc, BGE_MI_MODE,
1141                     sc->bge_mi_mode & ~BGE_MIMODE_AUTOPOLL);
1142                 DELAY(80);
1143         }
1144
1145         CSR_WRITE_4(sc, BGE_MI_COMM, BGE_MICMD_READ | BGE_MICOMM_BUSY |
1146             BGE_MIPHY(phy) | BGE_MIREG(reg));
1147
1148         /* Poll for the PHY register access to complete. */
1149         for (i = 0; i < BGE_TIMEOUT; i++) {
1150                 DELAY(10);
1151                 val = CSR_READ_4(sc, BGE_MI_COMM);
1152                 if ((val & BGE_MICOMM_BUSY) == 0) {
1153                         DELAY(5);
1154                         val = CSR_READ_4(sc, BGE_MI_COMM);
1155                         break;
1156                 }
1157         }
1158
1159         if (i == BGE_TIMEOUT) {
1160                 device_printf(sc->bge_dev,
1161                     "PHY read timed out (phy %d, reg %d, val 0x%08x)\n",
1162                     phy, reg, val);
1163                 val = 0;
1164         }
1165
1166         /* Restore the autopoll bit if necessary. */
1167         if ((sc->bge_mi_mode & BGE_MIMODE_AUTOPOLL) != 0) {
1168                 CSR_WRITE_4(sc, BGE_MI_MODE, sc->bge_mi_mode);
1169                 DELAY(80);
1170         }
1171
1172         bge_ape_unlock(sc, sc->bge_phy_ape_lock);
1173
1174         if (val & BGE_MICOMM_READFAIL)
1175                 return (0);
1176
1177         return (val & 0xFFFF);
1178 }
1179
1180 static int
1181 bge_miibus_writereg(device_t dev, int phy, int reg, int val)
1182 {
1183         struct bge_softc *sc;
1184         int i;
1185
1186         sc = device_get_softc(dev);
1187
1188         if (sc->bge_asicrev == BGE_ASICREV_BCM5906 &&
1189             (reg == BRGPHY_MII_1000CTL || reg == BRGPHY_MII_AUXCTL))
1190                 return (0);
1191
1192         if (bge_ape_lock(sc, sc->bge_phy_ape_lock) != 0)
1193                 return (0);
1194
1195         /* Clear the autopoll bit if set, otherwise may trigger PCI errors. */
1196         if ((sc->bge_mi_mode & BGE_MIMODE_AUTOPOLL) != 0) {
1197                 CSR_WRITE_4(sc, BGE_MI_MODE,
1198                     sc->bge_mi_mode & ~BGE_MIMODE_AUTOPOLL);
1199                 DELAY(80);
1200         }
1201
1202         CSR_WRITE_4(sc, BGE_MI_COMM, BGE_MICMD_WRITE | BGE_MICOMM_BUSY |
1203             BGE_MIPHY(phy) | BGE_MIREG(reg) | val);
1204
1205         for (i = 0; i < BGE_TIMEOUT; i++) {
1206                 DELAY(10);
1207                 if (!(CSR_READ_4(sc, BGE_MI_COMM) & BGE_MICOMM_BUSY)) {
1208                         DELAY(5);
1209                         CSR_READ_4(sc, BGE_MI_COMM); /* dummy read */
1210                         break;
1211                 }
1212         }
1213
1214         /* Restore the autopoll bit if necessary. */
1215         if ((sc->bge_mi_mode & BGE_MIMODE_AUTOPOLL) != 0) {
1216                 CSR_WRITE_4(sc, BGE_MI_MODE, sc->bge_mi_mode);
1217                 DELAY(80);
1218         }
1219
1220         bge_ape_unlock(sc, sc->bge_phy_ape_lock);
1221
1222         if (i == BGE_TIMEOUT)
1223                 device_printf(sc->bge_dev,
1224                     "PHY write timed out (phy %d, reg %d, val 0x%04x)\n",
1225                     phy, reg, val);
1226
1227         return (0);
1228 }
1229
1230 static void
1231 bge_miibus_statchg(device_t dev)
1232 {
1233         struct bge_softc *sc;
1234         struct mii_data *mii;
1235         uint32_t mac_mode, rx_mode, tx_mode;
1236
1237         sc = device_get_softc(dev);
1238         if ((sc->bge_ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1239                 return;
1240         mii = device_get_softc(sc->bge_miibus);
1241
1242         if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) ==
1243             (IFM_ACTIVE | IFM_AVALID)) {
1244                 switch (IFM_SUBTYPE(mii->mii_media_active)) {
1245                 case IFM_10_T:
1246                 case IFM_100_TX:
1247                         sc->bge_link = 1;
1248                         break;
1249                 case IFM_1000_T:
1250                 case IFM_1000_SX:
1251                 case IFM_2500_SX:
1252                         if (sc->bge_asicrev != BGE_ASICREV_BCM5906)
1253                                 sc->bge_link = 1;
1254                         else
1255                                 sc->bge_link = 0;
1256                         break;
1257                 default:
1258                         sc->bge_link = 0;
1259                         break;
1260                 }
1261         } else
1262                 sc->bge_link = 0;
1263         if (sc->bge_link == 0)
1264                 return;
1265
1266         /*
1267          * APE firmware touches these registers to keep the MAC
1268          * connected to the outside world.  Try to keep the
1269          * accesses atomic.
1270          */
1271
1272         /* Set the port mode (MII/GMII) to match the link speed. */
1273         mac_mode = CSR_READ_4(sc, BGE_MAC_MODE) &
1274             ~(BGE_MACMODE_PORTMODE | BGE_MACMODE_HALF_DUPLEX);
1275         tx_mode = CSR_READ_4(sc, BGE_TX_MODE);
1276         rx_mode = CSR_READ_4(sc, BGE_RX_MODE);
1277
1278         if (IFM_SUBTYPE(mii->mii_media_active) == IFM_1000_T ||
1279             IFM_SUBTYPE(mii->mii_media_active) == IFM_1000_SX)
1280                 mac_mode |= BGE_PORTMODE_GMII;
1281         else
1282                 mac_mode |= BGE_PORTMODE_MII;
1283
1284         /* Set MAC flow control behavior to match link flow control settings. */
1285         tx_mode &= ~BGE_TXMODE_FLOWCTL_ENABLE;
1286         rx_mode &= ~BGE_RXMODE_FLOWCTL_ENABLE;
1287         if ((IFM_OPTIONS(mii->mii_media_active) & IFM_FDX) != 0) {
1288                 if ((IFM_OPTIONS(mii->mii_media_active) & IFM_ETH_TXPAUSE) != 0)
1289                         tx_mode |= BGE_TXMODE_FLOWCTL_ENABLE;
1290                 if ((IFM_OPTIONS(mii->mii_media_active) & IFM_ETH_RXPAUSE) != 0)
1291                         rx_mode |= BGE_RXMODE_FLOWCTL_ENABLE;
1292         } else
1293                 mac_mode |= BGE_MACMODE_HALF_DUPLEX;
1294
1295         CSR_WRITE_4(sc, BGE_MAC_MODE, mac_mode);
1296         DELAY(40);
1297         CSR_WRITE_4(sc, BGE_TX_MODE, tx_mode);
1298         CSR_WRITE_4(sc, BGE_RX_MODE, rx_mode);
1299 }
1300
1301 /*
1302  * Intialize a standard receive ring descriptor.
1303  */
1304 static int
1305 bge_newbuf_std(struct bge_softc *sc, int i)
1306 {
1307         struct mbuf *m;
1308         struct bge_rx_bd *r;
1309         bus_dma_segment_t segs[1];
1310         bus_dmamap_t map;
1311         int error, nsegs;
1312
1313         if (sc->bge_flags & BGE_FLAG_JUMBO_STD &&
1314             (sc->bge_ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN +
1315             ETHER_VLAN_ENCAP_LEN > (MCLBYTES - ETHER_ALIGN))) {
1316                 m = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, MJUM9BYTES);
1317                 if (m == NULL)
1318                         return (ENOBUFS);
1319                 m->m_len = m->m_pkthdr.len = MJUM9BYTES;
1320         } else {
1321                 m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
1322                 if (m == NULL)
1323                         return (ENOBUFS);
1324                 m->m_len = m->m_pkthdr.len = MCLBYTES;
1325         }
1326         if ((sc->bge_flags & BGE_FLAG_RX_ALIGNBUG) == 0)
1327                 m_adj(m, ETHER_ALIGN);
1328
1329         error = bus_dmamap_load_mbuf_sg(sc->bge_cdata.bge_rx_mtag,
1330             sc->bge_cdata.bge_rx_std_sparemap, m, segs, &nsegs, 0);
1331         if (error != 0) {
1332                 m_freem(m);
1333                 return (error);
1334         }
1335         if (sc->bge_cdata.bge_rx_std_chain[i] != NULL) {
1336                 bus_dmamap_sync(sc->bge_cdata.bge_rx_mtag,
1337                     sc->bge_cdata.bge_rx_std_dmamap[i], BUS_DMASYNC_POSTREAD);
1338                 bus_dmamap_unload(sc->bge_cdata.bge_rx_mtag,
1339                     sc->bge_cdata.bge_rx_std_dmamap[i]);
1340         }
1341         map = sc->bge_cdata.bge_rx_std_dmamap[i];
1342         sc->bge_cdata.bge_rx_std_dmamap[i] = sc->bge_cdata.bge_rx_std_sparemap;
1343         sc->bge_cdata.bge_rx_std_sparemap = map;
1344         sc->bge_cdata.bge_rx_std_chain[i] = m;
1345         sc->bge_cdata.bge_rx_std_seglen[i] = segs[0].ds_len;
1346         r = &sc->bge_ldata.bge_rx_std_ring[sc->bge_std];
1347         r->bge_addr.bge_addr_lo = BGE_ADDR_LO(segs[0].ds_addr);
1348         r->bge_addr.bge_addr_hi = BGE_ADDR_HI(segs[0].ds_addr);
1349         r->bge_flags = BGE_RXBDFLAG_END;
1350         r->bge_len = segs[0].ds_len;
1351         r->bge_idx = i;
1352
1353         bus_dmamap_sync(sc->bge_cdata.bge_rx_mtag,
1354             sc->bge_cdata.bge_rx_std_dmamap[i], BUS_DMASYNC_PREREAD);
1355
1356         return (0);
1357 }
1358
1359 /*
1360  * Initialize a jumbo receive ring descriptor. This allocates
1361  * a jumbo buffer from the pool managed internally by the driver.
1362  */
1363 static int
1364 bge_newbuf_jumbo(struct bge_softc *sc, int i)
1365 {
1366         bus_dma_segment_t segs[BGE_NSEG_JUMBO];
1367         bus_dmamap_t map;
1368         struct bge_extrx_bd *r;
1369         struct mbuf *m;
1370         int error, nsegs;
1371
1372         MGETHDR(m, M_NOWAIT, MT_DATA);
1373         if (m == NULL)
1374                 return (ENOBUFS);
1375
1376         m_cljget(m, M_NOWAIT, MJUM9BYTES);
1377         if (!(m->m_flags & M_EXT)) {
1378                 m_freem(m);
1379                 return (ENOBUFS);
1380         }
1381         m->m_len = m->m_pkthdr.len = MJUM9BYTES;
1382         if ((sc->bge_flags & BGE_FLAG_RX_ALIGNBUG) == 0)
1383                 m_adj(m, ETHER_ALIGN);
1384
1385         error = bus_dmamap_load_mbuf_sg(sc->bge_cdata.bge_mtag_jumbo,
1386             sc->bge_cdata.bge_rx_jumbo_sparemap, m, segs, &nsegs, 0);
1387         if (error != 0) {
1388                 m_freem(m);
1389                 return (error);
1390         }
1391
1392         if (sc->bge_cdata.bge_rx_jumbo_chain[i] != NULL) {
1393                 bus_dmamap_sync(sc->bge_cdata.bge_mtag_jumbo,
1394                     sc->bge_cdata.bge_rx_jumbo_dmamap[i], BUS_DMASYNC_POSTREAD);
1395                 bus_dmamap_unload(sc->bge_cdata.bge_mtag_jumbo,
1396                     sc->bge_cdata.bge_rx_jumbo_dmamap[i]);
1397         }
1398         map = sc->bge_cdata.bge_rx_jumbo_dmamap[i];
1399         sc->bge_cdata.bge_rx_jumbo_dmamap[i] =
1400             sc->bge_cdata.bge_rx_jumbo_sparemap;
1401         sc->bge_cdata.bge_rx_jumbo_sparemap = map;
1402         sc->bge_cdata.bge_rx_jumbo_chain[i] = m;
1403         sc->bge_cdata.bge_rx_jumbo_seglen[i][0] = 0;
1404         sc->bge_cdata.bge_rx_jumbo_seglen[i][1] = 0;
1405         sc->bge_cdata.bge_rx_jumbo_seglen[i][2] = 0;
1406         sc->bge_cdata.bge_rx_jumbo_seglen[i][3] = 0;
1407
1408         /*
1409          * Fill in the extended RX buffer descriptor.
1410          */
1411         r = &sc->bge_ldata.bge_rx_jumbo_ring[sc->bge_jumbo];
1412         r->bge_flags = BGE_RXBDFLAG_JUMBO_RING | BGE_RXBDFLAG_END;
1413         r->bge_idx = i;
1414         r->bge_len3 = r->bge_len2 = r->bge_len1 = 0;
1415         switch (nsegs) {
1416         case 4:
1417                 r->bge_addr3.bge_addr_lo = BGE_ADDR_LO(segs[3].ds_addr);
1418                 r->bge_addr3.bge_addr_hi = BGE_ADDR_HI(segs[3].ds_addr);
1419                 r->bge_len3 = segs[3].ds_len;
1420                 sc->bge_cdata.bge_rx_jumbo_seglen[i][3] = segs[3].ds_len;
1421         case 3:
1422                 r->bge_addr2.bge_addr_lo = BGE_ADDR_LO(segs[2].ds_addr);
1423                 r->bge_addr2.bge_addr_hi = BGE_ADDR_HI(segs[2].ds_addr);
1424                 r->bge_len2 = segs[2].ds_len;
1425                 sc->bge_cdata.bge_rx_jumbo_seglen[i][2] = segs[2].ds_len;
1426         case 2:
1427                 r->bge_addr1.bge_addr_lo = BGE_ADDR_LO(segs[1].ds_addr);
1428                 r->bge_addr1.bge_addr_hi = BGE_ADDR_HI(segs[1].ds_addr);
1429                 r->bge_len1 = segs[1].ds_len;
1430                 sc->bge_cdata.bge_rx_jumbo_seglen[i][1] = segs[1].ds_len;
1431         case 1:
1432                 r->bge_addr0.bge_addr_lo = BGE_ADDR_LO(segs[0].ds_addr);
1433                 r->bge_addr0.bge_addr_hi = BGE_ADDR_HI(segs[0].ds_addr);
1434                 r->bge_len0 = segs[0].ds_len;
1435                 sc->bge_cdata.bge_rx_jumbo_seglen[i][0] = segs[0].ds_len;
1436                 break;
1437         default:
1438                 panic("%s: %d segments\n", __func__, nsegs);
1439         }
1440
1441         bus_dmamap_sync(sc->bge_cdata.bge_mtag_jumbo,
1442             sc->bge_cdata.bge_rx_jumbo_dmamap[i], BUS_DMASYNC_PREREAD);
1443
1444         return (0);
1445 }
1446
1447 static int
1448 bge_init_rx_ring_std(struct bge_softc *sc)
1449 {
1450         int error, i;
1451
1452         bzero(sc->bge_ldata.bge_rx_std_ring, BGE_STD_RX_RING_SZ);
1453         sc->bge_std = 0;
1454         for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
1455                 if ((error = bge_newbuf_std(sc, i)) != 0)
1456                         return (error);
1457                 BGE_INC(sc->bge_std, BGE_STD_RX_RING_CNT);
1458         }
1459
1460         bus_dmamap_sync(sc->bge_cdata.bge_rx_std_ring_tag,
1461             sc->bge_cdata.bge_rx_std_ring_map, BUS_DMASYNC_PREWRITE);
1462
1463         sc->bge_std = 0;
1464         bge_writembx(sc, BGE_MBX_RX_STD_PROD_LO, BGE_STD_RX_RING_CNT - 1);
1465
1466         return (0);
1467 }
1468
1469 static void
1470 bge_free_rx_ring_std(struct bge_softc *sc)
1471 {
1472         int i;
1473
1474         for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
1475                 if (sc->bge_cdata.bge_rx_std_chain[i] != NULL) {
1476                         bus_dmamap_sync(sc->bge_cdata.bge_rx_mtag,
1477                             sc->bge_cdata.bge_rx_std_dmamap[i],
1478                             BUS_DMASYNC_POSTREAD);
1479                         bus_dmamap_unload(sc->bge_cdata.bge_rx_mtag,
1480                             sc->bge_cdata.bge_rx_std_dmamap[i]);
1481                         m_freem(sc->bge_cdata.bge_rx_std_chain[i]);
1482                         sc->bge_cdata.bge_rx_std_chain[i] = NULL;
1483                 }
1484                 bzero((char *)&sc->bge_ldata.bge_rx_std_ring[i],
1485                     sizeof(struct bge_rx_bd));
1486         }
1487 }
1488
1489 static int
1490 bge_init_rx_ring_jumbo(struct bge_softc *sc)
1491 {
1492         struct bge_rcb *rcb;
1493         int error, i;
1494
1495         bzero(sc->bge_ldata.bge_rx_jumbo_ring, BGE_JUMBO_RX_RING_SZ);
1496         sc->bge_jumbo = 0;
1497         for (i = 0; i < BGE_JUMBO_RX_RING_CNT; i++) {
1498                 if ((error = bge_newbuf_jumbo(sc, i)) != 0)
1499                         return (error);
1500                 BGE_INC(sc->bge_jumbo, BGE_JUMBO_RX_RING_CNT);
1501         }
1502
1503         bus_dmamap_sync(sc->bge_cdata.bge_rx_jumbo_ring_tag,
1504             sc->bge_cdata.bge_rx_jumbo_ring_map, BUS_DMASYNC_PREWRITE);
1505
1506         sc->bge_jumbo = 0;
1507
1508         /* Enable the jumbo receive producer ring. */
1509         rcb = &sc->bge_ldata.bge_info.bge_jumbo_rx_rcb;
1510         rcb->bge_maxlen_flags =
1511             BGE_RCB_MAXLEN_FLAGS(0, BGE_RCB_FLAG_USE_EXT_RX_BD);
1512         CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_MAXLEN_FLAGS, rcb->bge_maxlen_flags);
1513
1514         bge_writembx(sc, BGE_MBX_RX_JUMBO_PROD_LO, BGE_JUMBO_RX_RING_CNT - 1);
1515
1516         return (0);
1517 }
1518
1519 static void
1520 bge_free_rx_ring_jumbo(struct bge_softc *sc)
1521 {
1522         int i;
1523
1524         for (i = 0; i < BGE_JUMBO_RX_RING_CNT; i++) {
1525                 if (sc->bge_cdata.bge_rx_jumbo_chain[i] != NULL) {
1526                         bus_dmamap_sync(sc->bge_cdata.bge_mtag_jumbo,
1527                             sc->bge_cdata.bge_rx_jumbo_dmamap[i],
1528                             BUS_DMASYNC_POSTREAD);
1529                         bus_dmamap_unload(sc->bge_cdata.bge_mtag_jumbo,
1530                             sc->bge_cdata.bge_rx_jumbo_dmamap[i]);
1531                         m_freem(sc->bge_cdata.bge_rx_jumbo_chain[i]);
1532                         sc->bge_cdata.bge_rx_jumbo_chain[i] = NULL;
1533                 }
1534                 bzero((char *)&sc->bge_ldata.bge_rx_jumbo_ring[i],
1535                     sizeof(struct bge_extrx_bd));
1536         }
1537 }
1538
1539 static void
1540 bge_free_tx_ring(struct bge_softc *sc)
1541 {
1542         int i;
1543
1544         if (sc->bge_ldata.bge_tx_ring == NULL)
1545                 return;
1546
1547         for (i = 0; i < BGE_TX_RING_CNT; i++) {
1548                 if (sc->bge_cdata.bge_tx_chain[i] != NULL) {
1549                         bus_dmamap_sync(sc->bge_cdata.bge_tx_mtag,
1550                             sc->bge_cdata.bge_tx_dmamap[i],
1551                             BUS_DMASYNC_POSTWRITE);
1552                         bus_dmamap_unload(sc->bge_cdata.bge_tx_mtag,
1553                             sc->bge_cdata.bge_tx_dmamap[i]);
1554                         m_freem(sc->bge_cdata.bge_tx_chain[i]);
1555                         sc->bge_cdata.bge_tx_chain[i] = NULL;
1556                 }
1557                 bzero((char *)&sc->bge_ldata.bge_tx_ring[i],
1558                     sizeof(struct bge_tx_bd));
1559         }
1560 }
1561
1562 static int
1563 bge_init_tx_ring(struct bge_softc *sc)
1564 {
1565         sc->bge_txcnt = 0;
1566         sc->bge_tx_saved_considx = 0;
1567
1568         bzero(sc->bge_ldata.bge_tx_ring, BGE_TX_RING_SZ);
1569         bus_dmamap_sync(sc->bge_cdata.bge_tx_ring_tag,
1570             sc->bge_cdata.bge_tx_ring_map, BUS_DMASYNC_PREWRITE);
1571
1572         /* Initialize transmit producer index for host-memory send ring. */
1573         sc->bge_tx_prodidx = 0;
1574         bge_writembx(sc, BGE_MBX_TX_HOST_PROD0_LO, sc->bge_tx_prodidx);
1575
1576         /* 5700 b2 errata */
1577         if (sc->bge_chiprev == BGE_CHIPREV_5700_BX)
1578                 bge_writembx(sc, BGE_MBX_TX_HOST_PROD0_LO, sc->bge_tx_prodidx);
1579
1580         /* NIC-memory send ring not used; initialize to zero. */
1581         bge_writembx(sc, BGE_MBX_TX_NIC_PROD0_LO, 0);
1582         /* 5700 b2 errata */
1583         if (sc->bge_chiprev == BGE_CHIPREV_5700_BX)
1584                 bge_writembx(sc, BGE_MBX_TX_NIC_PROD0_LO, 0);
1585
1586         return (0);
1587 }
1588
1589 static void
1590 bge_setpromisc(struct bge_softc *sc)
1591 {
1592         struct ifnet *ifp;
1593
1594         BGE_LOCK_ASSERT(sc);
1595
1596         ifp = sc->bge_ifp;
1597
1598         /* Enable or disable promiscuous mode as needed. */
1599         if (ifp->if_flags & IFF_PROMISC)
1600                 BGE_SETBIT(sc, BGE_RX_MODE, BGE_RXMODE_RX_PROMISC);
1601         else
1602                 BGE_CLRBIT(sc, BGE_RX_MODE, BGE_RXMODE_RX_PROMISC);
1603 }
1604
1605 static void
1606 bge_setmulti(struct bge_softc *sc)
1607 {
1608         struct ifnet *ifp;
1609         struct ifmultiaddr *ifma;
1610         uint32_t hashes[4] = { 0, 0, 0, 0 };
1611         int h, i;
1612
1613         BGE_LOCK_ASSERT(sc);
1614
1615         ifp = sc->bge_ifp;
1616
1617         if (ifp->if_flags & IFF_ALLMULTI || ifp->if_flags & IFF_PROMISC) {
1618                 for (i = 0; i < 4; i++)
1619                         CSR_WRITE_4(sc, BGE_MAR0 + (i * 4), 0xFFFFFFFF);
1620                 return;
1621         }
1622
1623         /* First, zot all the existing filters. */
1624         for (i = 0; i < 4; i++)
1625                 CSR_WRITE_4(sc, BGE_MAR0 + (i * 4), 0);
1626
1627         /* Now program new ones. */
1628         if_maddr_rlock(ifp);
1629         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1630                 if (ifma->ifma_addr->sa_family != AF_LINK)
1631                         continue;
1632                 h = ether_crc32_le(LLADDR((struct sockaddr_dl *)
1633                     ifma->ifma_addr), ETHER_ADDR_LEN) & 0x7F;
1634                 hashes[(h & 0x60) >> 5] |= 1 << (h & 0x1F);
1635         }
1636         if_maddr_runlock(ifp);
1637
1638         for (i = 0; i < 4; i++)
1639                 CSR_WRITE_4(sc, BGE_MAR0 + (i * 4), hashes[i]);
1640 }
1641
1642 static void
1643 bge_setvlan(struct bge_softc *sc)
1644 {
1645         struct ifnet *ifp;
1646
1647         BGE_LOCK_ASSERT(sc);
1648
1649         ifp = sc->bge_ifp;
1650
1651         /* Enable or disable VLAN tag stripping as needed. */
1652         if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
1653                 BGE_CLRBIT(sc, BGE_RX_MODE, BGE_RXMODE_RX_KEEP_VLAN_DIAG);
1654         else
1655                 BGE_SETBIT(sc, BGE_RX_MODE, BGE_RXMODE_RX_KEEP_VLAN_DIAG);
1656 }
1657
1658 static void
1659 bge_sig_pre_reset(struct bge_softc *sc, int type)
1660 {
1661
1662         /*
1663          * Some chips don't like this so only do this if ASF is enabled
1664          */
1665         if (sc->bge_asf_mode)
1666                 bge_writemem_ind(sc, BGE_SRAM_FW_MB, BGE_SRAM_FW_MB_MAGIC);
1667
1668         if (sc->bge_asf_mode & ASF_NEW_HANDSHAKE) {
1669                 switch (type) {
1670                 case BGE_RESET_START:
1671                         bge_writemem_ind(sc, BGE_SRAM_FW_DRV_STATE_MB,
1672                             BGE_FW_DRV_STATE_START);
1673                         break;
1674                 case BGE_RESET_SHUTDOWN:
1675                         bge_writemem_ind(sc, BGE_SRAM_FW_DRV_STATE_MB,
1676                             BGE_FW_DRV_STATE_UNLOAD);
1677                         break;
1678                 case BGE_RESET_SUSPEND:
1679                         bge_writemem_ind(sc, BGE_SRAM_FW_DRV_STATE_MB,
1680                             BGE_FW_DRV_STATE_SUSPEND);
1681                         break;
1682                 }
1683         }
1684
1685         if (type == BGE_RESET_START || type == BGE_RESET_SUSPEND)
1686                 bge_ape_driver_state_change(sc, type);
1687 }
1688
1689 static void
1690 bge_sig_post_reset(struct bge_softc *sc, int type)
1691 {
1692
1693         if (sc->bge_asf_mode & ASF_NEW_HANDSHAKE) {
1694                 switch (type) {
1695                 case BGE_RESET_START:
1696                         bge_writemem_ind(sc, BGE_SRAM_FW_DRV_STATE_MB,
1697                             BGE_FW_DRV_STATE_START_DONE);
1698                         /* START DONE */
1699                         break;
1700                 case BGE_RESET_SHUTDOWN:
1701                         bge_writemem_ind(sc, BGE_SRAM_FW_DRV_STATE_MB,
1702                             BGE_FW_DRV_STATE_UNLOAD_DONE);
1703                         break;
1704                 }
1705         }
1706         if (type == BGE_RESET_SHUTDOWN)
1707                 bge_ape_driver_state_change(sc, type);
1708 }
1709
1710 static void
1711 bge_sig_legacy(struct bge_softc *sc, int type)
1712 {
1713
1714         if (sc->bge_asf_mode) {
1715                 switch (type) {
1716                 case BGE_RESET_START:
1717                         bge_writemem_ind(sc, BGE_SRAM_FW_DRV_STATE_MB,
1718                             BGE_FW_DRV_STATE_START);
1719                         break;
1720                 case BGE_RESET_SHUTDOWN:
1721                         bge_writemem_ind(sc, BGE_SRAM_FW_DRV_STATE_MB,
1722                             BGE_FW_DRV_STATE_UNLOAD);
1723                         break;
1724                 }
1725         }
1726 }
1727
1728 static void
1729 bge_stop_fw(struct bge_softc *sc)
1730 {
1731         int i;
1732
1733         if (sc->bge_asf_mode) {
1734                 bge_writemem_ind(sc, BGE_SRAM_FW_CMD_MB, BGE_FW_CMD_PAUSE);
1735                 CSR_WRITE_4(sc, BGE_RX_CPU_EVENT,
1736                     CSR_READ_4(sc, BGE_RX_CPU_EVENT) | BGE_RX_CPU_DRV_EVENT);
1737
1738                 for (i = 0; i < 100; i++ ) {
1739                         if (!(CSR_READ_4(sc, BGE_RX_CPU_EVENT) &
1740                             BGE_RX_CPU_DRV_EVENT))
1741                                 break;
1742                         DELAY(10);
1743                 }
1744         }
1745 }
1746
1747 static uint32_t
1748 bge_dma_swap_options(struct bge_softc *sc)
1749 {
1750         uint32_t dma_options;
1751
1752         dma_options = BGE_MODECTL_WORDSWAP_NONFRAME |
1753             BGE_MODECTL_BYTESWAP_DATA | BGE_MODECTL_WORDSWAP_DATA;
1754 #if BYTE_ORDER == BIG_ENDIAN
1755         dma_options |= BGE_MODECTL_BYTESWAP_NONFRAME;
1756 #endif
1757         return (dma_options);
1758 }
1759
1760 /*
1761  * Do endian, PCI and DMA initialization.
1762  */
1763 static int
1764 bge_chipinit(struct bge_softc *sc)
1765 {
1766         uint32_t dma_rw_ctl, misc_ctl, mode_ctl;
1767         uint16_t val;
1768         int i;
1769
1770         /* Set endianness before we access any non-PCI registers. */
1771         misc_ctl = BGE_INIT;
1772         if (sc->bge_flags & BGE_FLAG_TAGGED_STATUS)
1773                 misc_ctl |= BGE_PCIMISCCTL_TAGGED_STATUS;
1774         pci_write_config(sc->bge_dev, BGE_PCI_MISC_CTL, misc_ctl, 4);
1775
1776         /*
1777          * Clear the MAC statistics block in the NIC's
1778          * internal memory.
1779          */
1780         for (i = BGE_STATS_BLOCK;
1781             i < BGE_STATS_BLOCK_END + 1; i += sizeof(uint32_t))
1782                 BGE_MEMWIN_WRITE(sc, i, 0);
1783
1784         for (i = BGE_STATUS_BLOCK;
1785             i < BGE_STATUS_BLOCK_END + 1; i += sizeof(uint32_t))
1786                 BGE_MEMWIN_WRITE(sc, i, 0);
1787
1788         if (sc->bge_chiprev == BGE_CHIPREV_5704_BX) {
1789                 /*
1790                  *  Fix data corruption caused by non-qword write with WB.
1791                  *  Fix master abort in PCI mode.
1792                  *  Fix PCI latency timer.
1793                  */
1794                 val = pci_read_config(sc->bge_dev, BGE_PCI_MSI_DATA + 2, 2);
1795                 val |= (1 << 10) | (1 << 12) | (1 << 13);
1796                 pci_write_config(sc->bge_dev, BGE_PCI_MSI_DATA + 2, val, 2);
1797         }
1798
1799         /*
1800          * Set up the PCI DMA control register.
1801          */
1802         dma_rw_ctl = BGE_PCIDMARWCTL_RD_CMD_SHIFT(6) |
1803             BGE_PCIDMARWCTL_WR_CMD_SHIFT(7);
1804         if (sc->bge_flags & BGE_FLAG_PCIE) {
1805                 if (sc->bge_mps >= 256)
1806                         dma_rw_ctl |= BGE_PCIDMARWCTL_WR_WAT_SHIFT(7);
1807                 else
1808                         dma_rw_ctl |= BGE_PCIDMARWCTL_WR_WAT_SHIFT(3);
1809         } else if (sc->bge_flags & BGE_FLAG_PCIX) {
1810                 if (BGE_IS_5714_FAMILY(sc)) {
1811                         /* 256 bytes for read and write. */
1812                         dma_rw_ctl |= BGE_PCIDMARWCTL_RD_WAT_SHIFT(2) |
1813                             BGE_PCIDMARWCTL_WR_WAT_SHIFT(2);
1814                         dma_rw_ctl |= (sc->bge_asicrev == BGE_ASICREV_BCM5780) ?
1815                             BGE_PCIDMARWCTL_ONEDMA_ATONCE_GLOBAL :
1816                             BGE_PCIDMARWCTL_ONEDMA_ATONCE_LOCAL;
1817                 } else if (sc->bge_asicrev == BGE_ASICREV_BCM5703) {
1818                         /*
1819                          * In the BCM5703, the DMA read watermark should
1820                          * be set to less than or equal to the maximum
1821                          * memory read byte count of the PCI-X command
1822                          * register.
1823                          */
1824                         dma_rw_ctl |= BGE_PCIDMARWCTL_RD_WAT_SHIFT(4) |
1825                             BGE_PCIDMARWCTL_WR_WAT_SHIFT(3);
1826                 } else if (sc->bge_asicrev == BGE_ASICREV_BCM5704) {
1827                         /* 1536 bytes for read, 384 bytes for write. */
1828                         dma_rw_ctl |= BGE_PCIDMARWCTL_RD_WAT_SHIFT(7) |
1829                             BGE_PCIDMARWCTL_WR_WAT_SHIFT(3);
1830                 } else {
1831                         /* 384 bytes for read and write. */
1832                         dma_rw_ctl |= BGE_PCIDMARWCTL_RD_WAT_SHIFT(3) |
1833                             BGE_PCIDMARWCTL_WR_WAT_SHIFT(3) |
1834                             0x0F;
1835                 }
1836                 if (sc->bge_asicrev == BGE_ASICREV_BCM5703 ||
1837                     sc->bge_asicrev == BGE_ASICREV_BCM5704) {
1838                         uint32_t tmp;
1839
1840                         /* Set ONE_DMA_AT_ONCE for hardware workaround. */
1841                         tmp = CSR_READ_4(sc, BGE_PCI_CLKCTL) & 0x1F;
1842                         if (tmp == 6 || tmp == 7)
1843                                 dma_rw_ctl |=
1844                                     BGE_PCIDMARWCTL_ONEDMA_ATONCE_GLOBAL;
1845
1846                         /* Set PCI-X DMA write workaround. */
1847                         dma_rw_ctl |= BGE_PCIDMARWCTL_ASRT_ALL_BE;
1848                 }
1849         } else {
1850                 /* Conventional PCI bus: 256 bytes for read and write. */
1851                 dma_rw_ctl |= BGE_PCIDMARWCTL_RD_WAT_SHIFT(7) |
1852                     BGE_PCIDMARWCTL_WR_WAT_SHIFT(7);
1853
1854                 if (sc->bge_asicrev != BGE_ASICREV_BCM5705 &&
1855                     sc->bge_asicrev != BGE_ASICREV_BCM5750)
1856                         dma_rw_ctl |= 0x0F;
1857         }
1858         if (sc->bge_asicrev == BGE_ASICREV_BCM5700 ||
1859             sc->bge_asicrev == BGE_ASICREV_BCM5701)
1860                 dma_rw_ctl |= BGE_PCIDMARWCTL_USE_MRM |
1861                     BGE_PCIDMARWCTL_ASRT_ALL_BE;
1862         if (sc->bge_asicrev == BGE_ASICREV_BCM5703 ||
1863             sc->bge_asicrev == BGE_ASICREV_BCM5704)
1864                 dma_rw_ctl &= ~BGE_PCIDMARWCTL_MINDMA;
1865         if (BGE_IS_5717_PLUS(sc)) {
1866                 dma_rw_ctl &= ~BGE_PCIDMARWCTL_DIS_CACHE_ALIGNMENT;
1867                 if (sc->bge_chipid == BGE_CHIPID_BCM57765_A0)
1868                         dma_rw_ctl &= ~BGE_PCIDMARWCTL_CRDRDR_RDMA_MRRS_MSK;
1869                 /*
1870                  * Enable HW workaround for controllers that misinterpret
1871                  * a status tag update and leave interrupts permanently
1872                  * disabled.
1873                  */
1874                 if (sc->bge_asicrev != BGE_ASICREV_BCM5717 &&
1875                     sc->bge_asicrev != BGE_ASICREV_BCM57765)
1876                         dma_rw_ctl |= BGE_PCIDMARWCTL_TAGGED_STATUS_WA;
1877         }
1878         pci_write_config(sc->bge_dev, BGE_PCI_DMA_RW_CTL, dma_rw_ctl, 4);
1879
1880         /*
1881          * Set up general mode register.
1882          */
1883         mode_ctl = bge_dma_swap_options(sc);
1884         if (sc->bge_asicrev == BGE_ASICREV_BCM5720) {
1885                 /* Retain Host-2-BMC settings written by APE firmware. */
1886                 mode_ctl |= CSR_READ_4(sc, BGE_MODE_CTL) &
1887                     (BGE_MODECTL_BYTESWAP_B2HRX_DATA |
1888                     BGE_MODECTL_WORDSWAP_B2HRX_DATA |
1889                     BGE_MODECTL_B2HRX_ENABLE | BGE_MODECTL_HTX2B_ENABLE);
1890         }
1891         mode_ctl |= BGE_MODECTL_MAC_ATTN_INTR | BGE_MODECTL_HOST_SEND_BDS |
1892             BGE_MODECTL_TX_NO_PHDR_CSUM;
1893
1894         /*
1895          * BCM5701 B5 have a bug causing data corruption when using
1896          * 64-bit DMA reads, which can be terminated early and then
1897          * completed later as 32-bit accesses, in combination with
1898          * certain bridges.
1899          */
1900         if (sc->bge_asicrev == BGE_ASICREV_BCM5701 &&
1901             sc->bge_chipid == BGE_CHIPID_BCM5701_B5)
1902                 mode_ctl |= BGE_MODECTL_FORCE_PCI32;
1903
1904         /*
1905          * Tell the firmware the driver is running
1906          */
1907         if (sc->bge_asf_mode & ASF_STACKUP)
1908                 mode_ctl |= BGE_MODECTL_STACKUP;
1909
1910         CSR_WRITE_4(sc, BGE_MODE_CTL, mode_ctl);
1911
1912         /*
1913          * Disable memory write invalidate.  Apparently it is not supported
1914          * properly by these devices.  Also ensure that INTx isn't disabled,
1915          * as these chips need it even when using MSI.
1916          */
1917         PCI_CLRBIT(sc->bge_dev, BGE_PCI_CMD,
1918             PCIM_CMD_INTxDIS | PCIM_CMD_MWIEN, 4);
1919
1920         /* Set the timer prescaler (always 66 MHz). */
1921         CSR_WRITE_4(sc, BGE_MISC_CFG, BGE_32BITTIME_66MHZ);
1922
1923         /* XXX: The Linux tg3 driver does this at the start of brgphy_reset. */
1924         if (sc->bge_asicrev == BGE_ASICREV_BCM5906) {
1925                 DELAY(40);      /* XXX */
1926
1927                 /* Put PHY into ready state */
1928                 BGE_CLRBIT(sc, BGE_MISC_CFG, BGE_MISCCFG_EPHY_IDDQ);
1929                 CSR_READ_4(sc, BGE_MISC_CFG); /* Flush */
1930                 DELAY(40);
1931         }
1932
1933         return (0);
1934 }
1935
1936 static int
1937 bge_blockinit(struct bge_softc *sc)
1938 {
1939         struct bge_rcb *rcb;
1940         bus_size_t vrcb;
1941         bge_hostaddr taddr;
1942         uint32_t dmactl, val;
1943         int i, limit;
1944
1945         /*
1946          * Initialize the memory window pointer register so that
1947          * we can access the first 32K of internal NIC RAM. This will
1948          * allow us to set up the TX send ring RCBs and the RX return
1949          * ring RCBs, plus other things which live in NIC memory.
1950          */
1951         CSR_WRITE_4(sc, BGE_PCI_MEMWIN_BASEADDR, 0);
1952
1953         /* Note: the BCM5704 has a smaller mbuf space than other chips. */
1954
1955         if (!(BGE_IS_5705_PLUS(sc))) {
1956                 /* Configure mbuf memory pool */
1957                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_BASEADDR, BGE_BUFFPOOL_1);
1958                 if (sc->bge_asicrev == BGE_ASICREV_BCM5704)
1959                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_LEN, 0x10000);
1960                 else
1961                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_LEN, 0x18000);
1962
1963                 /* Configure DMA resource pool */
1964                 CSR_WRITE_4(sc, BGE_BMAN_DMA_DESCPOOL_BASEADDR,
1965                     BGE_DMA_DESCRIPTORS);
1966                 CSR_WRITE_4(sc, BGE_BMAN_DMA_DESCPOOL_LEN, 0x2000);
1967         }
1968
1969         /* Configure mbuf pool watermarks */
1970         if (BGE_IS_5717_PLUS(sc)) {
1971                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_READDMA_LOWAT, 0x0);
1972                 if (sc->bge_ifp->if_mtu > ETHERMTU) {
1973                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x7e);
1974                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0xea);
1975                 } else {
1976                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x2a);
1977                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0xa0);
1978                 }
1979         } else if (!BGE_IS_5705_PLUS(sc)) {
1980                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_READDMA_LOWAT, 0x50);
1981                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x20);
1982                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0x60);
1983         } else if (sc->bge_asicrev == BGE_ASICREV_BCM5906) {
1984                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_READDMA_LOWAT, 0x0);
1985                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x04);
1986                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0x10);
1987         } else {
1988                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_READDMA_LOWAT, 0x0);
1989                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x10);
1990                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0x60);
1991         }
1992
1993         /* Configure DMA resource watermarks */
1994         CSR_WRITE_4(sc, BGE_BMAN_DMA_DESCPOOL_LOWAT, 5);
1995         CSR_WRITE_4(sc, BGE_BMAN_DMA_DESCPOOL_HIWAT, 10);
1996
1997         /* Enable buffer manager */
1998         val = BGE_BMANMODE_ENABLE | BGE_BMANMODE_LOMBUF_ATTN;
1999         /*
2000          * Change the arbitration algorithm of TXMBUF read request to
2001          * round-robin instead of priority based for BCM5719.  When
2002          * TXFIFO is almost empty, RDMA will hold its request until
2003          * TXFIFO is not almost empty.
2004          */
2005         if (sc->bge_asicrev == BGE_ASICREV_BCM5719)
2006                 val |= BGE_BMANMODE_NO_TX_UNDERRUN;
2007         CSR_WRITE_4(sc, BGE_BMAN_MODE, val);
2008
2009         /* Poll for buffer manager start indication */
2010         for (i = 0; i < BGE_TIMEOUT; i++) {
2011                 DELAY(10);
2012                 if (CSR_READ_4(sc, BGE_BMAN_MODE) & BGE_BMANMODE_ENABLE)
2013                         break;
2014         }
2015
2016         if (i == BGE_TIMEOUT) {
2017                 device_printf(sc->bge_dev, "buffer manager failed to start\n");
2018                 return (ENXIO);
2019         }
2020
2021         /* Enable flow-through queues */
2022         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0xFFFFFFFF);
2023         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0);
2024
2025         /* Wait until queue initialization is complete */
2026         for (i = 0; i < BGE_TIMEOUT; i++) {
2027                 DELAY(10);
2028                 if (CSR_READ_4(sc, BGE_FTQ_RESET) == 0)
2029                         break;
2030         }
2031
2032         if (i == BGE_TIMEOUT) {
2033                 device_printf(sc->bge_dev, "flow-through queue init failed\n");
2034                 return (ENXIO);
2035         }
2036
2037         /*
2038          * Summary of rings supported by the controller:
2039          *
2040          * Standard Receive Producer Ring
2041          * - This ring is used to feed receive buffers for "standard"
2042          *   sized frames (typically 1536 bytes) to the controller.
2043          *
2044          * Jumbo Receive Producer Ring
2045          * - This ring is used to feed receive buffers for jumbo sized
2046          *   frames (i.e. anything bigger than the "standard" frames)
2047          *   to the controller.
2048          *
2049          * Mini Receive Producer Ring
2050          * - This ring is used to feed receive buffers for "mini"
2051          *   sized frames to the controller.
2052          * - This feature required external memory for the controller
2053          *   but was never used in a production system.  Should always
2054          *   be disabled.
2055          *
2056          * Receive Return Ring
2057          * - After the controller has placed an incoming frame into a
2058          *   receive buffer that buffer is moved into a receive return
2059          *   ring.  The driver is then responsible to passing the
2060          *   buffer up to the stack.  Many versions of the controller
2061          *   support multiple RR rings.
2062          *
2063          * Send Ring
2064          * - This ring is used for outgoing frames.  Many versions of
2065          *   the controller support multiple send rings.
2066          */
2067
2068         /* Initialize the standard receive producer ring control block. */
2069         rcb = &sc->bge_ldata.bge_info.bge_std_rx_rcb;
2070         rcb->bge_hostaddr.bge_addr_lo =
2071             BGE_ADDR_LO(sc->bge_ldata.bge_rx_std_ring_paddr);
2072         rcb->bge_hostaddr.bge_addr_hi =
2073             BGE_ADDR_HI(sc->bge_ldata.bge_rx_std_ring_paddr);
2074         bus_dmamap_sync(sc->bge_cdata.bge_rx_std_ring_tag,
2075             sc->bge_cdata.bge_rx_std_ring_map, BUS_DMASYNC_PREREAD);
2076         if (BGE_IS_5717_PLUS(sc)) {
2077                 /*
2078                  * Bits 31-16: Programmable ring size (2048, 1024, 512, .., 32)
2079                  * Bits 15-2 : Maximum RX frame size
2080                  * Bit 1     : 1 = Ring Disabled, 0 = Ring ENabled
2081                  * Bit 0     : Reserved
2082                  */
2083                 rcb->bge_maxlen_flags =
2084                     BGE_RCB_MAXLEN_FLAGS(512, BGE_MAX_FRAMELEN << 2);
2085         } else if (BGE_IS_5705_PLUS(sc)) {
2086                 /*
2087                  * Bits 31-16: Programmable ring size (512, 256, 128, 64, 32)
2088                  * Bits 15-2 : Reserved (should be 0)
2089                  * Bit 1     : 1 = Ring Disabled, 0 = Ring Enabled
2090                  * Bit 0     : Reserved
2091                  */
2092                 rcb->bge_maxlen_flags = BGE_RCB_MAXLEN_FLAGS(512, 0);
2093         } else {
2094                 /*
2095                  * Ring size is always XXX entries
2096                  * Bits 31-16: Maximum RX frame size
2097                  * Bits 15-2 : Reserved (should be 0)
2098                  * Bit 1     : 1 = Ring Disabled, 0 = Ring Enabled
2099                  * Bit 0     : Reserved
2100                  */
2101                 rcb->bge_maxlen_flags =
2102                     BGE_RCB_MAXLEN_FLAGS(BGE_MAX_FRAMELEN, 0);
2103         }
2104         if (sc->bge_asicrev == BGE_ASICREV_BCM5717 ||
2105             sc->bge_asicrev == BGE_ASICREV_BCM5719 ||
2106             sc->bge_asicrev == BGE_ASICREV_BCM5720)
2107                 rcb->bge_nicaddr = BGE_STD_RX_RINGS_5717;
2108         else
2109                 rcb->bge_nicaddr = BGE_STD_RX_RINGS;
2110         /* Write the standard receive producer ring control block. */
2111         CSR_WRITE_4(sc, BGE_RX_STD_RCB_HADDR_HI, rcb->bge_hostaddr.bge_addr_hi);
2112         CSR_WRITE_4(sc, BGE_RX_STD_RCB_HADDR_LO, rcb->bge_hostaddr.bge_addr_lo);
2113         CSR_WRITE_4(sc, BGE_RX_STD_RCB_MAXLEN_FLAGS, rcb->bge_maxlen_flags);
2114         CSR_WRITE_4(sc, BGE_RX_STD_RCB_NICADDR, rcb->bge_nicaddr);
2115
2116         /* Reset the standard receive producer ring producer index. */
2117         bge_writembx(sc, BGE_MBX_RX_STD_PROD_LO, 0);
2118
2119         /*
2120          * Initialize the jumbo RX producer ring control
2121          * block.  We set the 'ring disabled' bit in the
2122          * flags field until we're actually ready to start
2123          * using this ring (i.e. once we set the MTU
2124          * high enough to require it).
2125          */
2126         if (BGE_IS_JUMBO_CAPABLE(sc)) {
2127                 rcb = &sc->bge_ldata.bge_info.bge_jumbo_rx_rcb;
2128                 /* Get the jumbo receive producer ring RCB parameters. */
2129                 rcb->bge_hostaddr.bge_addr_lo =
2130                     BGE_ADDR_LO(sc->bge_ldata.bge_rx_jumbo_ring_paddr);
2131                 rcb->bge_hostaddr.bge_addr_hi =
2132                     BGE_ADDR_HI(sc->bge_ldata.bge_rx_jumbo_ring_paddr);
2133                 bus_dmamap_sync(sc->bge_cdata.bge_rx_jumbo_ring_tag,
2134                     sc->bge_cdata.bge_rx_jumbo_ring_map,
2135                     BUS_DMASYNC_PREREAD);
2136                 rcb->bge_maxlen_flags = BGE_RCB_MAXLEN_FLAGS(0,
2137                     BGE_RCB_FLAG_USE_EXT_RX_BD | BGE_RCB_FLAG_RING_DISABLED);
2138                 if (sc->bge_asicrev == BGE_ASICREV_BCM5717 ||
2139                     sc->bge_asicrev == BGE_ASICREV_BCM5719 ||
2140                     sc->bge_asicrev == BGE_ASICREV_BCM5720)
2141                         rcb->bge_nicaddr = BGE_JUMBO_RX_RINGS_5717;
2142                 else
2143                         rcb->bge_nicaddr = BGE_JUMBO_RX_RINGS;
2144                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_HADDR_HI,
2145                     rcb->bge_hostaddr.bge_addr_hi);
2146                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_HADDR_LO,
2147                     rcb->bge_hostaddr.bge_addr_lo);
2148                 /* Program the jumbo receive producer ring RCB parameters. */
2149                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_MAXLEN_FLAGS,
2150                     rcb->bge_maxlen_flags);
2151                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_NICADDR, rcb->bge_nicaddr);
2152                 /* Reset the jumbo receive producer ring producer index. */
2153                 bge_writembx(sc, BGE_MBX_RX_JUMBO_PROD_LO, 0);
2154         }
2155
2156         /* Disable the mini receive producer ring RCB. */
2157         if (BGE_IS_5700_FAMILY(sc)) {
2158                 rcb = &sc->bge_ldata.bge_info.bge_mini_rx_rcb;
2159                 rcb->bge_maxlen_flags =
2160                     BGE_RCB_MAXLEN_FLAGS(0, BGE_RCB_FLAG_RING_DISABLED);
2161                 CSR_WRITE_4(sc, BGE_RX_MINI_RCB_MAXLEN_FLAGS,
2162                     rcb->bge_maxlen_flags);
2163                 /* Reset the mini receive producer ring producer index. */
2164                 bge_writembx(sc, BGE_MBX_RX_MINI_PROD_LO, 0);
2165         }
2166
2167         /* Choose de-pipeline mode for BCM5906 A0, A1 and A2. */
2168         if (sc->bge_asicrev == BGE_ASICREV_BCM5906) {
2169                 if (sc->bge_chipid == BGE_CHIPID_BCM5906_A0 ||
2170                     sc->bge_chipid == BGE_CHIPID_BCM5906_A1 ||
2171                     sc->bge_chipid == BGE_CHIPID_BCM5906_A2)
2172                         CSR_WRITE_4(sc, BGE_ISO_PKT_TX,
2173                             (CSR_READ_4(sc, BGE_ISO_PKT_TX) & ~3) | 2);
2174         }
2175         /*
2176          * The BD ring replenish thresholds control how often the
2177          * hardware fetches new BD's from the producer rings in host
2178          * memory.  Setting the value too low on a busy system can
2179          * starve the hardware and recue the throughpout.
2180          *
2181          * Set the BD ring replentish thresholds. The recommended
2182          * values are 1/8th the number of descriptors allocated to
2183          * each ring.
2184          * XXX The 5754 requires a lower threshold, so it might be a
2185          * requirement of all 575x family chips.  The Linux driver sets
2186          * the lower threshold for all 5705 family chips as well, but there
2187          * are reports that it might not need to be so strict.
2188          *
2189          * XXX Linux does some extra fiddling here for the 5906 parts as
2190          * well.
2191          */
2192         if (BGE_IS_5705_PLUS(sc))
2193                 val = 8;
2194         else
2195                 val = BGE_STD_RX_RING_CNT / 8;
2196         CSR_WRITE_4(sc, BGE_RBDI_STD_REPL_THRESH, val);
2197         if (BGE_IS_JUMBO_CAPABLE(sc))
2198                 CSR_WRITE_4(sc, BGE_RBDI_JUMBO_REPL_THRESH,
2199                     BGE_JUMBO_RX_RING_CNT/8);
2200         if (BGE_IS_5717_PLUS(sc)) {
2201                 CSR_WRITE_4(sc, BGE_STD_REPLENISH_LWM, 32);
2202                 CSR_WRITE_4(sc, BGE_JMB_REPLENISH_LWM, 16);
2203         }
2204
2205         /*
2206          * Disable all send rings by setting the 'ring disabled' bit
2207          * in the flags field of all the TX send ring control blocks,
2208          * located in NIC memory.
2209          */
2210         if (!BGE_IS_5705_PLUS(sc))
2211                 /* 5700 to 5704 had 16 send rings. */
2212                 limit = BGE_TX_RINGS_EXTSSRAM_MAX;
2213         else
2214                 limit = 1;
2215         vrcb = BGE_MEMWIN_START + BGE_SEND_RING_RCB;
2216         for (i = 0; i < limit; i++) {
2217                 RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
2218                     BGE_RCB_MAXLEN_FLAGS(0, BGE_RCB_FLAG_RING_DISABLED));
2219                 RCB_WRITE_4(sc, vrcb, bge_nicaddr, 0);
2220                 vrcb += sizeof(struct bge_rcb);
2221         }
2222
2223         /* Configure send ring RCB 0 (we use only the first ring) */
2224         vrcb = BGE_MEMWIN_START + BGE_SEND_RING_RCB;
2225         BGE_HOSTADDR(taddr, sc->bge_ldata.bge_tx_ring_paddr);
2226         RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi, taddr.bge_addr_hi);
2227         RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo, taddr.bge_addr_lo);
2228         if (sc->bge_asicrev == BGE_ASICREV_BCM5717 ||
2229             sc->bge_asicrev == BGE_ASICREV_BCM5719 ||
2230             sc->bge_asicrev == BGE_ASICREV_BCM5720)
2231                 RCB_WRITE_4(sc, vrcb, bge_nicaddr, BGE_SEND_RING_5717);
2232         else
2233                 RCB_WRITE_4(sc, vrcb, bge_nicaddr,
2234                     BGE_NIC_TXRING_ADDR(0, BGE_TX_RING_CNT));
2235         RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
2236             BGE_RCB_MAXLEN_FLAGS(BGE_TX_RING_CNT, 0));
2237
2238         /*
2239          * Disable all receive return rings by setting the
2240          * 'ring diabled' bit in the flags field of all the receive
2241          * return ring control blocks, located in NIC memory.
2242          */
2243         if (sc->bge_asicrev == BGE_ASICREV_BCM5717 ||
2244             sc->bge_asicrev == BGE_ASICREV_BCM5719 ||
2245             sc->bge_asicrev == BGE_ASICREV_BCM5720) {
2246                 /* Should be 17, use 16 until we get an SRAM map. */
2247                 limit = 16;
2248         } else if (!BGE_IS_5705_PLUS(sc))
2249                 limit = BGE_RX_RINGS_MAX;
2250         else if (sc->bge_asicrev == BGE_ASICREV_BCM5755 ||
2251             BGE_IS_57765_PLUS(sc))
2252                 limit = 4;
2253         else
2254                 limit = 1;
2255         /* Disable all receive return rings. */
2256         vrcb = BGE_MEMWIN_START + BGE_RX_RETURN_RING_RCB;
2257         for (i = 0; i < limit; i++) {
2258                 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi, 0);
2259                 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo, 0);
2260                 RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
2261                     BGE_RCB_FLAG_RING_DISABLED);
2262                 RCB_WRITE_4(sc, vrcb, bge_nicaddr, 0);
2263                 bge_writembx(sc, BGE_MBX_RX_CONS0_LO +
2264                     (i * (sizeof(uint64_t))), 0);
2265                 vrcb += sizeof(struct bge_rcb);
2266         }
2267
2268         /*
2269          * Set up receive return ring 0.  Note that the NIC address
2270          * for RX return rings is 0x0.  The return rings live entirely
2271          * within the host, so the nicaddr field in the RCB isn't used.
2272          */
2273         vrcb = BGE_MEMWIN_START + BGE_RX_RETURN_RING_RCB;
2274         BGE_HOSTADDR(taddr, sc->bge_ldata.bge_rx_return_ring_paddr);
2275         RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi, taddr.bge_addr_hi);
2276         RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo, taddr.bge_addr_lo);
2277         RCB_WRITE_4(sc, vrcb, bge_nicaddr, 0);
2278         RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
2279             BGE_RCB_MAXLEN_FLAGS(sc->bge_return_ring_cnt, 0));
2280
2281         /* Set random backoff seed for TX */
2282         CSR_WRITE_4(sc, BGE_TX_RANDOM_BACKOFF,
2283             (IF_LLADDR(sc->bge_ifp)[0] + IF_LLADDR(sc->bge_ifp)[1] +
2284             IF_LLADDR(sc->bge_ifp)[2] + IF_LLADDR(sc->bge_ifp)[3] +
2285             IF_LLADDR(sc->bge_ifp)[4] + IF_LLADDR(sc->bge_ifp)[5]) &
2286             BGE_TX_BACKOFF_SEED_MASK);
2287
2288         /* Set inter-packet gap */
2289         val = 0x2620;
2290         if (sc->bge_asicrev == BGE_ASICREV_BCM5720)
2291                 val |= CSR_READ_4(sc, BGE_TX_LENGTHS) &
2292                     (BGE_TXLEN_JMB_FRM_LEN_MSK | BGE_TXLEN_CNT_DN_VAL_MSK);
2293         CSR_WRITE_4(sc, BGE_TX_LENGTHS, val);
2294
2295         /*
2296          * Specify which ring to use for packets that don't match
2297          * any RX rules.
2298          */
2299         CSR_WRITE_4(sc, BGE_RX_RULES_CFG, 0x08);
2300
2301         /*
2302          * Configure number of RX lists. One interrupt distribution
2303          * list, sixteen active lists, one bad frames class.
2304          */
2305         CSR_WRITE_4(sc, BGE_RXLP_CFG, 0x181);
2306
2307         /* Inialize RX list placement stats mask. */
2308         CSR_WRITE_4(sc, BGE_RXLP_STATS_ENABLE_MASK, 0x007FFFFF);
2309         CSR_WRITE_4(sc, BGE_RXLP_STATS_CTL, 0x1);
2310
2311         /* Disable host coalescing until we get it set up */
2312         CSR_WRITE_4(sc, BGE_HCC_MODE, 0x00000000);
2313
2314         /* Poll to make sure it's shut down. */
2315         for (i = 0; i < BGE_TIMEOUT; i++) {
2316                 DELAY(10);
2317                 if (!(CSR_READ_4(sc, BGE_HCC_MODE) & BGE_HCCMODE_ENABLE))
2318                         break;
2319         }
2320
2321         if (i == BGE_TIMEOUT) {
2322                 device_printf(sc->bge_dev,
2323                     "host coalescing engine failed to idle\n");
2324                 return (ENXIO);
2325         }
2326
2327         /* Set up host coalescing defaults */
2328         CSR_WRITE_4(sc, BGE_HCC_RX_COAL_TICKS, sc->bge_rx_coal_ticks);
2329         CSR_WRITE_4(sc, BGE_HCC_TX_COAL_TICKS, sc->bge_tx_coal_ticks);
2330         CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS, sc->bge_rx_max_coal_bds);
2331         CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS, sc->bge_tx_max_coal_bds);
2332         if (!(BGE_IS_5705_PLUS(sc))) {
2333                 CSR_WRITE_4(sc, BGE_HCC_RX_COAL_TICKS_INT, 0);
2334                 CSR_WRITE_4(sc, BGE_HCC_TX_COAL_TICKS_INT, 0);
2335         }
2336         CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS_INT, 1);
2337         CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS_INT, 1);
2338
2339         /* Set up address of statistics block */
2340         if (!(BGE_IS_5705_PLUS(sc))) {
2341                 CSR_WRITE_4(sc, BGE_HCC_STATS_ADDR_HI,
2342                     BGE_ADDR_HI(sc->bge_ldata.bge_stats_paddr));
2343                 CSR_WRITE_4(sc, BGE_HCC_STATS_ADDR_LO,
2344                     BGE_ADDR_LO(sc->bge_ldata.bge_stats_paddr));
2345                 CSR_WRITE_4(sc, BGE_HCC_STATS_BASEADDR, BGE_STATS_BLOCK);
2346                 CSR_WRITE_4(sc, BGE_HCC_STATUSBLK_BASEADDR, BGE_STATUS_BLOCK);
2347                 CSR_WRITE_4(sc, BGE_HCC_STATS_TICKS, sc->bge_stat_ticks);
2348         }
2349
2350         /* Set up address of status block */
2351         CSR_WRITE_4(sc, BGE_HCC_STATUSBLK_ADDR_HI,
2352             BGE_ADDR_HI(sc->bge_ldata.bge_status_block_paddr));
2353         CSR_WRITE_4(sc, BGE_HCC_STATUSBLK_ADDR_LO,
2354             BGE_ADDR_LO(sc->bge_ldata.bge_status_block_paddr));
2355
2356         /* Set up status block size. */
2357         if (sc->bge_asicrev == BGE_ASICREV_BCM5700 &&
2358             sc->bge_chipid != BGE_CHIPID_BCM5700_C0) {
2359                 val = BGE_STATBLKSZ_FULL;
2360                 bzero(sc->bge_ldata.bge_status_block, BGE_STATUS_BLK_SZ);
2361         } else {
2362                 val = BGE_STATBLKSZ_32BYTE;
2363                 bzero(sc->bge_ldata.bge_status_block, 32);
2364         }
2365         bus_dmamap_sync(sc->bge_cdata.bge_status_tag,
2366             sc->bge_cdata.bge_status_map,
2367             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2368
2369         /* Turn on host coalescing state machine */
2370         CSR_WRITE_4(sc, BGE_HCC_MODE, val | BGE_HCCMODE_ENABLE);
2371
2372         /* Turn on RX BD completion state machine and enable attentions */
2373         CSR_WRITE_4(sc, BGE_RBDC_MODE,
2374             BGE_RBDCMODE_ENABLE | BGE_RBDCMODE_ATTN);
2375
2376         /* Turn on RX list placement state machine */
2377         CSR_WRITE_4(sc, BGE_RXLP_MODE, BGE_RXLPMODE_ENABLE);
2378
2379         /* Turn on RX list selector state machine. */
2380         if (!(BGE_IS_5705_PLUS(sc)))
2381                 CSR_WRITE_4(sc, BGE_RXLS_MODE, BGE_RXLSMODE_ENABLE);
2382
2383         /* Turn on DMA, clear stats. */
2384         val = BGE_MACMODE_TXDMA_ENB | BGE_MACMODE_RXDMA_ENB |
2385             BGE_MACMODE_RX_STATS_CLEAR | BGE_MACMODE_TX_STATS_CLEAR |
2386             BGE_MACMODE_RX_STATS_ENB | BGE_MACMODE_TX_STATS_ENB |
2387             BGE_MACMODE_FRMHDR_DMA_ENB;
2388
2389         if (sc->bge_flags & BGE_FLAG_TBI)
2390                 val |= BGE_PORTMODE_TBI;
2391         else if (sc->bge_flags & BGE_FLAG_MII_SERDES)
2392                 val |= BGE_PORTMODE_GMII;
2393         else
2394                 val |= BGE_PORTMODE_MII;
2395
2396         /* Allow APE to send/receive frames. */
2397         if ((sc->bge_mfw_flags & BGE_MFW_ON_APE) != 0)
2398                 val |= BGE_MACMODE_APE_RX_EN | BGE_MACMODE_APE_TX_EN;
2399
2400         CSR_WRITE_4(sc, BGE_MAC_MODE, val);
2401         DELAY(40);
2402
2403         /* Set misc. local control, enable interrupts on attentions */
2404         BGE_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_INTR_ONATTN);
2405
2406 #ifdef notdef
2407         /* Assert GPIO pins for PHY reset */
2408         BGE_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_MISCIO_OUT0 |
2409             BGE_MLC_MISCIO_OUT1 | BGE_MLC_MISCIO_OUT2);
2410         BGE_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_MISCIO_OUTEN0 |
2411             BGE_MLC_MISCIO_OUTEN1 | BGE_MLC_MISCIO_OUTEN2);
2412 #endif
2413
2414         /* Turn on DMA completion state machine */
2415         if (!(BGE_IS_5705_PLUS(sc)))
2416                 CSR_WRITE_4(sc, BGE_DMAC_MODE, BGE_DMACMODE_ENABLE);
2417
2418         val = BGE_WDMAMODE_ENABLE | BGE_WDMAMODE_ALL_ATTNS;
2419
2420         /* Enable host coalescing bug fix. */
2421         if (BGE_IS_5755_PLUS(sc))
2422                 val |= BGE_WDMAMODE_STATUS_TAG_FIX;
2423
2424         /* Request larger DMA burst size to get better performance. */
2425         if (sc->bge_asicrev == BGE_ASICREV_BCM5785)
2426                 val |= BGE_WDMAMODE_BURST_ALL_DATA;
2427
2428         /* Turn on write DMA state machine */
2429         CSR_WRITE_4(sc, BGE_WDMA_MODE, val);
2430         DELAY(40);
2431
2432         /* Turn on read DMA state machine */
2433         val = BGE_RDMAMODE_ENABLE | BGE_RDMAMODE_ALL_ATTNS;
2434
2435         if (sc->bge_asicrev == BGE_ASICREV_BCM5717)
2436                 val |= BGE_RDMAMODE_MULT_DMA_RD_DIS;
2437
2438         if (sc->bge_asicrev == BGE_ASICREV_BCM5784 ||
2439             sc->bge_asicrev == BGE_ASICREV_BCM5785 ||
2440             sc->bge_asicrev == BGE_ASICREV_BCM57780)
2441                 val |= BGE_RDMAMODE_BD_SBD_CRPT_ATTN |
2442                     BGE_RDMAMODE_MBUF_RBD_CRPT_ATTN |
2443                     BGE_RDMAMODE_MBUF_SBD_CRPT_ATTN;
2444         if (sc->bge_flags & BGE_FLAG_PCIE)
2445                 val |= BGE_RDMAMODE_FIFO_LONG_BURST;
2446         if (sc->bge_flags & (BGE_FLAG_TSO | BGE_FLAG_TSO3)) {
2447                 val |= BGE_RDMAMODE_TSO4_ENABLE;
2448                 if (sc->bge_flags & BGE_FLAG_TSO3 ||
2449                     sc->bge_asicrev == BGE_ASICREV_BCM5785 ||
2450                     sc->bge_asicrev == BGE_ASICREV_BCM57780)
2451                         val |= BGE_RDMAMODE_TSO6_ENABLE;
2452         }
2453
2454         if (sc->bge_asicrev == BGE_ASICREV_BCM5720) {
2455                 val |= CSR_READ_4(sc, BGE_RDMA_MODE) &
2456                         BGE_RDMAMODE_H2BNC_VLAN_DET;
2457                 /*
2458                  * Allow multiple outstanding read requests from
2459                  * non-LSO read DMA engine.
2460                  */
2461                 val &= ~BGE_RDMAMODE_MULT_DMA_RD_DIS;
2462         }
2463
2464         if (sc->bge_asicrev == BGE_ASICREV_BCM5761 ||
2465             sc->bge_asicrev == BGE_ASICREV_BCM5784 ||
2466             sc->bge_asicrev == BGE_ASICREV_BCM5785 ||
2467             sc->bge_asicrev == BGE_ASICREV_BCM57780 ||
2468             BGE_IS_5717_PLUS(sc)) {
2469                 dmactl = CSR_READ_4(sc, BGE_RDMA_RSRVCTRL);
2470                 /*
2471                  * Adjust tx margin to prevent TX data corruption and
2472                  * fix internal FIFO overflow.
2473                  */
2474                 if (sc->bge_asicrev == BGE_ASICREV_BCM5719 &&
2475                     sc->bge_chipid == BGE_CHIPID_BCM5719_A0) {
2476                         dmactl &= ~(BGE_RDMA_RSRVCTRL_FIFO_LWM_MASK |
2477                             BGE_RDMA_RSRVCTRL_FIFO_HWM_MASK |
2478                             BGE_RDMA_RSRVCTRL_TXMRGN_MASK);
2479                         dmactl |= BGE_RDMA_RSRVCTRL_FIFO_LWM_1_5K |
2480                             BGE_RDMA_RSRVCTRL_FIFO_HWM_1_5K |
2481                             BGE_RDMA_RSRVCTRL_TXMRGN_320B;
2482                 }
2483                 /*
2484                  * Enable fix for read DMA FIFO overruns.
2485                  * The fix is to limit the number of RX BDs
2486                  * the hardware would fetch at a fime.
2487                  */
2488                 CSR_WRITE_4(sc, BGE_RDMA_RSRVCTRL, dmactl |
2489                     BGE_RDMA_RSRVCTRL_FIFO_OFLW_FIX);
2490         }
2491
2492         if (sc->bge_asicrev == BGE_ASICREV_BCM5719) {
2493                 CSR_WRITE_4(sc, BGE_RDMA_LSO_CRPTEN_CTRL,
2494                     CSR_READ_4(sc, BGE_RDMA_LSO_CRPTEN_CTRL) |
2495                     BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_BD_4K |
2496                     BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_LSO_4K);
2497         } else if (sc->bge_asicrev == BGE_ASICREV_BCM5720) {
2498                 /*
2499                  * Allow 4KB burst length reads for non-LSO frames.
2500                  * Enable 512B burst length reads for buffer descriptors.
2501                  */
2502                 CSR_WRITE_4(sc, BGE_RDMA_LSO_CRPTEN_CTRL,
2503                     CSR_READ_4(sc, BGE_RDMA_LSO_CRPTEN_CTRL) |
2504                     BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_BD_512 |
2505                     BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_LSO_4K);
2506         }
2507
2508         CSR_WRITE_4(sc, BGE_RDMA_MODE, val);
2509         DELAY(40);
2510
2511         /* Turn on RX data completion state machine */
2512         CSR_WRITE_4(sc, BGE_RDC_MODE, BGE_RDCMODE_ENABLE);
2513
2514         /* Turn on RX BD initiator state machine */
2515         CSR_WRITE_4(sc, BGE_RBDI_MODE, BGE_RBDIMODE_ENABLE);
2516
2517         /* Turn on RX data and RX BD initiator state machine */
2518         CSR_WRITE_4(sc, BGE_RDBDI_MODE, BGE_RDBDIMODE_ENABLE);
2519
2520         /* Turn on Mbuf cluster free state machine */
2521         if (!(BGE_IS_5705_PLUS(sc)))
2522                 CSR_WRITE_4(sc, BGE_MBCF_MODE, BGE_MBCFMODE_ENABLE);
2523
2524         /* Turn on send BD completion state machine */
2525         CSR_WRITE_4(sc, BGE_SBDC_MODE, BGE_SBDCMODE_ENABLE);
2526
2527         /* Turn on send data completion state machine */
2528         val = BGE_SDCMODE_ENABLE;
2529         if (sc->bge_asicrev == BGE_ASICREV_BCM5761)
2530                 val |= BGE_SDCMODE_CDELAY;
2531         CSR_WRITE_4(sc, BGE_SDC_MODE, val);
2532
2533         /* Turn on send data initiator state machine */
2534         if (sc->bge_flags & (BGE_FLAG_TSO | BGE_FLAG_TSO3))
2535                 CSR_WRITE_4(sc, BGE_SDI_MODE, BGE_SDIMODE_ENABLE |
2536                     BGE_SDIMODE_HW_LSO_PRE_DMA);
2537         else
2538                 CSR_WRITE_4(sc, BGE_SDI_MODE, BGE_SDIMODE_ENABLE);
2539
2540         /* Turn on send BD initiator state machine */
2541         CSR_WRITE_4(sc, BGE_SBDI_MODE, BGE_SBDIMODE_ENABLE);
2542
2543         /* Turn on send BD selector state machine */
2544         CSR_WRITE_4(sc, BGE_SRS_MODE, BGE_SRSMODE_ENABLE);
2545
2546         CSR_WRITE_4(sc, BGE_SDI_STATS_ENABLE_MASK, 0x007FFFFF);
2547         CSR_WRITE_4(sc, BGE_SDI_STATS_CTL,
2548             BGE_SDISTATSCTL_ENABLE | BGE_SDISTATSCTL_FASTER);
2549
2550         /* ack/clear link change events */
2551         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED |
2552             BGE_MACSTAT_CFG_CHANGED | BGE_MACSTAT_MI_COMPLETE |
2553             BGE_MACSTAT_LINK_CHANGED);
2554         CSR_WRITE_4(sc, BGE_MI_STS, 0);
2555
2556         /*
2557          * Enable attention when the link has changed state for
2558          * devices that use auto polling.
2559          */
2560         if (sc->bge_flags & BGE_FLAG_TBI) {
2561                 CSR_WRITE_4(sc, BGE_MI_STS, BGE_MISTS_LINK);
2562         } else {
2563                 if (sc->bge_mi_mode & BGE_MIMODE_AUTOPOLL) {
2564                         CSR_WRITE_4(sc, BGE_MI_MODE, sc->bge_mi_mode);
2565                         DELAY(80);
2566                 }
2567                 if (sc->bge_asicrev == BGE_ASICREV_BCM5700 &&
2568                     sc->bge_chipid != BGE_CHIPID_BCM5700_B2)
2569                         CSR_WRITE_4(sc, BGE_MAC_EVT_ENB,
2570                             BGE_EVTENB_MI_INTERRUPT);
2571         }
2572
2573         /*
2574          * Clear any pending link state attention.
2575          * Otherwise some link state change events may be lost until attention
2576          * is cleared by bge_intr() -> bge_link_upd() sequence.
2577          * It's not necessary on newer BCM chips - perhaps enabling link
2578          * state change attentions implies clearing pending attention.
2579          */
2580         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED |
2581             BGE_MACSTAT_CFG_CHANGED | BGE_MACSTAT_MI_COMPLETE |
2582             BGE_MACSTAT_LINK_CHANGED);
2583
2584         /* Enable link state change attentions. */
2585         BGE_SETBIT(sc, BGE_MAC_EVT_ENB, BGE_EVTENB_LINK_CHANGED);
2586
2587         return (0);
2588 }
2589
2590 static const struct bge_revision *
2591 bge_lookup_rev(uint32_t chipid)
2592 {
2593         const struct bge_revision *br;
2594
2595         for (br = bge_revisions; br->br_name != NULL; br++) {
2596                 if (br->br_chipid == chipid)
2597                         return (br);
2598         }
2599
2600         for (br = bge_majorrevs; br->br_name != NULL; br++) {
2601                 if (br->br_chipid == BGE_ASICREV(chipid))
2602                         return (br);
2603         }
2604
2605         return (NULL);
2606 }
2607
2608 static const struct bge_vendor *
2609 bge_lookup_vendor(uint16_t vid)
2610 {
2611         const struct bge_vendor *v;
2612
2613         for (v = bge_vendors; v->v_name != NULL; v++)
2614                 if (v->v_id == vid)
2615                         return (v);
2616
2617         return (NULL);
2618 }
2619
2620 static uint32_t
2621 bge_chipid(device_t dev)
2622 {
2623         uint32_t id;
2624
2625         id = pci_read_config(dev, BGE_PCI_MISC_CTL, 4) >>
2626             BGE_PCIMISCCTL_ASICREV_SHIFT;
2627         if (BGE_ASICREV(id) == BGE_ASICREV_USE_PRODID_REG) {
2628                 /*
2629                  * Find the ASCI revision.  Different chips use different
2630                  * registers.
2631                  */
2632                 switch (pci_get_device(dev)) {
2633                 case BCOM_DEVICEID_BCM5717:
2634                 case BCOM_DEVICEID_BCM5718:
2635                 case BCOM_DEVICEID_BCM5719:
2636                 case BCOM_DEVICEID_BCM5720:
2637                         id = pci_read_config(dev,
2638                             BGE_PCI_GEN2_PRODID_ASICREV, 4);
2639                         break;
2640                 case BCOM_DEVICEID_BCM57761:
2641                 case BCOM_DEVICEID_BCM57762:
2642                 case BCOM_DEVICEID_BCM57765:
2643                 case BCOM_DEVICEID_BCM57766:
2644                 case BCOM_DEVICEID_BCM57781:
2645                 case BCOM_DEVICEID_BCM57785:
2646                 case BCOM_DEVICEID_BCM57791:
2647                 case BCOM_DEVICEID_BCM57795:
2648                         id = pci_read_config(dev,
2649                             BGE_PCI_GEN15_PRODID_ASICREV, 4);
2650                         break;
2651                 default:
2652                         id = pci_read_config(dev, BGE_PCI_PRODID_ASICREV, 4);
2653                 }
2654         }
2655         return (id);
2656 }
2657
2658 /*
2659  * Probe for a Broadcom chip. Check the PCI vendor and device IDs
2660  * against our list and return its name if we find a match.
2661  *
2662  * Note that since the Broadcom controller contains VPD support, we
2663  * try to get the device name string from the controller itself instead
2664  * of the compiled-in string. It guarantees we'll always announce the
2665  * right product name. We fall back to the compiled-in string when
2666  * VPD is unavailable or corrupt.
2667  */
2668 static int
2669 bge_probe(device_t dev)
2670 {
2671         char buf[96];
2672         char model[64];
2673         const struct bge_revision *br;
2674         const char *pname;
2675         struct bge_softc *sc;
2676         const struct bge_type *t = bge_devs;
2677         const struct bge_vendor *v;
2678         uint32_t id;
2679         uint16_t did, vid;
2680
2681         sc = device_get_softc(dev);
2682         sc->bge_dev = dev;
2683         vid = pci_get_vendor(dev);
2684         did = pci_get_device(dev);
2685         while(t->bge_vid != 0) {
2686                 if ((vid == t->bge_vid) && (did == t->bge_did)) {
2687                         id = bge_chipid(dev);
2688                         br = bge_lookup_rev(id);
2689                         if (bge_has_eaddr(sc) &&
2690                             pci_get_vpd_ident(dev, &pname) == 0)
2691                                 snprintf(model, sizeof(model), "%s", pname);
2692                         else {
2693                                 v = bge_lookup_vendor(vid);
2694                                 snprintf(model, sizeof(model), "%s %s",
2695                                     v != NULL ? v->v_name : "Unknown",
2696                                     br != NULL ? br->br_name :
2697                                     "NetXtreme/NetLink Ethernet Controller");
2698                         }
2699                         snprintf(buf, sizeof(buf), "%s, %sASIC rev. %#08x",
2700                             model, br != NULL ? "" : "unknown ", id);
2701                         device_set_desc_copy(dev, buf);
2702                         return (BUS_PROBE_DEFAULT);
2703                 }
2704                 t++;
2705         }
2706
2707         return (ENXIO);
2708 }
2709
2710 static void
2711 bge_dma_free(struct bge_softc *sc)
2712 {
2713         int i;
2714
2715         /* Destroy DMA maps for RX buffers. */
2716         for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
2717                 if (sc->bge_cdata.bge_rx_std_dmamap[i])
2718                         bus_dmamap_destroy(sc->bge_cdata.bge_rx_mtag,
2719                             sc->bge_cdata.bge_rx_std_dmamap[i]);
2720         }
2721         if (sc->bge_cdata.bge_rx_std_sparemap)
2722                 bus_dmamap_destroy(sc->bge_cdata.bge_rx_mtag,
2723                     sc->bge_cdata.bge_rx_std_sparemap);
2724
2725         /* Destroy DMA maps for jumbo RX buffers. */
2726         for (i = 0; i < BGE_JUMBO_RX_RING_CNT; i++) {
2727                 if (sc->bge_cdata.bge_rx_jumbo_dmamap[i])
2728                         bus_dmamap_destroy(sc->bge_cdata.bge_mtag_jumbo,
2729                             sc->bge_cdata.bge_rx_jumbo_dmamap[i]);
2730         }
2731         if (sc->bge_cdata.bge_rx_jumbo_sparemap)
2732                 bus_dmamap_destroy(sc->bge_cdata.bge_mtag_jumbo,
2733                     sc->bge_cdata.bge_rx_jumbo_sparemap);
2734
2735         /* Destroy DMA maps for TX buffers. */
2736         for (i = 0; i < BGE_TX_RING_CNT; i++) {
2737                 if (sc->bge_cdata.bge_tx_dmamap[i])
2738                         bus_dmamap_destroy(sc->bge_cdata.bge_tx_mtag,
2739                             sc->bge_cdata.bge_tx_dmamap[i]);
2740         }
2741
2742         if (sc->bge_cdata.bge_rx_mtag)
2743                 bus_dma_tag_destroy(sc->bge_cdata.bge_rx_mtag);
2744         if (sc->bge_cdata.bge_mtag_jumbo)
2745                 bus_dma_tag_destroy(sc->bge_cdata.bge_mtag_jumbo);
2746         if (sc->bge_cdata.bge_tx_mtag)
2747                 bus_dma_tag_destroy(sc->bge_cdata.bge_tx_mtag);
2748
2749         /* Destroy standard RX ring. */
2750         if (sc->bge_cdata.bge_rx_std_ring_map)
2751                 bus_dmamap_unload(sc->bge_cdata.bge_rx_std_ring_tag,
2752                     sc->bge_cdata.bge_rx_std_ring_map);
2753         if (sc->bge_cdata.bge_rx_std_ring_map && sc->bge_ldata.bge_rx_std_ring)
2754                 bus_dmamem_free(sc->bge_cdata.bge_rx_std_ring_tag,
2755                     sc->bge_ldata.bge_rx_std_ring,
2756                     sc->bge_cdata.bge_rx_std_ring_map);
2757
2758         if (sc->bge_cdata.bge_rx_std_ring_tag)
2759                 bus_dma_tag_destroy(sc->bge_cdata.bge_rx_std_ring_tag);
2760
2761         /* Destroy jumbo RX ring. */
2762         if (sc->bge_cdata.bge_rx_jumbo_ring_map)
2763                 bus_dmamap_unload(sc->bge_cdata.bge_rx_jumbo_ring_tag,
2764                     sc->bge_cdata.bge_rx_jumbo_ring_map);
2765
2766         if (sc->bge_cdata.bge_rx_jumbo_ring_map &&
2767             sc->bge_ldata.bge_rx_jumbo_ring)
2768                 bus_dmamem_free(sc->bge_cdata.bge_rx_jumbo_ring_tag,
2769                     sc->bge_ldata.bge_rx_jumbo_ring,
2770                     sc->bge_cdata.bge_rx_jumbo_ring_map);
2771
2772         if (sc->bge_cdata.bge_rx_jumbo_ring_tag)
2773                 bus_dma_tag_destroy(sc->bge_cdata.bge_rx_jumbo_ring_tag);
2774
2775         /* Destroy RX return ring. */
2776         if (sc->bge_cdata.bge_rx_return_ring_map)
2777                 bus_dmamap_unload(sc->bge_cdata.bge_rx_return_ring_tag,
2778                     sc->bge_cdata.bge_rx_return_ring_map);
2779
2780         if (sc->bge_cdata.bge_rx_return_ring_map &&
2781             sc->bge_ldata.bge_rx_return_ring)
2782                 bus_dmamem_free(sc->bge_cdata.bge_rx_return_ring_tag,
2783                     sc->bge_ldata.bge_rx_return_ring,
2784                     sc->bge_cdata.bge_rx_return_ring_map);
2785
2786         if (sc->bge_cdata.bge_rx_return_ring_tag)
2787                 bus_dma_tag_destroy(sc->bge_cdata.bge_rx_return_ring_tag);
2788
2789         /* Destroy TX ring. */
2790         if (sc->bge_cdata.bge_tx_ring_map)
2791                 bus_dmamap_unload(sc->bge_cdata.bge_tx_ring_tag,
2792                     sc->bge_cdata.bge_tx_ring_map);
2793
2794         if (sc->bge_cdata.bge_tx_ring_map && sc->bge_ldata.bge_tx_ring)
2795                 bus_dmamem_free(sc->bge_cdata.bge_tx_ring_tag,
2796                     sc->bge_ldata.bge_tx_ring,
2797                     sc->bge_cdata.bge_tx_ring_map);
2798
2799         if (sc->bge_cdata.bge_tx_ring_tag)
2800                 bus_dma_tag_destroy(sc->bge_cdata.bge_tx_ring_tag);
2801
2802         /* Destroy status block. */
2803         if (sc->bge_cdata.bge_status_map)
2804                 bus_dmamap_unload(sc->bge_cdata.bge_status_tag,
2805                     sc->bge_cdata.bge_status_map);
2806
2807         if (sc->bge_cdata.bge_status_map && sc->bge_ldata.bge_status_block)
2808                 bus_dmamem_free(sc->bge_cdata.bge_status_tag,
2809                     sc->bge_ldata.bge_status_block,
2810                     sc->bge_cdata.bge_status_map);
2811
2812         if (sc->bge_cdata.bge_status_tag)
2813                 bus_dma_tag_destroy(sc->bge_cdata.bge_status_tag);
2814
2815         /* Destroy statistics block. */
2816         if (sc->bge_cdata.bge_stats_map)
2817                 bus_dmamap_unload(sc->bge_cdata.bge_stats_tag,
2818                     sc->bge_cdata.bge_stats_map);
2819
2820         if (sc->bge_cdata.bge_stats_map && sc->bge_ldata.bge_stats)
2821                 bus_dmamem_free(sc->bge_cdata.bge_stats_tag,
2822                     sc->bge_ldata.bge_stats,
2823                     sc->bge_cdata.bge_stats_map);
2824
2825         if (sc->bge_cdata.bge_stats_tag)
2826                 bus_dma_tag_destroy(sc->bge_cdata.bge_stats_tag);
2827
2828         if (sc->bge_cdata.bge_buffer_tag)
2829                 bus_dma_tag_destroy(sc->bge_cdata.bge_buffer_tag);
2830
2831         /* Destroy the parent tag. */
2832         if (sc->bge_cdata.bge_parent_tag)
2833                 bus_dma_tag_destroy(sc->bge_cdata.bge_parent_tag);
2834 }
2835
2836 static int
2837 bge_dma_ring_alloc(struct bge_softc *sc, bus_size_t alignment,
2838     bus_size_t maxsize, bus_dma_tag_t *tag, uint8_t **ring, bus_dmamap_t *map,
2839     bus_addr_t *paddr, const char *msg)
2840 {
2841         struct bge_dmamap_arg ctx;
2842         bus_addr_t lowaddr;
2843         bus_size_t ring_end;
2844         int error;
2845
2846         lowaddr = BUS_SPACE_MAXADDR;
2847 again:
2848         error = bus_dma_tag_create(sc->bge_cdata.bge_parent_tag,
2849             alignment, 0, lowaddr, BUS_SPACE_MAXADDR, NULL,
2850             NULL, maxsize, 1, maxsize, 0, NULL, NULL, tag);
2851         if (error != 0) {
2852                 device_printf(sc->bge_dev,
2853                     "could not create %s dma tag\n", msg);
2854                 return (ENOMEM);
2855         }
2856         /* Allocate DMA'able memory for ring. */
2857         error = bus_dmamem_alloc(*tag, (void **)ring,
2858             BUS_DMA_NOWAIT | BUS_DMA_ZERO | BUS_DMA_COHERENT, map);
2859         if (error != 0) {
2860                 device_printf(sc->bge_dev,
2861                     "could not allocate DMA'able memory for %s\n", msg);
2862                 return (ENOMEM);
2863         }
2864         /* Load the address of the ring. */
2865         ctx.bge_busaddr = 0;
2866         error = bus_dmamap_load(*tag, *map, *ring, maxsize, bge_dma_map_addr,
2867             &ctx, BUS_DMA_NOWAIT);
2868         if (error != 0) {
2869                 device_printf(sc->bge_dev,
2870                     "could not load DMA'able memory for %s\n", msg);
2871                 return (ENOMEM);
2872         }
2873         *paddr = ctx.bge_busaddr;
2874         ring_end = *paddr + maxsize;
2875         if ((sc->bge_flags & BGE_FLAG_4G_BNDRY_BUG) != 0 &&
2876             BGE_ADDR_HI(*paddr) != BGE_ADDR_HI(ring_end)) {
2877                 /*
2878                  * 4GB boundary crossed.  Limit maximum allowable DMA
2879                  * address space to 32bit and try again.
2880                  */
2881                 bus_dmamap_unload(*tag, *map);
2882                 bus_dmamem_free(*tag, *ring, *map);
2883                 bus_dma_tag_destroy(*tag);
2884                 if (bootverbose)
2885                         device_printf(sc->bge_dev, "4GB boundary crossed, "
2886                             "limit DMA address space to 32bit for %s\n", msg);
2887                 *ring = NULL;
2888                 *tag = NULL;
2889                 *map = NULL;
2890                 lowaddr = BUS_SPACE_MAXADDR_32BIT;
2891                 goto again;
2892         }
2893         return (0);
2894 }
2895
2896 static int
2897 bge_dma_alloc(struct bge_softc *sc)
2898 {
2899         bus_addr_t lowaddr;
2900         bus_size_t boundary, sbsz, rxmaxsegsz, txsegsz, txmaxsegsz;
2901         int i, error;
2902
2903         lowaddr = BUS_SPACE_MAXADDR;
2904         if ((sc->bge_flags & BGE_FLAG_40BIT_BUG) != 0)
2905                 lowaddr = BGE_DMA_MAXADDR;
2906         /*
2907          * Allocate the parent bus DMA tag appropriate for PCI.
2908          */
2909         error = bus_dma_tag_create(bus_get_dma_tag(sc->bge_dev),
2910             1, 0, lowaddr, BUS_SPACE_MAXADDR, NULL,
2911             NULL, BUS_SPACE_MAXSIZE_32BIT, 0, BUS_SPACE_MAXSIZE_32BIT,
2912             0, NULL, NULL, &sc->bge_cdata.bge_parent_tag);
2913         if (error != 0) {
2914                 device_printf(sc->bge_dev,
2915                     "could not allocate parent dma tag\n");
2916                 return (ENOMEM);
2917         }
2918
2919         /* Create tag for standard RX ring. */
2920         error = bge_dma_ring_alloc(sc, PAGE_SIZE, BGE_STD_RX_RING_SZ,
2921             &sc->bge_cdata.bge_rx_std_ring_tag,
2922             (uint8_t **)&sc->bge_ldata.bge_rx_std_ring,
2923             &sc->bge_cdata.bge_rx_std_ring_map,
2924             &sc->bge_ldata.bge_rx_std_ring_paddr, "RX ring");
2925         if (error)
2926                 return (error);
2927
2928         /* Create tag for RX return ring. */
2929         error = bge_dma_ring_alloc(sc, PAGE_SIZE, BGE_RX_RTN_RING_SZ(sc),
2930             &sc->bge_cdata.bge_rx_return_ring_tag,
2931             (uint8_t **)&sc->bge_ldata.bge_rx_return_ring,
2932             &sc->bge_cdata.bge_rx_return_ring_map,
2933             &sc->bge_ldata.bge_rx_return_ring_paddr, "RX return ring");
2934         if (error)
2935                 return (error);
2936
2937         /* Create tag for TX ring. */
2938         error = bge_dma_ring_alloc(sc, PAGE_SIZE, BGE_TX_RING_SZ,
2939             &sc->bge_cdata.bge_tx_ring_tag,
2940             (uint8_t **)&sc->bge_ldata.bge_tx_ring,
2941             &sc->bge_cdata.bge_tx_ring_map,
2942             &sc->bge_ldata.bge_tx_ring_paddr, "TX ring");
2943         if (error)
2944                 return (error);
2945
2946         /*
2947          * Create tag for status block.
2948          * Because we only use single Tx/Rx/Rx return ring, use
2949          * minimum status block size except BCM5700 AX/BX which
2950          * seems to want to see full status block size regardless
2951          * of configured number of ring.
2952          */
2953         if (sc->bge_asicrev == BGE_ASICREV_BCM5700 &&
2954             sc->bge_chipid != BGE_CHIPID_BCM5700_C0)
2955                 sbsz = BGE_STATUS_BLK_SZ;
2956         else
2957                 sbsz = 32;
2958         error = bge_dma_ring_alloc(sc, PAGE_SIZE, sbsz,
2959             &sc->bge_cdata.bge_status_tag,
2960             (uint8_t **)&sc->bge_ldata.bge_status_block,
2961             &sc->bge_cdata.bge_status_map,
2962             &sc->bge_ldata.bge_status_block_paddr, "status block");
2963         if (error)
2964                 return (error);
2965
2966         /* Create tag for statistics block. */
2967         error = bge_dma_ring_alloc(sc, PAGE_SIZE, BGE_STATS_SZ,
2968             &sc->bge_cdata.bge_stats_tag,
2969             (uint8_t **)&sc->bge_ldata.bge_stats,
2970             &sc->bge_cdata.bge_stats_map,
2971             &sc->bge_ldata.bge_stats_paddr, "statistics block");
2972         if (error)
2973                 return (error);
2974
2975         /* Create tag for jumbo RX ring. */
2976         if (BGE_IS_JUMBO_CAPABLE(sc)) {
2977                 error = bge_dma_ring_alloc(sc, PAGE_SIZE, BGE_JUMBO_RX_RING_SZ,
2978                     &sc->bge_cdata.bge_rx_jumbo_ring_tag,
2979                     (uint8_t **)&sc->bge_ldata.bge_rx_jumbo_ring,
2980                     &sc->bge_cdata.bge_rx_jumbo_ring_map,
2981                     &sc->bge_ldata.bge_rx_jumbo_ring_paddr, "jumbo RX ring");
2982                 if (error)
2983                         return (error);
2984         }
2985
2986         /* Create parent tag for buffers. */
2987         boundary = 0;
2988         if ((sc->bge_flags & BGE_FLAG_4G_BNDRY_BUG) != 0) {
2989                 boundary = BGE_DMA_BNDRY;
2990                 /*
2991                  * XXX
2992                  * watchdog timeout issue was observed on BCM5704 which
2993                  * lives behind PCI-X bridge(e.g AMD 8131 PCI-X bridge).
2994                  * Both limiting DMA address space to 32bits and flushing
2995                  * mailbox write seem to address the issue.
2996                  */
2997                 if (sc->bge_pcixcap != 0)
2998                         lowaddr = BUS_SPACE_MAXADDR_32BIT;
2999         }
3000         error = bus_dma_tag_create(bus_get_dma_tag(sc->bge_dev),
3001             1, boundary, lowaddr, BUS_SPACE_MAXADDR, NULL,
3002             NULL, BUS_SPACE_MAXSIZE_32BIT, 0, BUS_SPACE_MAXSIZE_32BIT,
3003             0, NULL, NULL, &sc->bge_cdata.bge_buffer_tag);
3004         if (error != 0) {
3005                 device_printf(sc->bge_dev,
3006                     "could not allocate buffer dma tag\n");
3007                 return (ENOMEM);
3008         }
3009         /* Create tag for Tx mbufs. */
3010         if (sc->bge_flags & (BGE_FLAG_TSO | BGE_FLAG_TSO3)) {
3011                 txsegsz = BGE_TSOSEG_SZ;
3012                 txmaxsegsz = 65535 + sizeof(struct ether_vlan_header);
3013         } else {
3014                 txsegsz = MCLBYTES;
3015                 txmaxsegsz = MCLBYTES * BGE_NSEG_NEW;
3016         }
3017         error = bus_dma_tag_create(sc->bge_cdata.bge_buffer_tag, 1,
3018             0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
3019             txmaxsegsz, BGE_NSEG_NEW, txsegsz, 0, NULL, NULL,
3020             &sc->bge_cdata.bge_tx_mtag);
3021
3022         if (error) {
3023                 device_printf(sc->bge_dev, "could not allocate TX dma tag\n");
3024                 return (ENOMEM);
3025         }
3026
3027         /* Create tag for Rx mbufs. */
3028         if (sc->bge_flags & BGE_FLAG_JUMBO_STD)
3029                 rxmaxsegsz = MJUM9BYTES;
3030         else
3031                 rxmaxsegsz = MCLBYTES;
3032         error = bus_dma_tag_create(sc->bge_cdata.bge_buffer_tag, 1, 0,
3033             BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, rxmaxsegsz, 1,
3034             rxmaxsegsz, 0, NULL, NULL, &sc->bge_cdata.bge_rx_mtag);
3035
3036         if (error) {
3037                 device_printf(sc->bge_dev, "could not allocate RX dma tag\n");
3038                 return (ENOMEM);
3039         }
3040
3041         /* Create DMA maps for RX buffers. */
3042         error = bus_dmamap_create(sc->bge_cdata.bge_rx_mtag, 0,
3043             &sc->bge_cdata.bge_rx_std_sparemap);
3044         if (error) {
3045                 device_printf(sc->bge_dev,
3046                     "can't create spare DMA map for RX\n");
3047                 return (ENOMEM);
3048         }
3049         for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
3050                 error = bus_dmamap_create(sc->bge_cdata.bge_rx_mtag, 0,
3051                             &sc->bge_cdata.bge_rx_std_dmamap[i]);
3052                 if (error) {
3053                         device_printf(sc->bge_dev,
3054                             "can't create DMA map for RX\n");
3055                         return (ENOMEM);
3056                 }
3057         }
3058
3059         /* Create DMA maps for TX buffers. */
3060         for (i = 0; i < BGE_TX_RING_CNT; i++) {
3061                 error = bus_dmamap_create(sc->bge_cdata.bge_tx_mtag, 0,
3062                             &sc->bge_cdata.bge_tx_dmamap[i]);
3063                 if (error) {
3064                         device_printf(sc->bge_dev,
3065                             "can't create DMA map for TX\n");
3066                         return (ENOMEM);
3067                 }
3068         }
3069
3070         /* Create tags for jumbo RX buffers. */
3071         if (BGE_IS_JUMBO_CAPABLE(sc)) {
3072                 error = bus_dma_tag_create(sc->bge_cdata.bge_buffer_tag,
3073                     1, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL,
3074                     NULL, MJUM9BYTES, BGE_NSEG_JUMBO, PAGE_SIZE,
3075                     0, NULL, NULL, &sc->bge_cdata.bge_mtag_jumbo);
3076                 if (error) {
3077                         device_printf(sc->bge_dev,
3078                             "could not allocate jumbo dma tag\n");
3079                         return (ENOMEM);
3080                 }
3081                 /* Create DMA maps for jumbo RX buffers. */
3082                 error = bus_dmamap_create(sc->bge_cdata.bge_mtag_jumbo,
3083                     0, &sc->bge_cdata.bge_rx_jumbo_sparemap);
3084                 if (error) {
3085                         device_printf(sc->bge_dev,
3086                             "can't create spare DMA map for jumbo RX\n");
3087                         return (ENOMEM);
3088                 }
3089                 for (i = 0; i < BGE_JUMBO_RX_RING_CNT; i++) {
3090                         error = bus_dmamap_create(sc->bge_cdata.bge_mtag_jumbo,
3091                                     0, &sc->bge_cdata.bge_rx_jumbo_dmamap[i]);
3092                         if (error) {
3093                                 device_printf(sc->bge_dev,
3094                                     "can't create DMA map for jumbo RX\n");
3095                                 return (ENOMEM);
3096                         }
3097                 }
3098         }
3099
3100         return (0);
3101 }
3102
3103 /*
3104  * Return true if this device has more than one port.
3105  */
3106 static int
3107 bge_has_multiple_ports(struct bge_softc *sc)
3108 {
3109         device_t dev = sc->bge_dev;
3110         u_int b, d, f, fscan, s;
3111
3112         d = pci_get_domain(dev);
3113         b = pci_get_bus(dev);
3114         s = pci_get_slot(dev);
3115         f = pci_get_function(dev);
3116         for (fscan = 0; fscan <= PCI_FUNCMAX; fscan++)
3117                 if (fscan != f && pci_find_dbsf(d, b, s, fscan) != NULL)
3118                         return (1);
3119         return (0);
3120 }
3121
3122 /*
3123  * Return true if MSI can be used with this device.
3124  */
3125 static int
3126 bge_can_use_msi(struct bge_softc *sc)
3127 {
3128         int can_use_msi = 0;
3129
3130         if (sc->bge_msi == 0)
3131                 return (0);
3132
3133         /* Disable MSI for polling(4). */
3134 #ifdef DEVICE_POLLING
3135         return (0);
3136 #endif
3137         switch (sc->bge_asicrev) {
3138         case BGE_ASICREV_BCM5714_A0:
3139         case BGE_ASICREV_BCM5714:
3140                 /*
3141                  * Apparently, MSI doesn't work when these chips are
3142                  * configured in single-port mode.
3143                  */
3144                 if (bge_has_multiple_ports(sc))
3145                         can_use_msi = 1;
3146                 break;
3147         case BGE_ASICREV_BCM5750:
3148                 if (sc->bge_chiprev != BGE_CHIPREV_5750_AX &&
3149                     sc->bge_chiprev != BGE_CHIPREV_5750_BX)
3150                         can_use_msi = 1;
3151                 break;
3152         default:
3153                 if (BGE_IS_575X_PLUS(sc))
3154                         can_use_msi = 1;
3155         }
3156         return (can_use_msi);
3157 }
3158
3159 static int
3160 bge_mbox_reorder(struct bge_softc *sc)
3161 {
3162         /* Lists of PCI bridges that are known to reorder mailbox writes. */
3163         static const struct mbox_reorder {
3164                 const uint16_t vendor;
3165                 const uint16_t device;
3166                 const char *desc;
3167         } mbox_reorder_lists[] = {
3168                 { 0x1022, 0x7450, "AMD-8131 PCI-X Bridge" },
3169         };
3170         devclass_t pci, pcib;
3171         device_t bus, dev;
3172         int i;
3173
3174         pci = devclass_find("pci");
3175         pcib = devclass_find("pcib");
3176         dev = sc->bge_dev;
3177         bus = device_get_parent(dev);
3178         for (;;) {
3179                 dev = device_get_parent(bus);
3180                 bus = device_get_parent(dev);
3181                 if (device_get_devclass(dev) != pcib)
3182                         break;
3183                 for (i = 0; i < nitems(mbox_reorder_lists); i++) {
3184                         if (pci_get_vendor(dev) ==
3185                             mbox_reorder_lists[i].vendor &&
3186                             pci_get_device(dev) ==
3187                             mbox_reorder_lists[i].device) {
3188                                 device_printf(sc->bge_dev,
3189                                     "enabling MBOX workaround for %s\n",
3190                                     mbox_reorder_lists[i].desc);
3191                                 return (1);
3192                         }
3193                 }
3194                 if (device_get_devclass(bus) != pci)
3195                         break;
3196         }
3197         return (0);
3198 }
3199
3200 static void
3201 bge_devinfo(struct bge_softc *sc)
3202 {
3203         uint32_t cfg, clk;
3204
3205         device_printf(sc->bge_dev,
3206             "CHIP ID 0x%08x; ASIC REV 0x%02x; CHIP REV 0x%02x; ",
3207             sc->bge_chipid, sc->bge_asicrev, sc->bge_chiprev);
3208         if (sc->bge_flags & BGE_FLAG_PCIE)
3209                 printf("PCI-E\n");
3210         else if (sc->bge_flags & BGE_FLAG_PCIX) {
3211                 printf("PCI-X ");
3212                 cfg = CSR_READ_4(sc, BGE_MISC_CFG) & BGE_MISCCFG_BOARD_ID_MASK;
3213                 if (cfg == BGE_MISCCFG_BOARD_ID_5704CIOBE)
3214                         clk = 133;
3215                 else {
3216                         clk = CSR_READ_4(sc, BGE_PCI_CLKCTL) & 0x1F;
3217                         switch (clk) {
3218                         case 0:
3219                                 clk = 33;
3220                                 break;
3221                         case 2:
3222                                 clk = 50;
3223                                 break;
3224                         case 4:
3225                                 clk = 66;
3226                                 break;
3227                         case 6:
3228                                 clk = 100;
3229                                 break;
3230                         case 7:
3231                                 clk = 133;
3232                                 break;
3233                         }
3234                 }
3235                 printf("%u MHz\n", clk);
3236         } else {
3237                 if (sc->bge_pcixcap != 0)
3238                         printf("PCI on PCI-X ");
3239                 else
3240                         printf("PCI ");
3241                 cfg = pci_read_config(sc->bge_dev, BGE_PCI_PCISTATE, 4);
3242                 if (cfg & BGE_PCISTATE_PCI_BUSSPEED)
3243                         clk = 66;
3244                 else
3245                         clk = 33;
3246                 if (cfg & BGE_PCISTATE_32BIT_BUS)
3247                         printf("%u MHz; 32bit\n", clk);
3248                 else
3249                         printf("%u MHz; 64bit\n", clk);
3250         }
3251 }
3252
3253 static int
3254 bge_attach(device_t dev)
3255 {
3256         struct ifnet *ifp;
3257         struct bge_softc *sc;
3258         uint32_t hwcfg = 0, misccfg, pcistate;
3259         u_char eaddr[ETHER_ADDR_LEN];
3260         int capmask, error, msicount, reg, rid, trys;
3261
3262         sc = device_get_softc(dev);
3263         sc->bge_dev = dev;
3264
3265         BGE_LOCK_INIT(sc, device_get_nameunit(dev));
3266         TASK_INIT(&sc->bge_intr_task, 0, bge_intr_task, sc);
3267         callout_init_mtx(&sc->bge_stat_ch, &sc->bge_mtx, 0);
3268
3269         /*
3270          * Map control/status registers.
3271          */
3272         pci_enable_busmaster(dev);
3273
3274         rid = PCIR_BAR(0);
3275         sc->bge_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
3276             RF_ACTIVE);
3277
3278         if (sc->bge_res == NULL) {
3279                 device_printf (sc->bge_dev, "couldn't map BAR0 memory\n");
3280                 error = ENXIO;
3281                 goto fail;
3282         }
3283
3284         /* Save various chip information. */
3285         sc->bge_func_addr = pci_get_function(dev);
3286         sc->bge_chipid = bge_chipid(dev);
3287         sc->bge_asicrev = BGE_ASICREV(sc->bge_chipid);
3288         sc->bge_chiprev = BGE_CHIPREV(sc->bge_chipid);
3289
3290         /* Set default PHY address. */
3291         sc->bge_phy_addr = 1;
3292          /*
3293           * PHY address mapping for various devices.
3294           *
3295           *          | F0 Cu | F0 Sr | F1 Cu | F1 Sr |
3296           * ---------+-------+-------+-------+-------+
3297           * BCM57XX  |   1   |   X   |   X   |   X   |
3298           * BCM5704  |   1   |   X   |   1   |   X   |
3299           * BCM5717  |   1   |   8   |   2   |   9   |
3300           * BCM5719  |   1   |   8   |   2   |   9   |
3301           * BCM5720  |   1   |   8   |   2   |   9   |
3302           *
3303           *          | F2 Cu | F2 Sr | F3 Cu | F3 Sr |
3304           * ---------+-------+-------+-------+-------+
3305           * BCM57XX  |   X   |   X   |   X   |   X   |
3306           * BCM5704  |   X   |   X   |   X   |   X   |
3307           * BCM5717  |   X   |   X   |   X   |   X   |
3308           * BCM5719  |   3   |   10  |   4   |   11  |
3309           * BCM5720  |   X   |   X   |   X   |   X   |
3310           *
3311           * Other addresses may respond but they are not
3312           * IEEE compliant PHYs and should be ignored.
3313           */
3314         if (sc->bge_asicrev == BGE_ASICREV_BCM5717 ||
3315             sc->bge_asicrev == BGE_ASICREV_BCM5719 ||
3316             sc->bge_asicrev == BGE_ASICREV_BCM5720) {
3317                 if (sc->bge_chipid != BGE_CHIPID_BCM5717_A0) {
3318                         if (CSR_READ_4(sc, BGE_SGDIG_STS) &
3319                             BGE_SGDIGSTS_IS_SERDES)
3320                                 sc->bge_phy_addr = sc->bge_func_addr + 8;
3321                         else
3322                                 sc->bge_phy_addr = sc->bge_func_addr + 1;
3323                 } else {
3324                         if (CSR_READ_4(sc, BGE_CPMU_PHY_STRAP) &
3325                             BGE_CPMU_PHY_STRAP_IS_SERDES)
3326                                 sc->bge_phy_addr = sc->bge_func_addr + 8;
3327                         else
3328                                 sc->bge_phy_addr = sc->bge_func_addr + 1;
3329                 }
3330         }
3331
3332         if (bge_has_eaddr(sc))
3333                 sc->bge_flags |= BGE_FLAG_EADDR;
3334
3335         /* Save chipset family. */
3336         switch (sc->bge_asicrev) {
3337         case BGE_ASICREV_BCM57765:
3338         case BGE_ASICREV_BCM57766:
3339                 sc->bge_flags |= BGE_FLAG_57765_PLUS;
3340                 /* FALLTHROUGH */
3341         case BGE_ASICREV_BCM5717:
3342         case BGE_ASICREV_BCM5719:
3343         case BGE_ASICREV_BCM5720:
3344                 sc->bge_flags |= BGE_FLAG_5717_PLUS | BGE_FLAG_5755_PLUS |
3345                     BGE_FLAG_575X_PLUS | BGE_FLAG_5705_PLUS | BGE_FLAG_JUMBO |
3346                     BGE_FLAG_JUMBO_FRAME;
3347                 if (sc->bge_asicrev == BGE_ASICREV_BCM5719 &&
3348                     sc->bge_chipid == BGE_CHIPID_BCM5719_A0) {
3349                         /* Jumbo frame on BCM5719 A0 does not work. */
3350                         sc->bge_flags &= ~BGE_FLAG_JUMBO;
3351                 }
3352                 break;
3353         case BGE_ASICREV_BCM5755:
3354         case BGE_ASICREV_BCM5761:
3355         case BGE_ASICREV_BCM5784:
3356         case BGE_ASICREV_BCM5785:
3357         case BGE_ASICREV_BCM5787:
3358         case BGE_ASICREV_BCM57780:
3359                 sc->bge_flags |= BGE_FLAG_5755_PLUS | BGE_FLAG_575X_PLUS |
3360                     BGE_FLAG_5705_PLUS;
3361                 break;
3362         case BGE_ASICREV_BCM5700:
3363         case BGE_ASICREV_BCM5701:
3364         case BGE_ASICREV_BCM5703:
3365         case BGE_ASICREV_BCM5704:
3366                 sc->bge_flags |= BGE_FLAG_5700_FAMILY | BGE_FLAG_JUMBO;
3367                 break;
3368         case BGE_ASICREV_BCM5714_A0:
3369         case BGE_ASICREV_BCM5780:
3370         case BGE_ASICREV_BCM5714:
3371                 sc->bge_flags |= BGE_FLAG_5714_FAMILY | BGE_FLAG_JUMBO_STD;
3372                 /* FALLTHROUGH */
3373         case BGE_ASICREV_BCM5750:
3374         case BGE_ASICREV_BCM5752:
3375         case BGE_ASICREV_BCM5906:
3376                 sc->bge_flags |= BGE_FLAG_575X_PLUS;
3377                 /* FALLTHROUGH */
3378         case BGE_ASICREV_BCM5705:
3379                 sc->bge_flags |= BGE_FLAG_5705_PLUS;
3380                 break;
3381         }
3382
3383         /* Identify chips with APE processor. */
3384         switch (sc->bge_asicrev) {
3385         case BGE_ASICREV_BCM5717:
3386         case BGE_ASICREV_BCM5719:
3387         case BGE_ASICREV_BCM5720:
3388         case BGE_ASICREV_BCM5761:
3389                 sc->bge_flags |= BGE_FLAG_APE;
3390                 break;
3391         }
3392
3393         /* Chips with APE need BAR2 access for APE registers/memory. */
3394         if ((sc->bge_flags & BGE_FLAG_APE) != 0) {
3395                 rid = PCIR_BAR(2);
3396                 sc->bge_res2 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
3397                     RF_ACTIVE);
3398                 if (sc->bge_res2 == NULL) {
3399                         device_printf (sc->bge_dev,
3400                             "couldn't map BAR2 memory\n");
3401                         error = ENXIO;
3402                         goto fail;
3403                 }
3404
3405                 /* Enable APE register/memory access by host driver. */
3406                 pcistate = pci_read_config(dev, BGE_PCI_PCISTATE, 4);
3407                 pcistate |= BGE_PCISTATE_ALLOW_APE_CTLSPC_WR |
3408                     BGE_PCISTATE_ALLOW_APE_SHMEM_WR |
3409                     BGE_PCISTATE_ALLOW_APE_PSPACE_WR;
3410                 pci_write_config(dev, BGE_PCI_PCISTATE, pcistate, 4);
3411
3412                 bge_ape_lock_init(sc);
3413                 bge_ape_read_fw_ver(sc);
3414         }
3415
3416         /* Add SYSCTLs, requires the chipset family to be set. */
3417         bge_add_sysctls(sc);
3418
3419         /* Identify the chips that use an CPMU. */
3420         if (BGE_IS_5717_PLUS(sc) ||
3421             sc->bge_asicrev == BGE_ASICREV_BCM5784 ||
3422             sc->bge_asicrev == BGE_ASICREV_BCM5761 ||
3423             sc->bge_asicrev == BGE_ASICREV_BCM5785 ||
3424             sc->bge_asicrev == BGE_ASICREV_BCM57780)
3425                 sc->bge_flags |= BGE_FLAG_CPMU_PRESENT;
3426         if ((sc->bge_flags & BGE_FLAG_CPMU_PRESENT) != 0)
3427                 sc->bge_mi_mode = BGE_MIMODE_500KHZ_CONST;
3428         else
3429                 sc->bge_mi_mode = BGE_MIMODE_BASE;
3430         /* Enable auto polling for BCM570[0-5]. */
3431         if (BGE_IS_5700_FAMILY(sc) || sc->bge_asicrev == BGE_ASICREV_BCM5705)
3432                 sc->bge_mi_mode |= BGE_MIMODE_AUTOPOLL;
3433
3434         /*
3435          * All Broadcom controllers have 4GB boundary DMA bug.
3436          * Whenever an address crosses a multiple of the 4GB boundary
3437          * (including 4GB, 8Gb, 12Gb, etc.) and makes the transition
3438          * from 0xX_FFFF_FFFF to 0x(X+1)_0000_0000 an internal DMA
3439          * state machine will lockup and cause the device to hang.
3440          */
3441         sc->bge_flags |= BGE_FLAG_4G_BNDRY_BUG;
3442
3443         /* BCM5755 or higher and BCM5906 have short DMA bug. */
3444         if (BGE_IS_5755_PLUS(sc) || sc->bge_asicrev == BGE_ASICREV_BCM5906)
3445                 sc->bge_flags |= BGE_FLAG_SHORT_DMA_BUG;
3446
3447         /*
3448          * BCM5719 cannot handle DMA requests for DMA segments that
3449          * have larger than 4KB in size.  However the maximum DMA
3450          * segment size created in DMA tag is 4KB for TSO, so we
3451          * wouldn't encounter the issue here.
3452          */
3453         if (sc->bge_asicrev == BGE_ASICREV_BCM5719)
3454                 sc->bge_flags |= BGE_FLAG_4K_RDMA_BUG;
3455
3456         misccfg = CSR_READ_4(sc, BGE_MISC_CFG) & BGE_MISCCFG_BOARD_ID_MASK;
3457         if (sc->bge_asicrev == BGE_ASICREV_BCM5705) {
3458                 if (misccfg == BGE_MISCCFG_BOARD_ID_5788 ||
3459                     misccfg == BGE_MISCCFG_BOARD_ID_5788M)
3460                         sc->bge_flags |= BGE_FLAG_5788;
3461         }
3462
3463         capmask = BMSR_DEFCAPMASK;
3464         if ((sc->bge_asicrev == BGE_ASICREV_BCM5703 &&
3465             (misccfg == 0x4000 || misccfg == 0x8000)) ||
3466             (sc->bge_asicrev == BGE_ASICREV_BCM5705 &&
3467             pci_get_vendor(dev) == BCOM_VENDORID &&
3468             (pci_get_device(dev) == BCOM_DEVICEID_BCM5901 ||
3469             pci_get_device(dev) == BCOM_DEVICEID_BCM5901A2 ||
3470             pci_get_device(dev) == BCOM_DEVICEID_BCM5705F)) ||
3471             (pci_get_vendor(dev) == BCOM_VENDORID &&
3472             (pci_get_device(dev) == BCOM_DEVICEID_BCM5751F ||
3473             pci_get_device(dev) == BCOM_DEVICEID_BCM5753F ||
3474             pci_get_device(dev) == BCOM_DEVICEID_BCM5787F)) ||
3475             pci_get_device(dev) == BCOM_DEVICEID_BCM57790 ||
3476             pci_get_device(dev) == BCOM_DEVICEID_BCM57791 ||
3477             pci_get_device(dev) == BCOM_DEVICEID_BCM57795 ||
3478             sc->bge_asicrev == BGE_ASICREV_BCM5906) {
3479                 /* These chips are 10/100 only. */
3480                 capmask &= ~BMSR_EXTSTAT;
3481                 sc->bge_phy_flags |= BGE_PHY_NO_WIRESPEED;
3482         }
3483
3484         /*
3485          * Some controllers seem to require a special firmware to use
3486          * TSO. But the firmware is not available to FreeBSD and Linux
3487          * claims that the TSO performed by the firmware is slower than
3488          * hardware based TSO. Moreover the firmware based TSO has one
3489          * known bug which can't handle TSO if Ethernet header + IP/TCP
3490          * header is greater than 80 bytes. A workaround for the TSO
3491          * bug exist but it seems it's too expensive than not using
3492          * TSO at all. Some hardwares also have the TSO bug so limit
3493          * the TSO to the controllers that are not affected TSO issues
3494          * (e.g. 5755 or higher).
3495          */
3496         if (BGE_IS_5717_PLUS(sc)) {
3497                 /* BCM5717 requires different TSO configuration. */
3498                 sc->bge_flags |= BGE_FLAG_TSO3;
3499                 if (sc->bge_asicrev == BGE_ASICREV_BCM5719 &&
3500                     sc->bge_chipid == BGE_CHIPID_BCM5719_A0) {
3501                         /* TSO on BCM5719 A0 does not work. */
3502                         sc->bge_flags &= ~BGE_FLAG_TSO3;
3503                 }
3504         } else if (BGE_IS_5755_PLUS(sc)) {
3505                 /*
3506                  * BCM5754 and BCM5787 shares the same ASIC id so
3507                  * explicit device id check is required.
3508                  * Due to unknown reason TSO does not work on BCM5755M.
3509                  */
3510                 if (pci_get_device(dev) != BCOM_DEVICEID_BCM5754 &&
3511                     pci_get_device(dev) != BCOM_DEVICEID_BCM5754M &&
3512                     pci_get_device(dev) != BCOM_DEVICEID_BCM5755M)
3513                         sc->bge_flags |= BGE_FLAG_TSO;
3514         }
3515
3516         /*
3517          * Check if this is a PCI-X or PCI Express device.
3518          */
3519         if (pci_find_cap(dev, PCIY_EXPRESS, &reg) == 0) {
3520                 /*
3521                  * Found a PCI Express capabilities register, this
3522                  * must be a PCI Express device.
3523                  */
3524                 sc->bge_flags |= BGE_FLAG_PCIE;
3525                 sc->bge_expcap = reg;
3526                 /* Extract supported maximum payload size. */
3527                 sc->bge_mps = pci_read_config(dev, sc->bge_expcap +
3528                     PCIER_DEVICE_CAP, 2);
3529                 sc->bge_mps = 128 << (sc->bge_mps & PCIEM_CAP_MAX_PAYLOAD);
3530                 if (sc->bge_asicrev == BGE_ASICREV_BCM5719 ||
3531                     sc->bge_asicrev == BGE_ASICREV_BCM5720)
3532                         sc->bge_expmrq = 2048;
3533                 else
3534                         sc->bge_expmrq = 4096;
3535                 pci_set_max_read_req(dev, sc->bge_expmrq);
3536         } else {
3537                 /*
3538                  * Check if the device is in PCI-X Mode.
3539                  * (This bit is not valid on PCI Express controllers.)
3540                  */
3541                 if (pci_find_cap(dev, PCIY_PCIX, &reg) == 0)
3542                         sc->bge_pcixcap = reg;
3543                 if ((pci_read_config(dev, BGE_PCI_PCISTATE, 4) &
3544                     BGE_PCISTATE_PCI_BUSMODE) == 0)
3545                         sc->bge_flags |= BGE_FLAG_PCIX;
3546         }
3547
3548         /*
3549          * The 40bit DMA bug applies to the 5714/5715 controllers and is
3550          * not actually a MAC controller bug but an issue with the embedded
3551          * PCIe to PCI-X bridge in the device. Use 40bit DMA workaround.
3552          */
3553         if (BGE_IS_5714_FAMILY(sc) && (sc->bge_flags & BGE_FLAG_PCIX))
3554                 sc->bge_flags |= BGE_FLAG_40BIT_BUG;
3555         /*
3556          * Some PCI-X bridges are known to trigger write reordering to
3557          * the mailbox registers. Typical phenomena is watchdog timeouts
3558          * caused by out-of-order TX completions.  Enable workaround for
3559          * PCI-X devices that live behind these bridges.
3560          * Note, PCI-X controllers can run in PCI mode so we can't use
3561          * BGE_FLAG_PCIX flag to detect PCI-X controllers.
3562          */
3563         if (sc->bge_pcixcap != 0 && bge_mbox_reorder(sc) != 0)
3564                 sc->bge_flags |= BGE_FLAG_MBOX_REORDER;
3565         /*
3566          * Allocate the interrupt, using MSI if possible.  These devices
3567          * support 8 MSI messages, but only the first one is used in
3568          * normal operation.
3569          */
3570         rid = 0;
3571         if (pci_find_cap(sc->bge_dev, PCIY_MSI, &reg) == 0) {
3572                 sc->bge_msicap = reg;
3573                 if (bge_can_use_msi(sc)) {
3574                         msicount = pci_msi_count(dev);
3575                         if (msicount > 1)
3576                                 msicount = 1;
3577                 } else
3578                         msicount = 0;
3579                 if (msicount == 1 && pci_alloc_msi(dev, &msicount) == 0) {
3580                         rid = 1;
3581                         sc->bge_flags |= BGE_FLAG_MSI;
3582                 }
3583         }
3584
3585         /*
3586          * All controllers except BCM5700 supports tagged status but
3587          * we use tagged status only for MSI case on BCM5717. Otherwise
3588          * MSI on BCM5717 does not work.
3589          */
3590 #ifndef DEVICE_POLLING
3591         if (sc->bge_flags & BGE_FLAG_MSI && BGE_IS_5717_PLUS(sc))
3592                 sc->bge_flags |= BGE_FLAG_TAGGED_STATUS;
3593 #endif
3594
3595         sc->bge_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
3596             RF_SHAREABLE | RF_ACTIVE);
3597
3598         if (sc->bge_irq == NULL) {
3599                 device_printf(sc->bge_dev, "couldn't map interrupt\n");
3600                 error = ENXIO;
3601                 goto fail;
3602         }
3603
3604         bge_devinfo(sc);
3605
3606         sc->bge_asf_mode = 0;
3607         /* No ASF if APE present. */
3608         if ((sc->bge_flags & BGE_FLAG_APE) == 0) {
3609                 if (bge_allow_asf && (bge_readmem_ind(sc, BGE_SRAM_DATA_SIG) ==
3610                     BGE_SRAM_DATA_SIG_MAGIC)) {
3611                         if (bge_readmem_ind(sc, BGE_SRAM_DATA_CFG) &
3612                             BGE_HWCFG_ASF) {
3613                                 sc->bge_asf_mode |= ASF_ENABLE;
3614                                 sc->bge_asf_mode |= ASF_STACKUP;
3615                                 if (BGE_IS_575X_PLUS(sc))
3616                                         sc->bge_asf_mode |= ASF_NEW_HANDSHAKE;
3617                         }
3618                 }
3619         }
3620
3621         bge_stop_fw(sc);
3622         bge_sig_pre_reset(sc, BGE_RESET_SHUTDOWN);
3623         if (bge_reset(sc)) {
3624                 device_printf(sc->bge_dev, "chip reset failed\n");
3625                 error = ENXIO;
3626                 goto fail;
3627         }
3628
3629         bge_sig_legacy(sc, BGE_RESET_SHUTDOWN);
3630         bge_sig_post_reset(sc, BGE_RESET_SHUTDOWN);
3631
3632         if (bge_chipinit(sc)) {
3633                 device_printf(sc->bge_dev, "chip initialization failed\n");
3634                 error = ENXIO;
3635                 goto fail;
3636         }
3637
3638         error = bge_get_eaddr(sc, eaddr);
3639         if (error) {
3640                 device_printf(sc->bge_dev,
3641                     "failed to read station address\n");
3642                 error = ENXIO;
3643                 goto fail;
3644         }
3645
3646         /* 5705 limits RX return ring to 512 entries. */
3647         if (BGE_IS_5717_PLUS(sc))
3648                 sc->bge_return_ring_cnt = BGE_RETURN_RING_CNT;
3649         else if (BGE_IS_5705_PLUS(sc))
3650                 sc->bge_return_ring_cnt = BGE_RETURN_RING_CNT_5705;
3651         else
3652                 sc->bge_return_ring_cnt = BGE_RETURN_RING_CNT;
3653
3654         if (bge_dma_alloc(sc)) {
3655                 device_printf(sc->bge_dev,
3656                     "failed to allocate DMA resources\n");
3657                 error = ENXIO;
3658                 goto fail;
3659         }
3660
3661         /* Set default tuneable values. */
3662         sc->bge_stat_ticks = BGE_TICKS_PER_SEC;
3663         sc->bge_rx_coal_ticks = 150;
3664         sc->bge_tx_coal_ticks = 150;
3665         sc->bge_rx_max_coal_bds = 10;
3666         sc->bge_tx_max_coal_bds = 10;
3667
3668         /* Initialize checksum features to use. */
3669         sc->bge_csum_features = BGE_CSUM_FEATURES;
3670         if (sc->bge_forced_udpcsum != 0)
3671                 sc->bge_csum_features |= CSUM_UDP;
3672
3673         /* Set up ifnet structure */
3674         ifp = sc->bge_ifp = if_alloc(IFT_ETHER);
3675         if (ifp == NULL) {
3676                 device_printf(sc->bge_dev, "failed to if_alloc()\n");
3677                 error = ENXIO;
3678                 goto fail;
3679         }
3680         ifp->if_softc = sc;
3681         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
3682         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
3683         ifp->if_ioctl = bge_ioctl;
3684         ifp->if_start = bge_start;
3685         ifp->if_init = bge_init;
3686         ifp->if_snd.ifq_drv_maxlen = BGE_TX_RING_CNT - 1;
3687         IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen);
3688         IFQ_SET_READY(&ifp->if_snd);
3689         ifp->if_hwassist = sc->bge_csum_features;
3690         ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_VLAN_HWTAGGING |
3691             IFCAP_VLAN_MTU;
3692         if ((sc->bge_flags & (BGE_FLAG_TSO | BGE_FLAG_TSO3)) != 0) {
3693                 ifp->if_hwassist |= CSUM_TSO;
3694                 ifp->if_capabilities |= IFCAP_TSO4 | IFCAP_VLAN_HWTSO;
3695         }
3696 #ifdef IFCAP_VLAN_HWCSUM
3697         ifp->if_capabilities |= IFCAP_VLAN_HWCSUM;
3698 #endif
3699         ifp->if_capenable = ifp->if_capabilities;
3700 #ifdef DEVICE_POLLING
3701         ifp->if_capabilities |= IFCAP_POLLING;
3702 #endif
3703
3704         /*
3705          * 5700 B0 chips do not support checksumming correctly due
3706          * to hardware bugs.
3707          */
3708         if (sc->bge_chipid == BGE_CHIPID_BCM5700_B0) {
3709                 ifp->if_capabilities &= ~IFCAP_HWCSUM;
3710                 ifp->if_capenable &= ~IFCAP_HWCSUM;
3711                 ifp->if_hwassist = 0;
3712         }
3713
3714         /*
3715          * Figure out what sort of media we have by checking the
3716          * hardware config word in the first 32k of NIC internal memory,
3717          * or fall back to examining the EEPROM if necessary.
3718          * Note: on some BCM5700 cards, this value appears to be unset.
3719          * If that's the case, we have to rely on identifying the NIC
3720          * by its PCI subsystem ID, as we do below for the SysKonnect
3721          * SK-9D41.
3722          */
3723         if (bge_readmem_ind(sc, BGE_SRAM_DATA_SIG) == BGE_SRAM_DATA_SIG_MAGIC)
3724                 hwcfg = bge_readmem_ind(sc, BGE_SRAM_DATA_CFG);
3725         else if ((sc->bge_flags & BGE_FLAG_EADDR) &&
3726             (sc->bge_asicrev != BGE_ASICREV_BCM5906)) {
3727                 if (bge_read_eeprom(sc, (caddr_t)&hwcfg, BGE_EE_HWCFG_OFFSET,
3728                     sizeof(hwcfg))) {
3729                         device_printf(sc->bge_dev, "failed to read EEPROM\n");
3730                         error = ENXIO;
3731                         goto fail;
3732                 }
3733                 hwcfg = ntohl(hwcfg);
3734         }
3735
3736         /* The SysKonnect SK-9D41 is a 1000baseSX card. */
3737         if ((pci_read_config(dev, BGE_PCI_SUBSYS, 4) >> 16) ==
3738             SK_SUBSYSID_9D41 || (hwcfg & BGE_HWCFG_MEDIA) == BGE_MEDIA_FIBER) {
3739                 if (BGE_IS_5705_PLUS(sc)) {
3740                         sc->bge_flags |= BGE_FLAG_MII_SERDES;
3741                         sc->bge_phy_flags |= BGE_PHY_NO_WIRESPEED;
3742                 } else
3743                         sc->bge_flags |= BGE_FLAG_TBI;
3744         }
3745
3746         /* Set various PHY bug flags. */
3747         if (sc->bge_chipid == BGE_CHIPID_BCM5701_A0 ||
3748             sc->bge_chipid == BGE_CHIPID_BCM5701_B0)
3749                 sc->bge_phy_flags |= BGE_PHY_CRC_BUG;
3750         if (sc->bge_chiprev == BGE_CHIPREV_5703_AX ||
3751             sc->bge_chiprev == BGE_CHIPREV_5704_AX)
3752                 sc->bge_phy_flags |= BGE_PHY_ADC_BUG;
3753         if (sc->bge_chipid == BGE_CHIPID_BCM5704_A0)
3754                 sc->bge_phy_flags |= BGE_PHY_5704_A0_BUG;
3755         if (pci_get_subvendor(dev) == DELL_VENDORID)
3756                 sc->bge_phy_flags |= BGE_PHY_NO_3LED;
3757         if ((BGE_IS_5705_PLUS(sc)) &&
3758             sc->bge_asicrev != BGE_ASICREV_BCM5906 &&
3759             sc->bge_asicrev != BGE_ASICREV_BCM5785 &&
3760             sc->bge_asicrev != BGE_ASICREV_BCM57780 &&
3761             !BGE_IS_5717_PLUS(sc)) {
3762                 if (sc->bge_asicrev == BGE_ASICREV_BCM5755 ||
3763                     sc->bge_asicrev == BGE_ASICREV_BCM5761 ||
3764                     sc->bge_asicrev == BGE_ASICREV_BCM5784 ||
3765                     sc->bge_asicrev == BGE_ASICREV_BCM5787) {
3766                         if (pci_get_device(dev) != BCOM_DEVICEID_BCM5722 &&
3767                             pci_get_device(dev) != BCOM_DEVICEID_BCM5756)
3768                                 sc->bge_phy_flags |= BGE_PHY_JITTER_BUG;
3769                         if (pci_get_device(dev) == BCOM_DEVICEID_BCM5755M)
3770                                 sc->bge_phy_flags |= BGE_PHY_ADJUST_TRIM;
3771                 } else
3772                         sc->bge_phy_flags |= BGE_PHY_BER_BUG;
3773         }
3774
3775         /*
3776          * Don't enable Ethernet@WireSpeed for the 5700 or the
3777          * 5705 A0 and A1 chips.
3778          */
3779         if (sc->bge_asicrev == BGE_ASICREV_BCM5700 ||
3780             (sc->bge_asicrev == BGE_ASICREV_BCM5705 &&
3781             (sc->bge_chipid != BGE_CHIPID_BCM5705_A0 &&
3782             sc->bge_chipid != BGE_CHIPID_BCM5705_A1)))
3783                 sc->bge_phy_flags |= BGE_PHY_NO_WIRESPEED;
3784
3785         if (sc->bge_flags & BGE_FLAG_TBI) {
3786                 ifmedia_init(&sc->bge_ifmedia, IFM_IMASK, bge_ifmedia_upd,
3787                     bge_ifmedia_sts);
3788                 ifmedia_add(&sc->bge_ifmedia, IFM_ETHER | IFM_1000_SX, 0, NULL);
3789                 ifmedia_add(&sc->bge_ifmedia, IFM_ETHER | IFM_1000_SX | IFM_FDX,
3790                     0, NULL);
3791                 ifmedia_add(&sc->bge_ifmedia, IFM_ETHER | IFM_AUTO, 0, NULL);
3792                 ifmedia_set(&sc->bge_ifmedia, IFM_ETHER | IFM_AUTO);
3793                 sc->bge_ifmedia.ifm_media = sc->bge_ifmedia.ifm_cur->ifm_media;
3794         } else {
3795                 /*
3796                  * Do transceiver setup and tell the firmware the
3797                  * driver is down so we can try to get access the
3798                  * probe if ASF is running.  Retry a couple of times
3799                  * if we get a conflict with the ASF firmware accessing
3800                  * the PHY.
3801                  */
3802                 trys = 0;
3803                 BGE_CLRBIT(sc, BGE_MODE_CTL, BGE_MODECTL_STACKUP);
3804 again:
3805                 bge_asf_driver_up(sc);
3806
3807                 error = mii_attach(dev, &sc->bge_miibus, ifp, bge_ifmedia_upd,
3808                     bge_ifmedia_sts, capmask, sc->bge_phy_addr, MII_OFFSET_ANY,
3809                     MIIF_DOPAUSE);
3810                 if (error != 0) {
3811                         if (trys++ < 4) {
3812                                 device_printf(sc->bge_dev, "Try again\n");
3813                                 bge_miibus_writereg(sc->bge_dev,
3814                                     sc->bge_phy_addr, MII_BMCR, BMCR_RESET);
3815                                 goto again;
3816                         }
3817                         device_printf(sc->bge_dev, "attaching PHYs failed\n");
3818                         goto fail;
3819                 }
3820
3821                 /*
3822                  * Now tell the firmware we are going up after probing the PHY
3823                  */
3824                 if (sc->bge_asf_mode & ASF_STACKUP)
3825                         BGE_SETBIT(sc, BGE_MODE_CTL, BGE_MODECTL_STACKUP);
3826         }
3827
3828         /*
3829          * When using the BCM5701 in PCI-X mode, data corruption has
3830          * been observed in the first few bytes of some received packets.
3831          * Aligning the packet buffer in memory eliminates the corruption.
3832          * Unfortunately, this misaligns the packet payloads.  On platforms
3833          * which do not support unaligned accesses, we will realign the
3834          * payloads by copying the received packets.
3835          */
3836         if (sc->bge_asicrev == BGE_ASICREV_BCM5701 &&
3837             sc->bge_flags & BGE_FLAG_PCIX)
3838                 sc->bge_flags |= BGE_FLAG_RX_ALIGNBUG;
3839
3840         /*
3841          * Call MI attach routine.
3842          */
3843         ether_ifattach(ifp, eaddr);
3844
3845         /* Tell upper layer we support long frames. */
3846         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
3847
3848         /*
3849          * Hookup IRQ last.
3850          */
3851         if (BGE_IS_5755_PLUS(sc) && sc->bge_flags & BGE_FLAG_MSI) {
3852                 /* Take advantage of single-shot MSI. */
3853                 CSR_WRITE_4(sc, BGE_MSI_MODE, CSR_READ_4(sc, BGE_MSI_MODE) &
3854                     ~BGE_MSIMODE_ONE_SHOT_DISABLE);
3855                 sc->bge_tq = taskqueue_create_fast("bge_taskq", M_WAITOK,
3856                     taskqueue_thread_enqueue, &sc->bge_tq);
3857                 if (sc->bge_tq == NULL) {
3858                         device_printf(dev, "could not create taskqueue.\n");
3859                         ether_ifdetach(ifp);
3860                         error = ENOMEM;
3861                         goto fail;
3862                 }
3863                 error = taskqueue_start_threads(&sc->bge_tq, 1, PI_NET,
3864                     "%s taskq", device_get_nameunit(sc->bge_dev));
3865                 if (error != 0) {
3866                         device_printf(dev, "could not start threads.\n");
3867                         ether_ifdetach(ifp);
3868                         goto fail;
3869                 }
3870                 error = bus_setup_intr(dev, sc->bge_irq,
3871                     INTR_TYPE_NET | INTR_MPSAFE, bge_msi_intr, NULL, sc,
3872                     &sc->bge_intrhand);
3873         } else
3874                 error = bus_setup_intr(dev, sc->bge_irq,
3875                     INTR_TYPE_NET | INTR_MPSAFE, NULL, bge_intr, sc,
3876                     &sc->bge_intrhand);
3877
3878         if (error) {
3879                 ether_ifdetach(ifp);
3880                 device_printf(sc->bge_dev, "couldn't set up irq\n");
3881         }
3882
3883 fail:
3884         if (error)
3885                 bge_detach(dev);
3886         return (error);
3887 }
3888
3889 static int
3890 bge_detach(device_t dev)
3891 {
3892         struct bge_softc *sc;
3893         struct ifnet *ifp;
3894
3895         sc = device_get_softc(dev);
3896         ifp = sc->bge_ifp;
3897
3898 #ifdef DEVICE_POLLING
3899         if (ifp->if_capenable & IFCAP_POLLING)
3900                 ether_poll_deregister(ifp);
3901 #endif
3902
3903         if (device_is_attached(dev)) {
3904                 ether_ifdetach(ifp);
3905                 BGE_LOCK(sc);
3906                 bge_stop(sc);
3907                 BGE_UNLOCK(sc);
3908                 callout_drain(&sc->bge_stat_ch);
3909         }
3910
3911         if (sc->bge_tq)
3912                 taskqueue_drain(sc->bge_tq, &sc->bge_intr_task);
3913
3914         if (sc->bge_flags & BGE_FLAG_TBI)
3915                 ifmedia_removeall(&sc->bge_ifmedia);
3916         else if (sc->bge_miibus != NULL) {
3917                 bus_generic_detach(dev);
3918                 device_delete_child(dev, sc->bge_miibus);
3919         }
3920
3921         bge_release_resources(sc);
3922
3923         return (0);
3924 }
3925
3926 static void
3927 bge_release_resources(struct bge_softc *sc)
3928 {
3929         device_t dev;
3930
3931         dev = sc->bge_dev;
3932
3933         if (sc->bge_tq != NULL)
3934                 taskqueue_free(sc->bge_tq);
3935
3936         if (sc->bge_intrhand != NULL)
3937                 bus_teardown_intr(dev, sc->bge_irq, sc->bge_intrhand);
3938
3939         if (sc->bge_irq != NULL)
3940                 bus_release_resource(dev, SYS_RES_IRQ,
3941                     sc->bge_flags & BGE_FLAG_MSI ? 1 : 0, sc->bge_irq);
3942
3943         if (sc->bge_flags & BGE_FLAG_MSI)
3944                 pci_release_msi(dev);
3945
3946         if (sc->bge_res != NULL)
3947                 bus_release_resource(dev, SYS_RES_MEMORY,
3948                     PCIR_BAR(0), sc->bge_res);
3949
3950         if (sc->bge_res2 != NULL)
3951                 bus_release_resource(dev, SYS_RES_MEMORY,
3952                     PCIR_BAR(2), sc->bge_res2);
3953
3954         if (sc->bge_ifp != NULL)
3955                 if_free(sc->bge_ifp);
3956
3957         bge_dma_free(sc);
3958
3959         if (mtx_initialized(&sc->bge_mtx))      /* XXX */
3960                 BGE_LOCK_DESTROY(sc);
3961 }
3962
3963 static int
3964 bge_reset(struct bge_softc *sc)
3965 {
3966         device_t dev;
3967         uint32_t cachesize, command, mac_mode, mac_mode_mask, reset, val;
3968         void (*write_op)(struct bge_softc *, int, int);
3969         uint16_t devctl;
3970         int i;
3971
3972         dev = sc->bge_dev;
3973
3974         mac_mode_mask = BGE_MACMODE_HALF_DUPLEX | BGE_MACMODE_PORTMODE;
3975         if ((sc->bge_mfw_flags & BGE_MFW_ON_APE) != 0)
3976                 mac_mode_mask |= BGE_MACMODE_APE_RX_EN | BGE_MACMODE_APE_TX_EN;
3977         mac_mode = CSR_READ_4(sc, BGE_MAC_MODE) & mac_mode_mask;
3978
3979         if (BGE_IS_575X_PLUS(sc) && !BGE_IS_5714_FAMILY(sc) &&
3980             (sc->bge_asicrev != BGE_ASICREV_BCM5906)) {
3981                 if (sc->bge_flags & BGE_FLAG_PCIE)
3982                         write_op = bge_writemem_direct;
3983                 else
3984                         write_op = bge_writemem_ind;
3985         } else
3986                 write_op = bge_writereg_ind;
3987
3988         if (sc->bge_asicrev != BGE_ASICREV_BCM5700 &&
3989             sc->bge_asicrev != BGE_ASICREV_BCM5701) {
3990                 CSR_WRITE_4(sc, BGE_NVRAM_SWARB, BGE_NVRAMSWARB_SET1);
3991                 for (i = 0; i < 8000; i++) {
3992                         if (CSR_READ_4(sc, BGE_NVRAM_SWARB) &
3993                             BGE_NVRAMSWARB_GNT1)
3994                                 break;
3995                         DELAY(20);
3996                 }
3997                 if (i == 8000) {
3998                         if (bootverbose)
3999                                 device_printf(dev, "NVRAM lock timedout!\n");
4000                 }
4001         }
4002         /* Take APE lock when performing reset. */
4003         bge_ape_lock(sc, BGE_APE_LOCK_GRC);
4004
4005         /* Save some important PCI state. */
4006         cachesize = pci_read_config(dev, BGE_PCI_CACHESZ, 4);
4007         command = pci_read_config(dev, BGE_PCI_CMD, 4);
4008
4009         pci_write_config(dev, BGE_PCI_MISC_CTL,
4010             BGE_PCIMISCCTL_INDIRECT_ACCESS | BGE_PCIMISCCTL_MASK_PCI_INTR |
4011             BGE_HIF_SWAP_OPTIONS | BGE_PCIMISCCTL_PCISTATE_RW, 4);
4012
4013         /* Disable fastboot on controllers that support it. */
4014         if (sc->bge_asicrev == BGE_ASICREV_BCM5752 ||
4015             BGE_IS_5755_PLUS(sc)) {
4016                 if (bootverbose)
4017                         device_printf(dev, "Disabling fastboot\n");
4018                 CSR_WRITE_4(sc, BGE_FASTBOOT_PC, 0x0);
4019         }
4020
4021         /*
4022          * Write the magic number to SRAM at offset 0xB50.
4023          * When firmware finishes its initialization it will
4024          * write ~BGE_SRAM_FW_MB_MAGIC to the same location.
4025          */
4026         bge_writemem_ind(sc, BGE_SRAM_FW_MB, BGE_SRAM_FW_MB_MAGIC);
4027
4028         reset = BGE_MISCCFG_RESET_CORE_CLOCKS | BGE_32BITTIME_66MHZ;
4029
4030         /* XXX: Broadcom Linux driver. */
4031         if (sc->bge_flags & BGE_FLAG_PCIE) {
4032                 if (sc->bge_asicrev != BGE_ASICREV_BCM5785 &&
4033                     (sc->bge_flags & BGE_FLAG_5717_PLUS) == 0) {
4034                         if (CSR_READ_4(sc, 0x7E2C) == 0x60)     /* PCIE 1.0 */
4035                                 CSR_WRITE_4(sc, 0x7E2C, 0x20);
4036                 }
4037                 if (sc->bge_chipid != BGE_CHIPID_BCM5750_A0) {
4038                         /* Prevent PCIE link training during global reset */
4039                         CSR_WRITE_4(sc, BGE_MISC_CFG, 1 << 29);
4040                         reset |= 1 << 29;
4041                 }
4042         }
4043
4044         if (sc->bge_asicrev == BGE_ASICREV_BCM5906) {
4045                 val = CSR_READ_4(sc, BGE_VCPU_STATUS);
4046                 CSR_WRITE_4(sc, BGE_VCPU_STATUS,
4047                     val | BGE_VCPU_STATUS_DRV_RESET);
4048                 val = CSR_READ_4(sc, BGE_VCPU_EXT_CTRL);
4049                 CSR_WRITE_4(sc, BGE_VCPU_EXT_CTRL,
4050                     val & ~BGE_VCPU_EXT_CTRL_HALT_CPU);
4051         }
4052
4053         /*
4054          * Set GPHY Power Down Override to leave GPHY
4055          * powered up in D0 uninitialized.
4056          */
4057         if (BGE_IS_5705_PLUS(sc) &&
4058             (sc->bge_flags & BGE_FLAG_CPMU_PRESENT) == 0)
4059                 reset |= BGE_MISCCFG_GPHY_PD_OVERRIDE;
4060
4061         /* Issue global reset */
4062         write_op(sc, BGE_MISC_CFG, reset);
4063
4064         if (sc->bge_flags & BGE_FLAG_PCIE)
4065                 DELAY(100 * 1000);
4066         else
4067                 DELAY(1000);
4068
4069         /* XXX: Broadcom Linux driver. */
4070         if (sc->bge_flags & BGE_FLAG_PCIE) {
4071                 if (sc->bge_chipid == BGE_CHIPID_BCM5750_A0) {
4072                         DELAY(500000); /* wait for link training to complete */
4073                         val = pci_read_config(dev, 0xC4, 4);
4074                         pci_write_config(dev, 0xC4, val | (1 << 15), 4);
4075                 }
4076                 devctl = pci_read_config(dev,
4077                     sc->bge_expcap + PCIER_DEVICE_CTL, 2);
4078                 /* Clear enable no snoop and disable relaxed ordering. */
4079                 devctl &= ~(PCIEM_CTL_RELAXED_ORD_ENABLE |
4080                     PCIEM_CTL_NOSNOOP_ENABLE);
4081                 pci_write_config(dev, sc->bge_expcap + PCIER_DEVICE_CTL,
4082                     devctl, 2);
4083                 pci_set_max_read_req(dev, sc->bge_expmrq);
4084                 /* Clear error status. */
4085                 pci_write_config(dev, sc->bge_expcap + PCIER_DEVICE_STA,
4086                     PCIEM_STA_CORRECTABLE_ERROR |
4087                     PCIEM_STA_NON_FATAL_ERROR | PCIEM_STA_FATAL_ERROR |
4088                     PCIEM_STA_UNSUPPORTED_REQ, 2);
4089         }
4090
4091         /* Reset some of the PCI state that got zapped by reset. */
4092         pci_write_config(dev, BGE_PCI_MISC_CTL,
4093             BGE_PCIMISCCTL_INDIRECT_ACCESS | BGE_PCIMISCCTL_MASK_PCI_INTR |
4094             BGE_HIF_SWAP_OPTIONS | BGE_PCIMISCCTL_PCISTATE_RW, 4);
4095         val = BGE_PCISTATE_ROM_ENABLE | BGE_PCISTATE_ROM_RETRY_ENABLE;
4096         if (sc->bge_chipid == BGE_CHIPID_BCM5704_A0 &&
4097             (sc->bge_flags & BGE_FLAG_PCIX) != 0)
4098                 val |= BGE_PCISTATE_RETRY_SAME_DMA;
4099         if ((sc->bge_mfw_flags & BGE_MFW_ON_APE) != 0)
4100                 val |= BGE_PCISTATE_ALLOW_APE_CTLSPC_WR |
4101                     BGE_PCISTATE_ALLOW_APE_SHMEM_WR |
4102                     BGE_PCISTATE_ALLOW_APE_PSPACE_WR;
4103         pci_write_config(dev, BGE_PCI_PCISTATE, val, 4);
4104         pci_write_config(dev, BGE_PCI_CACHESZ, cachesize, 4);
4105         pci_write_config(dev, BGE_PCI_CMD, command, 4);
4106         /*
4107          * Disable PCI-X relaxed ordering to ensure status block update
4108          * comes first then packet buffer DMA. Otherwise driver may
4109          * read stale status block.
4110          */
4111         if (sc->bge_flags & BGE_FLAG_PCIX) {
4112                 devctl = pci_read_config(dev,
4113                     sc->bge_pcixcap + PCIXR_COMMAND, 2);
4114                 devctl &= ~PCIXM_COMMAND_ERO;
4115                 if (sc->bge_asicrev == BGE_ASICREV_BCM5703) {
4116                         devctl &= ~PCIXM_COMMAND_MAX_READ;
4117                         devctl |= PCIXM_COMMAND_MAX_READ_2048;
4118                 } else if (sc->bge_asicrev == BGE_ASICREV_BCM5704) {
4119                         devctl &= ~(PCIXM_COMMAND_MAX_SPLITS |
4120                             PCIXM_COMMAND_MAX_READ);
4121                         devctl |= PCIXM_COMMAND_MAX_READ_2048;
4122                 }
4123                 pci_write_config(dev, sc->bge_pcixcap + PCIXR_COMMAND,
4124                     devctl, 2);
4125         }
4126         /* Re-enable MSI, if necessary, and enable the memory arbiter. */
4127         if (BGE_IS_5714_FAMILY(sc)) {
4128                 /* This chip disables MSI on reset. */
4129                 if (sc->bge_flags & BGE_FLAG_MSI) {
4130                         val = pci_read_config(dev,
4131                             sc->bge_msicap + PCIR_MSI_CTRL, 2);
4132                         pci_write_config(dev,
4133                             sc->bge_msicap + PCIR_MSI_CTRL,
4134                             val | PCIM_MSICTRL_MSI_ENABLE, 2);
4135                         val = CSR_READ_4(sc, BGE_MSI_MODE);
4136                         CSR_WRITE_4(sc, BGE_MSI_MODE,
4137                             val | BGE_MSIMODE_ENABLE);
4138                 }
4139                 val = CSR_READ_4(sc, BGE_MARB_MODE);
4140                 CSR_WRITE_4(sc, BGE_MARB_MODE, BGE_MARBMODE_ENABLE | val);
4141         } else
4142                 CSR_WRITE_4(sc, BGE_MARB_MODE, BGE_MARBMODE_ENABLE);
4143
4144         /* Fix up byte swapping. */
4145         CSR_WRITE_4(sc, BGE_MODE_CTL, bge_dma_swap_options(sc));
4146
4147         val = CSR_READ_4(sc, BGE_MAC_MODE);
4148         val = (val & ~mac_mode_mask) | mac_mode;
4149         CSR_WRITE_4(sc, BGE_MAC_MODE, val);
4150         DELAY(40);
4151
4152         bge_ape_unlock(sc, BGE_APE_LOCK_GRC);
4153
4154         if (sc->bge_asicrev == BGE_ASICREV_BCM5906) {
4155                 for (i = 0; i < BGE_TIMEOUT; i++) {
4156                         val = CSR_READ_4(sc, BGE_VCPU_STATUS);
4157                         if (val & BGE_VCPU_STATUS_INIT_DONE)
4158                                 break;
4159                         DELAY(100);
4160                 }
4161                 if (i == BGE_TIMEOUT) {
4162                         device_printf(dev, "reset timed out\n");
4163                         return (1);
4164                 }
4165         } else {
4166                 /*
4167                  * Poll until we see the 1's complement of the magic number.
4168                  * This indicates that the firmware initialization is complete.
4169                  * We expect this to fail if no chip containing the Ethernet
4170                  * address is fitted though.
4171                  */
4172                 for (i = 0; i < BGE_TIMEOUT; i++) {
4173                         DELAY(10);
4174                         val = bge_readmem_ind(sc, BGE_SRAM_FW_MB);
4175                         if (val == ~BGE_SRAM_FW_MB_MAGIC)
4176                                 break;
4177                 }
4178
4179                 if ((sc->bge_flags & BGE_FLAG_EADDR) && i == BGE_TIMEOUT)
4180                         device_printf(dev,
4181                             "firmware handshake timed out, found 0x%08x\n",
4182                             val);
4183                 /* BCM57765 A0 needs additional time before accessing. */
4184                 if (sc->bge_chipid == BGE_CHIPID_BCM57765_A0)
4185                         DELAY(10 * 1000);       /* XXX */
4186         }
4187
4188         /*
4189          * The 5704 in TBI mode apparently needs some special
4190          * adjustment to insure the SERDES drive level is set
4191          * to 1.2V.
4192          */
4193         if (sc->bge_asicrev == BGE_ASICREV_BCM5704 &&
4194             sc->bge_flags & BGE_FLAG_TBI) {
4195                 val = CSR_READ_4(sc, BGE_SERDES_CFG);
4196                 val = (val & ~0xFFF) | 0x880;
4197                 CSR_WRITE_4(sc, BGE_SERDES_CFG, val);
4198         }
4199
4200         /* XXX: Broadcom Linux driver. */
4201         if (sc->bge_flags & BGE_FLAG_PCIE &&
4202             !BGE_IS_5717_PLUS(sc) &&
4203             sc->bge_chipid != BGE_CHIPID_BCM5750_A0 &&
4204             sc->bge_asicrev != BGE_ASICREV_BCM5785) {
4205                 /* Enable Data FIFO protection. */
4206                 val = CSR_READ_4(sc, 0x7C00);
4207                 CSR_WRITE_4(sc, 0x7C00, val | (1 << 25));
4208         }
4209
4210         if (sc->bge_asicrev == BGE_ASICREV_BCM5720)
4211                 BGE_CLRBIT(sc, BGE_CPMU_CLCK_ORIDE,
4212                     CPMU_CLCK_ORIDE_MAC_ORIDE_EN);
4213
4214         return (0);
4215 }
4216
4217 static __inline void
4218 bge_rxreuse_std(struct bge_softc *sc, int i)
4219 {
4220         struct bge_rx_bd *r;
4221
4222         r = &sc->bge_ldata.bge_rx_std_ring[sc->bge_std];
4223         r->bge_flags = BGE_RXBDFLAG_END;
4224         r->bge_len = sc->bge_cdata.bge_rx_std_seglen[i];
4225         r->bge_idx = i;
4226         BGE_INC(sc->bge_std, BGE_STD_RX_RING_CNT);
4227 }
4228
4229 static __inline void
4230 bge_rxreuse_jumbo(struct bge_softc *sc, int i)
4231 {
4232         struct bge_extrx_bd *r;
4233
4234         r = &sc->bge_ldata.bge_rx_jumbo_ring[sc->bge_jumbo];
4235         r->bge_flags = BGE_RXBDFLAG_JUMBO_RING | BGE_RXBDFLAG_END;
4236         r->bge_len0 = sc->bge_cdata.bge_rx_jumbo_seglen[i][0];
4237         r->bge_len1 = sc->bge_cdata.bge_rx_jumbo_seglen[i][1];
4238         r->bge_len2 = sc->bge_cdata.bge_rx_jumbo_seglen[i][2];
4239         r->bge_len3 = sc->bge_cdata.bge_rx_jumbo_seglen[i][3];
4240         r->bge_idx = i;
4241         BGE_INC(sc->bge_jumbo, BGE_JUMBO_RX_RING_CNT);
4242 }
4243
4244 /*
4245  * Frame reception handling. This is called if there's a frame
4246  * on the receive return list.
4247  *
4248  * Note: we have to be able to handle two possibilities here:
4249  * 1) the frame is from the jumbo receive ring
4250  * 2) the frame is from the standard receive ring
4251  */
4252
4253 static int
4254 bge_rxeof(struct bge_softc *sc, uint16_t rx_prod, int holdlck)
4255 {
4256         struct ifnet *ifp;
4257         int rx_npkts = 0, stdcnt = 0, jumbocnt = 0;
4258         uint16_t rx_cons;
4259
4260         rx_cons = sc->bge_rx_saved_considx;
4261
4262         /* Nothing to do. */
4263         if (rx_cons == rx_prod)
4264                 return (rx_npkts);
4265
4266         ifp = sc->bge_ifp;
4267
4268         bus_dmamap_sync(sc->bge_cdata.bge_rx_return_ring_tag,
4269             sc->bge_cdata.bge_rx_return_ring_map, BUS_DMASYNC_POSTREAD);
4270         bus_dmamap_sync(sc->bge_cdata.bge_rx_std_ring_tag,
4271             sc->bge_cdata.bge_rx_std_ring_map, BUS_DMASYNC_POSTWRITE);
4272         if (BGE_IS_JUMBO_CAPABLE(sc) &&
4273             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN + ETHER_VLAN_ENCAP_LEN >
4274             (MCLBYTES - ETHER_ALIGN))
4275                 bus_dmamap_sync(sc->bge_cdata.bge_rx_jumbo_ring_tag,
4276                     sc->bge_cdata.bge_rx_jumbo_ring_map, BUS_DMASYNC_POSTWRITE);
4277
4278         while (rx_cons != rx_prod) {
4279                 struct bge_rx_bd        *cur_rx;
4280                 uint32_t                rxidx;
4281                 struct mbuf             *m = NULL;
4282                 uint16_t                vlan_tag = 0;
4283                 int                     have_tag = 0;
4284
4285 #ifdef DEVICE_POLLING
4286                 if (ifp->if_capenable & IFCAP_POLLING) {
4287                         if (sc->rxcycles <= 0)
4288                                 break;
4289                         sc->rxcycles--;
4290                 }
4291 #endif
4292
4293                 cur_rx = &sc->bge_ldata.bge_rx_return_ring[rx_cons];
4294
4295                 rxidx = cur_rx->bge_idx;
4296                 BGE_INC(rx_cons, sc->bge_return_ring_cnt);
4297
4298                 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING &&
4299                     cur_rx->bge_flags & BGE_RXBDFLAG_VLAN_TAG) {
4300                         have_tag = 1;
4301                         vlan_tag = cur_rx->bge_vlan_tag;
4302                 }
4303
4304                 if (cur_rx->bge_flags & BGE_RXBDFLAG_JUMBO_RING) {
4305                         jumbocnt++;
4306                         m = sc->bge_cdata.bge_rx_jumbo_chain[rxidx];
4307                         if (cur_rx->bge_flags & BGE_RXBDFLAG_ERROR) {
4308                                 bge_rxreuse_jumbo(sc, rxidx);
4309                                 continue;
4310                         }
4311                         if (bge_newbuf_jumbo(sc, rxidx) != 0) {
4312                                 bge_rxreuse_jumbo(sc, rxidx);
4313                                 ifp->if_iqdrops++;
4314                                 continue;
4315                         }
4316                         BGE_INC(sc->bge_jumbo, BGE_JUMBO_RX_RING_CNT);
4317                 } else {
4318                         stdcnt++;
4319                         m = sc->bge_cdata.bge_rx_std_chain[rxidx];
4320                         if (cur_rx->bge_flags & BGE_RXBDFLAG_ERROR) {
4321                                 bge_rxreuse_std(sc, rxidx);
4322                                 continue;
4323                         }
4324                         if (bge_newbuf_std(sc, rxidx) != 0) {
4325                                 bge_rxreuse_std(sc, rxidx);
4326                                 ifp->if_iqdrops++;
4327                                 continue;
4328                         }
4329                         BGE_INC(sc->bge_std, BGE_STD_RX_RING_CNT);
4330                 }
4331
4332                 ifp->if_ipackets++;
4333 #ifndef __NO_STRICT_ALIGNMENT
4334                 /*
4335                  * For architectures with strict alignment we must make sure
4336                  * the payload is aligned.
4337                  */
4338                 if (sc->bge_flags & BGE_FLAG_RX_ALIGNBUG) {
4339                         bcopy(m->m_data, m->m_data + ETHER_ALIGN,
4340                             cur_rx->bge_len);
4341                         m->m_data += ETHER_ALIGN;
4342                 }
4343 #endif
4344                 m->m_pkthdr.len = m->m_len = cur_rx->bge_len - ETHER_CRC_LEN;
4345                 m->m_pkthdr.rcvif = ifp;
4346
4347                 if (ifp->if_capenable & IFCAP_RXCSUM)
4348                         bge_rxcsum(sc, cur_rx, m);
4349
4350                 /*
4351                  * If we received a packet with a vlan tag,
4352                  * attach that information to the packet.
4353                  */
4354                 if (have_tag) {
4355                         m->m_pkthdr.ether_vtag = vlan_tag;
4356                         m->m_flags |= M_VLANTAG;
4357                 }
4358
4359                 if (holdlck != 0) {
4360                         BGE_UNLOCK(sc);
4361                         (*ifp->if_input)(ifp, m);
4362                         BGE_LOCK(sc);
4363                 } else
4364                         (*ifp->if_input)(ifp, m);
4365                 rx_npkts++;
4366
4367                 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
4368                         return (rx_npkts);
4369         }
4370
4371         bus_dmamap_sync(sc->bge_cdata.bge_rx_return_ring_tag,
4372             sc->bge_cdata.bge_rx_return_ring_map, BUS_DMASYNC_PREREAD);
4373         if (stdcnt > 0)
4374                 bus_dmamap_sync(sc->bge_cdata.bge_rx_std_ring_tag,
4375                     sc->bge_cdata.bge_rx_std_ring_map, BUS_DMASYNC_PREWRITE);
4376
4377         if (jumbocnt > 0)
4378                 bus_dmamap_sync(sc->bge_cdata.bge_rx_jumbo_ring_tag,
4379                     sc->bge_cdata.bge_rx_jumbo_ring_map, BUS_DMASYNC_PREWRITE);
4380
4381         sc->bge_rx_saved_considx = rx_cons;
4382         bge_writembx(sc, BGE_MBX_RX_CONS0_LO, sc->bge_rx_saved_considx);
4383         if (stdcnt)
4384                 bge_writembx(sc, BGE_MBX_RX_STD_PROD_LO, (sc->bge_std +
4385                     BGE_STD_RX_RING_CNT - 1) % BGE_STD_RX_RING_CNT);
4386         if (jumbocnt)
4387                 bge_writembx(sc, BGE_MBX_RX_JUMBO_PROD_LO, (sc->bge_jumbo +
4388                     BGE_JUMBO_RX_RING_CNT - 1) % BGE_JUMBO_RX_RING_CNT);
4389 #ifdef notyet
4390         /*
4391          * This register wraps very quickly under heavy packet drops.
4392          * If you need correct statistics, you can enable this check.
4393          */
4394         if (BGE_IS_5705_PLUS(sc))
4395                 ifp->if_ierrors += CSR_READ_4(sc, BGE_RXLP_LOCSTAT_IFIN_DROPS);
4396 #endif
4397         return (rx_npkts);
4398 }
4399
4400 static void
4401 bge_rxcsum(struct bge_softc *sc, struct bge_rx_bd *cur_rx, struct mbuf *m)
4402 {
4403
4404         if (BGE_IS_5717_PLUS(sc)) {
4405                 if ((cur_rx->bge_flags & BGE_RXBDFLAG_IPV6) == 0) {
4406                         if (cur_rx->bge_flags & BGE_RXBDFLAG_IP_CSUM) {
4407                                 m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED;
4408                                 if ((cur_rx->bge_error_flag &
4409                                     BGE_RXERRFLAG_IP_CSUM_NOK) == 0)
4410                                         m->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4411                         }
4412                         if (cur_rx->bge_flags & BGE_RXBDFLAG_TCP_UDP_CSUM) {
4413                                 m->m_pkthdr.csum_data =
4414                                     cur_rx->bge_tcp_udp_csum;
4415                                 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID |
4416                                     CSUM_PSEUDO_HDR;
4417                         }
4418                 }
4419         } else {
4420                 if (cur_rx->bge_flags & BGE_RXBDFLAG_IP_CSUM) {
4421                         m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED;
4422                         if ((cur_rx->bge_ip_csum ^ 0xFFFF) == 0)
4423                                 m->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4424                 }
4425                 if (cur_rx->bge_flags & BGE_RXBDFLAG_TCP_UDP_CSUM &&
4426                     m->m_pkthdr.len >= ETHER_MIN_NOPAD) {
4427                         m->m_pkthdr.csum_data =
4428                             cur_rx->bge_tcp_udp_csum;
4429                         m->m_pkthdr.csum_flags |= CSUM_DATA_VALID |
4430                             CSUM_PSEUDO_HDR;
4431                 }
4432         }
4433 }
4434
4435 static void
4436 bge_txeof(struct bge_softc *sc, uint16_t tx_cons)
4437 {
4438         struct bge_tx_bd *cur_tx;
4439         struct ifnet *ifp;
4440
4441         BGE_LOCK_ASSERT(sc);
4442
4443         /* Nothing to do. */
4444         if (sc->bge_tx_saved_considx == tx_cons)
4445                 return;
4446
4447         ifp = sc->bge_ifp;
4448
4449         bus_dmamap_sync(sc->bge_cdata.bge_tx_ring_tag,
4450             sc->bge_cdata.bge_tx_ring_map, BUS_DMASYNC_POSTWRITE);
4451         /*
4452          * Go through our tx ring and free mbufs for those
4453          * frames that have been sent.
4454          */
4455         while (sc->bge_tx_saved_considx != tx_cons) {
4456                 uint32_t                idx;
4457
4458                 idx = sc->bge_tx_saved_considx;
4459                 cur_tx = &sc->bge_ldata.bge_tx_ring[idx];
4460                 if (cur_tx->bge_flags & BGE_TXBDFLAG_END)
4461                         ifp->if_opackets++;
4462                 if (sc->bge_cdata.bge_tx_chain[idx] != NULL) {
4463                         bus_dmamap_sync(sc->bge_cdata.bge_tx_mtag,
4464                             sc->bge_cdata.bge_tx_dmamap[idx],
4465                             BUS_DMASYNC_POSTWRITE);
4466                         bus_dmamap_unload(sc->bge_cdata.bge_tx_mtag,
4467                             sc->bge_cdata.bge_tx_dmamap[idx]);
4468                         m_freem(sc->bge_cdata.bge_tx_chain[idx]);
4469                         sc->bge_cdata.bge_tx_chain[idx] = NULL;
4470                 }
4471                 sc->bge_txcnt--;
4472                 BGE_INC(sc->bge_tx_saved_considx, BGE_TX_RING_CNT);
4473         }
4474
4475         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
4476         if (sc->bge_txcnt == 0)
4477                 sc->bge_timer = 0;
4478 }
4479
4480 #ifdef DEVICE_POLLING
4481 static int
4482 bge_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
4483 {
4484         struct bge_softc *sc = ifp->if_softc;
4485         uint16_t rx_prod, tx_cons;
4486         uint32_t statusword;
4487         int rx_npkts = 0;
4488
4489         BGE_LOCK(sc);
4490         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4491                 BGE_UNLOCK(sc);
4492                 return (rx_npkts);
4493         }
4494
4495         bus_dmamap_sync(sc->bge_cdata.bge_status_tag,
4496             sc->bge_cdata.bge_status_map,
4497             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4498         /* Fetch updates from the status block. */
4499         rx_prod = sc->bge_ldata.bge_status_block->bge_idx[0].bge_rx_prod_idx;
4500         tx_cons = sc->bge_ldata.bge_status_block->bge_idx[0].bge_tx_cons_idx;
4501
4502         statusword = sc->bge_ldata.bge_status_block->bge_status;
4503         /* Clear the status so the next pass only sees the changes. */
4504         sc->bge_ldata.bge_status_block->bge_status = 0;
4505
4506         bus_dmamap_sync(sc->bge_cdata.bge_status_tag,
4507             sc->bge_cdata.bge_status_map,
4508             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4509
4510         /* Note link event. It will be processed by POLL_AND_CHECK_STATUS. */
4511         if (statusword & BGE_STATFLAG_LINKSTATE_CHANGED)
4512                 sc->bge_link_evt++;
4513
4514         if (cmd == POLL_AND_CHECK_STATUS)
4515                 if ((sc->bge_asicrev == BGE_ASICREV_BCM5700 &&
4516                     sc->bge_chipid != BGE_CHIPID_BCM5700_B2) ||
4517                     sc->bge_link_evt || (sc->bge_flags & BGE_FLAG_TBI))
4518                         bge_link_upd(sc);
4519
4520         sc->rxcycles = count;
4521         rx_npkts = bge_rxeof(sc, rx_prod, 1);
4522         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4523                 BGE_UNLOCK(sc);
4524                 return (rx_npkts);
4525         }
4526         bge_txeof(sc, tx_cons);
4527         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
4528                 bge_start_locked(ifp);
4529
4530         BGE_UNLOCK(sc);
4531         return (rx_npkts);
4532 }
4533 #endif /* DEVICE_POLLING */
4534
4535 static int
4536 bge_msi_intr(void *arg)
4537 {
4538         struct bge_softc *sc;
4539
4540         sc = (struct bge_softc *)arg;
4541         /*
4542          * This interrupt is not shared and controller already
4543          * disabled further interrupt.
4544          */
4545         taskqueue_enqueue(sc->bge_tq, &sc->bge_intr_task);
4546         return (FILTER_HANDLED);
4547 }
4548
4549 static void
4550 bge_intr_task(void *arg, int pending)
4551 {
4552         struct bge_softc *sc;
4553         struct ifnet *ifp;
4554         uint32_t status, status_tag;
4555         uint16_t rx_prod, tx_cons;
4556
4557         sc = (struct bge_softc *)arg;
4558         ifp = sc->bge_ifp;
4559
4560         BGE_LOCK(sc);
4561         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
4562                 BGE_UNLOCK(sc);
4563                 return;
4564         }
4565
4566         /* Get updated status block. */
4567         bus_dmamap_sync(sc->bge_cdata.bge_status_tag,
4568             sc->bge_cdata.bge_status_map,
4569             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4570
4571         /* Save producer/consumer indices. */
4572         rx_prod = sc->bge_ldata.bge_status_block->bge_idx[0].bge_rx_prod_idx;
4573         tx_cons = sc->bge_ldata.bge_status_block->bge_idx[0].bge_tx_cons_idx;
4574         status = sc->bge_ldata.bge_status_block->bge_status;
4575         status_tag = sc->bge_ldata.bge_status_block->bge_status_tag << 24;
4576         /* Dirty the status flag. */
4577         sc->bge_ldata.bge_status_block->bge_status = 0;
4578         bus_dmamap_sync(sc->bge_cdata.bge_status_tag,
4579             sc->bge_cdata.bge_status_map,
4580             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4581         if ((sc->bge_flags & BGE_FLAG_TAGGED_STATUS) == 0)
4582                 status_tag = 0;
4583
4584         if ((status & BGE_STATFLAG_LINKSTATE_CHANGED) != 0)
4585                 bge_link_upd(sc);
4586
4587         /* Let controller work. */
4588         bge_writembx(sc, BGE_MBX_IRQ0_LO, status_tag);
4589
4590         if (ifp->if_drv_flags & IFF_DRV_RUNNING &&
4591             sc->bge_rx_saved_considx != rx_prod) {
4592                 /* Check RX return ring producer/consumer. */
4593                 BGE_UNLOCK(sc);
4594                 bge_rxeof(sc, rx_prod, 0);
4595                 BGE_LOCK(sc);
4596         }
4597         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
4598                 /* Check TX ring producer/consumer. */
4599                 bge_txeof(sc, tx_cons);
4600                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
4601                         bge_start_locked(ifp);
4602         }
4603         BGE_UNLOCK(sc);
4604 }
4605
4606 static void
4607 bge_intr(void *xsc)
4608 {
4609         struct bge_softc *sc;
4610         struct ifnet *ifp;
4611         uint32_t statusword;
4612         uint16_t rx_prod, tx_cons;
4613
4614         sc = xsc;
4615
4616         BGE_LOCK(sc);
4617
4618         ifp = sc->bge_ifp;
4619
4620 #ifdef DEVICE_POLLING
4621         if (ifp->if_capenable & IFCAP_POLLING) {
4622                 BGE_UNLOCK(sc);
4623                 return;
4624         }
4625 #endif
4626
4627         /*
4628          * Ack the interrupt by writing something to BGE_MBX_IRQ0_LO.  Don't
4629          * disable interrupts by writing nonzero like we used to, since with
4630          * our current organization this just gives complications and
4631          * pessimizations for re-enabling interrupts.  We used to have races
4632          * instead of the necessary complications.  Disabling interrupts
4633          * would just reduce the chance of a status update while we are
4634          * running (by switching to the interrupt-mode coalescence
4635          * parameters), but this chance is already very low so it is more
4636          * efficient to get another interrupt than prevent it.
4637          *
4638          * We do the ack first to ensure another interrupt if there is a
4639          * status update after the ack.  We don't check for the status
4640          * changing later because it is more efficient to get another
4641          * interrupt than prevent it, not quite as above (not checking is
4642          * a smaller optimization than not toggling the interrupt enable,
4643          * since checking doesn't involve PCI accesses and toggling require
4644          * the status check).  So toggling would probably be a pessimization
4645          * even with MSI.  It would only be needed for using a task queue.
4646          */
4647         bge_writembx(sc, BGE_MBX_IRQ0_LO, 0);
4648
4649         /*
4650          * Do the mandatory PCI flush as well as get the link status.
4651          */
4652         statusword = CSR_READ_4(sc, BGE_MAC_STS) & BGE_MACSTAT_LINK_CHANGED;
4653
4654         /* Make sure the descriptor ring indexes are coherent. */
4655         bus_dmamap_sync(sc->bge_cdata.bge_status_tag,
4656             sc->bge_cdata.bge_status_map,
4657             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4658         rx_prod = sc->bge_ldata.bge_status_block->bge_idx[0].bge_rx_prod_idx;
4659         tx_cons = sc->bge_ldata.bge_status_block->bge_idx[0].bge_tx_cons_idx;
4660         sc->bge_ldata.bge_status_block->bge_status = 0;
4661         bus_dmamap_sync(sc->bge_cdata.bge_status_tag,
4662             sc->bge_cdata.bge_status_map,
4663             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4664
4665         if ((sc->bge_asicrev == BGE_ASICREV_BCM5700 &&
4666             sc->bge_chipid != BGE_CHIPID_BCM5700_B2) ||
4667             statusword || sc->bge_link_evt)
4668                 bge_link_upd(sc);
4669
4670         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
4671                 /* Check RX return ring producer/consumer. */
4672                 bge_rxeof(sc, rx_prod, 1);
4673         }
4674
4675         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
4676                 /* Check TX ring producer/consumer. */
4677                 bge_txeof(sc, tx_cons);
4678         }
4679
4680         if (ifp->if_drv_flags & IFF_DRV_RUNNING &&
4681             !IFQ_DRV_IS_EMPTY(&ifp->if_snd))
4682                 bge_start_locked(ifp);
4683
4684         BGE_UNLOCK(sc);
4685 }
4686
4687 static void
4688 bge_asf_driver_up(struct bge_softc *sc)
4689 {
4690         if (sc->bge_asf_mode & ASF_STACKUP) {
4691                 /* Send ASF heartbeat aprox. every 2s */
4692                 if (sc->bge_asf_count)
4693                         sc->bge_asf_count --;
4694                 else {
4695                         sc->bge_asf_count = 2;
4696                         bge_writemem_ind(sc, BGE_SRAM_FW_CMD_MB,
4697                             BGE_FW_CMD_DRV_ALIVE);
4698                         bge_writemem_ind(sc, BGE_SRAM_FW_CMD_LEN_MB, 4);
4699                         bge_writemem_ind(sc, BGE_SRAM_FW_CMD_DATA_MB,
4700                             BGE_FW_HB_TIMEOUT_SEC);
4701                         CSR_WRITE_4(sc, BGE_RX_CPU_EVENT,
4702                             CSR_READ_4(sc, BGE_RX_CPU_EVENT) |
4703                             BGE_RX_CPU_DRV_EVENT);
4704                 }
4705         }
4706 }
4707
4708 static void
4709 bge_tick(void *xsc)
4710 {
4711         struct bge_softc *sc = xsc;
4712         struct mii_data *mii = NULL;
4713
4714         BGE_LOCK_ASSERT(sc);
4715
4716         /* Synchronize with possible callout reset/stop. */
4717         if (callout_pending(&sc->bge_stat_ch) ||
4718             !callout_active(&sc->bge_stat_ch))
4719                 return;
4720
4721         if (BGE_IS_5705_PLUS(sc))
4722                 bge_stats_update_regs(sc);
4723         else
4724                 bge_stats_update(sc);
4725
4726         /* XXX Add APE heartbeat check here? */
4727
4728         if ((sc->bge_flags & BGE_FLAG_TBI) == 0) {
4729                 mii = device_get_softc(sc->bge_miibus);
4730                 /*
4731                  * Do not touch PHY if we have link up. This could break
4732                  * IPMI/ASF mode or produce extra input errors
4733                  * (extra errors was reported for bcm5701 & bcm5704).
4734                  */
4735                 if (!sc->bge_link)
4736                         mii_tick(mii);
4737         } else {
4738                 /*
4739                  * Since in TBI mode auto-polling can't be used we should poll
4740                  * link status manually. Here we register pending link event
4741                  * and trigger interrupt.
4742                  */
4743 #ifdef DEVICE_POLLING
4744                 /* In polling mode we poll link state in bge_poll(). */
4745                 if (!(sc->bge_ifp->if_capenable & IFCAP_POLLING))
4746 #endif
4747                 {
4748                 sc->bge_link_evt++;
4749                 if (sc->bge_asicrev == BGE_ASICREV_BCM5700 ||
4750                     sc->bge_flags & BGE_FLAG_5788)
4751                         BGE_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_INTR_SET);
4752                 else
4753                         BGE_SETBIT(sc, BGE_HCC_MODE, BGE_HCCMODE_COAL_NOW);
4754                 }
4755         }
4756
4757         bge_asf_driver_up(sc);
4758         bge_watchdog(sc);
4759
4760         callout_reset(&sc->bge_stat_ch, hz, bge_tick, sc);
4761 }
4762
4763 static void
4764 bge_stats_update_regs(struct bge_softc *sc)
4765 {
4766         struct ifnet *ifp;
4767         struct bge_mac_stats *stats;
4768
4769         ifp = sc->bge_ifp;
4770         stats = &sc->bge_mac_stats;
4771
4772         stats->ifHCOutOctets +=
4773             CSR_READ_4(sc, BGE_TX_MAC_STATS_OCTETS);
4774         stats->etherStatsCollisions +=
4775             CSR_READ_4(sc, BGE_TX_MAC_STATS_COLLS);
4776         stats->outXonSent +=
4777             CSR_READ_4(sc, BGE_TX_MAC_STATS_XON_SENT);
4778         stats->outXoffSent +=
4779             CSR_READ_4(sc, BGE_TX_MAC_STATS_XOFF_SENT);
4780         stats->dot3StatsInternalMacTransmitErrors +=
4781             CSR_READ_4(sc, BGE_TX_MAC_STATS_ERRORS);
4782         stats->dot3StatsSingleCollisionFrames +=
4783             CSR_READ_4(sc, BGE_TX_MAC_STATS_SINGLE_COLL);
4784         stats->dot3StatsMultipleCollisionFrames +=
4785             CSR_READ_4(sc, BGE_TX_MAC_STATS_MULTI_COLL);
4786         stats->dot3StatsDeferredTransmissions +=
4787             CSR_READ_4(sc, BGE_TX_MAC_STATS_DEFERRED);
4788         stats->dot3StatsExcessiveCollisions +=
4789             CSR_READ_4(sc, BGE_TX_MAC_STATS_EXCESS_COLL);
4790         stats->dot3StatsLateCollisions +=
4791             CSR_READ_4(sc, BGE_TX_MAC_STATS_LATE_COLL);
4792         stats->ifHCOutUcastPkts +=
4793             CSR_READ_4(sc, BGE_TX_MAC_STATS_UCAST);
4794         stats->ifHCOutMulticastPkts +=
4795             CSR_READ_4(sc, BGE_TX_MAC_STATS_MCAST);
4796         stats->ifHCOutBroadcastPkts +=
4797             CSR_READ_4(sc, BGE_TX_MAC_STATS_BCAST);
4798
4799         stats->ifHCInOctets +=
4800             CSR_READ_4(sc, BGE_RX_MAC_STATS_OCTESTS);
4801         stats->etherStatsFragments +=
4802             CSR_READ_4(sc, BGE_RX_MAC_STATS_FRAGMENTS);
4803         stats->ifHCInUcastPkts +=
4804             CSR_READ_4(sc, BGE_RX_MAC_STATS_UCAST);
4805         stats->ifHCInMulticastPkts +=
4806             CSR_READ_4(sc, BGE_RX_MAC_STATS_MCAST);
4807         stats->ifHCInBroadcastPkts +=
4808             CSR_READ_4(sc, BGE_RX_MAC_STATS_BCAST);
4809         stats->dot3StatsFCSErrors +=
4810             CSR_READ_4(sc, BGE_RX_MAC_STATS_FCS_ERRORS);
4811         stats->dot3StatsAlignmentErrors +=
4812             CSR_READ_4(sc, BGE_RX_MAC_STATS_ALGIN_ERRORS);
4813         stats->xonPauseFramesReceived +=
4814             CSR_READ_4(sc, BGE_RX_MAC_STATS_XON_RCVD);
4815         stats->xoffPauseFramesReceived +=
4816             CSR_READ_4(sc, BGE_RX_MAC_STATS_XOFF_RCVD);
4817         stats->macControlFramesReceived +=
4818             CSR_READ_4(sc, BGE_RX_MAC_STATS_CTRL_RCVD);
4819         stats->xoffStateEntered +=
4820             CSR_READ_4(sc, BGE_RX_MAC_STATS_XOFF_ENTERED);
4821         stats->dot3StatsFramesTooLong +=
4822             CSR_READ_4(sc, BGE_RX_MAC_STATS_FRAME_TOO_LONG);
4823         stats->etherStatsJabbers +=
4824             CSR_READ_4(sc, BGE_RX_MAC_STATS_JABBERS);
4825         stats->etherStatsUndersizePkts +=
4826             CSR_READ_4(sc, BGE_RX_MAC_STATS_UNDERSIZE);
4827
4828         stats->FramesDroppedDueToFilters +=
4829             CSR_READ_4(sc, BGE_RXLP_LOCSTAT_FILTDROP);
4830         stats->DmaWriteQueueFull +=
4831             CSR_READ_4(sc, BGE_RXLP_LOCSTAT_DMA_WRQ_FULL);
4832         stats->DmaWriteHighPriQueueFull +=
4833             CSR_READ_4(sc, BGE_RXLP_LOCSTAT_DMA_HPWRQ_FULL);
4834         stats->NoMoreRxBDs +=
4835             CSR_READ_4(sc, BGE_RXLP_LOCSTAT_OUT_OF_BDS);
4836         /*
4837          * XXX
4838          * Unlike other controllers, BGE_RXLP_LOCSTAT_IFIN_DROPS
4839          * counter of BCM5717, BCM5718, BCM5719 A0 and BCM5720 A0
4840          * includes number of unwanted multicast frames.  This comes
4841          * from silicon bug and known workaround to get rough(not
4842          * exact) counter is to enable interrupt on MBUF low water
4843          * attention.  This can be accomplished by setting
4844          * BGE_HCCMODE_ATTN bit of BGE_HCC_MODE,
4845          * BGE_BMANMODE_LOMBUF_ATTN bit of BGE_BMAN_MODE and
4846          * BGE_MODECTL_FLOWCTL_ATTN_INTR bit of BGE_MODE_CTL.
4847          * However that change would generate more interrupts and
4848          * there are still possibilities of losing multiple frames
4849          * during BGE_MODECTL_FLOWCTL_ATTN_INTR interrupt handling.
4850          * Given that the workaround still would not get correct
4851          * counter I don't think it's worth to implement it.  So
4852          * ignore reading the counter on controllers that have the
4853          * silicon bug.
4854          */
4855         if (sc->bge_asicrev != BGE_ASICREV_BCM5717 &&
4856             sc->bge_chipid != BGE_CHIPID_BCM5719_A0 &&
4857             sc->bge_chipid != BGE_CHIPID_BCM5720_A0)
4858                 stats->InputDiscards +=
4859                     CSR_READ_4(sc, BGE_RXLP_LOCSTAT_IFIN_DROPS);
4860         stats->InputErrors +=
4861             CSR_READ_4(sc, BGE_RXLP_LOCSTAT_IFIN_ERRORS);
4862         stats->RecvThresholdHit +=
4863             CSR_READ_4(sc, BGE_RXLP_LOCSTAT_RXTHRESH_HIT);
4864
4865         ifp->if_collisions = (u_long)stats->etherStatsCollisions;
4866         ifp->if_ierrors = (u_long)(stats->NoMoreRxBDs + stats->InputDiscards +
4867             stats->InputErrors);
4868 }
4869
4870 static void
4871 bge_stats_clear_regs(struct bge_softc *sc)
4872 {
4873
4874         CSR_READ_4(sc, BGE_TX_MAC_STATS_OCTETS);
4875         CSR_READ_4(sc, BGE_TX_MAC_STATS_COLLS);
4876         CSR_READ_4(sc, BGE_TX_MAC_STATS_XON_SENT);
4877         CSR_READ_4(sc, BGE_TX_MAC_STATS_XOFF_SENT);
4878         CSR_READ_4(sc, BGE_TX_MAC_STATS_ERRORS);
4879         CSR_READ_4(sc, BGE_TX_MAC_STATS_SINGLE_COLL);
4880         CSR_READ_4(sc, BGE_TX_MAC_STATS_MULTI_COLL);
4881         CSR_READ_4(sc, BGE_TX_MAC_STATS_DEFERRED);
4882         CSR_READ_4(sc, BGE_TX_MAC_STATS_EXCESS_COLL);
4883         CSR_READ_4(sc, BGE_TX_MAC_STATS_LATE_COLL);
4884         CSR_READ_4(sc, BGE_TX_MAC_STATS_UCAST);
4885         CSR_READ_4(sc, BGE_TX_MAC_STATS_MCAST);
4886         CSR_READ_4(sc, BGE_TX_MAC_STATS_BCAST);
4887
4888         CSR_READ_4(sc, BGE_RX_MAC_STATS_OCTESTS);
4889         CSR_READ_4(sc, BGE_RX_MAC_STATS_FRAGMENTS);
4890         CSR_READ_4(sc, BGE_RX_MAC_STATS_UCAST);
4891         CSR_READ_4(sc, BGE_RX_MAC_STATS_MCAST);
4892         CSR_READ_4(sc, BGE_RX_MAC_STATS_BCAST);
4893         CSR_READ_4(sc, BGE_RX_MAC_STATS_FCS_ERRORS);
4894         CSR_READ_4(sc, BGE_RX_MAC_STATS_ALGIN_ERRORS);
4895         CSR_READ_4(sc, BGE_RX_MAC_STATS_XON_RCVD);
4896         CSR_READ_4(sc, BGE_RX_MAC_STATS_XOFF_RCVD);
4897         CSR_READ_4(sc, BGE_RX_MAC_STATS_CTRL_RCVD);
4898         CSR_READ_4(sc, BGE_RX_MAC_STATS_XOFF_ENTERED);
4899         CSR_READ_4(sc, BGE_RX_MAC_STATS_FRAME_TOO_LONG);
4900         CSR_READ_4(sc, BGE_RX_MAC_STATS_JABBERS);
4901         CSR_READ_4(sc, BGE_RX_MAC_STATS_UNDERSIZE);
4902
4903         CSR_READ_4(sc, BGE_RXLP_LOCSTAT_FILTDROP);
4904         CSR_READ_4(sc, BGE_RXLP_LOCSTAT_DMA_WRQ_FULL);
4905         CSR_READ_4(sc, BGE_RXLP_LOCSTAT_DMA_HPWRQ_FULL);
4906         CSR_READ_4(sc, BGE_RXLP_LOCSTAT_OUT_OF_BDS);
4907         CSR_READ_4(sc, BGE_RXLP_LOCSTAT_IFIN_DROPS);
4908         CSR_READ_4(sc, BGE_RXLP_LOCSTAT_IFIN_ERRORS);
4909         CSR_READ_4(sc, BGE_RXLP_LOCSTAT_RXTHRESH_HIT);
4910 }
4911
4912 static void
4913 bge_stats_update(struct bge_softc *sc)
4914 {
4915         struct ifnet *ifp;
4916         bus_size_t stats;
4917         uint32_t cnt;   /* current register value */
4918
4919         ifp = sc->bge_ifp;
4920
4921         stats = BGE_MEMWIN_START + BGE_STATS_BLOCK;
4922
4923 #define READ_STAT(sc, stats, stat) \
4924         CSR_READ_4(sc, stats + offsetof(struct bge_stats, stat))
4925
4926         cnt = READ_STAT(sc, stats, txstats.etherStatsCollisions.bge_addr_lo);
4927         ifp->if_collisions += (uint32_t)(cnt - sc->bge_tx_collisions);
4928         sc->bge_tx_collisions = cnt;
4929
4930         cnt = READ_STAT(sc, stats, nicNoMoreRxBDs.bge_addr_lo);
4931         ifp->if_ierrors += (uint32_t)(cnt - sc->bge_rx_nobds);
4932         sc->bge_rx_nobds = cnt;
4933         cnt = READ_STAT(sc, stats, ifInErrors.bge_addr_lo);
4934         ifp->if_ierrors += (uint32_t)(cnt - sc->bge_rx_inerrs);
4935         sc->bge_rx_inerrs = cnt;
4936         cnt = READ_STAT(sc, stats, ifInDiscards.bge_addr_lo);
4937         ifp->if_ierrors += (uint32_t)(cnt - sc->bge_rx_discards);
4938         sc->bge_rx_discards = cnt;
4939
4940         cnt = READ_STAT(sc, stats, txstats.ifOutDiscards.bge_addr_lo);
4941         ifp->if_oerrors += (uint32_t)(cnt - sc->bge_tx_discards);
4942         sc->bge_tx_discards = cnt;
4943
4944 #undef  READ_STAT
4945 }
4946
4947 /*
4948  * Pad outbound frame to ETHER_MIN_NOPAD for an unusual reason.
4949  * The bge hardware will pad out Tx runts to ETHER_MIN_NOPAD,
4950  * but when such padded frames employ the bge IP/TCP checksum offload,
4951  * the hardware checksum assist gives incorrect results (possibly
4952  * from incorporating its own padding into the UDP/TCP checksum; who knows).
4953  * If we pad such runts with zeros, the onboard checksum comes out correct.
4954  */
4955 static __inline int
4956 bge_cksum_pad(struct mbuf *m)
4957 {
4958         int padlen = ETHER_MIN_NOPAD - m->m_pkthdr.len;
4959         struct mbuf *last;
4960
4961         /* If there's only the packet-header and we can pad there, use it. */
4962         if (m->m_pkthdr.len == m->m_len && M_WRITABLE(m) &&
4963             M_TRAILINGSPACE(m) >= padlen) {
4964                 last = m;
4965         } else {
4966                 /*
4967                  * Walk packet chain to find last mbuf. We will either
4968                  * pad there, or append a new mbuf and pad it.
4969                  */
4970                 for (last = m; last->m_next != NULL; last = last->m_next);
4971                 if (!(M_WRITABLE(last) && M_TRAILINGSPACE(last) >= padlen)) {
4972                         /* Allocate new empty mbuf, pad it. Compact later. */
4973                         struct mbuf *n;
4974
4975                         MGET(n, M_NOWAIT, MT_DATA);
4976                         if (n == NULL)
4977                                 return (ENOBUFS);
4978                         n->m_len = 0;
4979                         last->m_next = n;
4980                         last = n;
4981                 }
4982         }
4983
4984         /* Now zero the pad area, to avoid the bge cksum-assist bug. */
4985         memset(mtod(last, caddr_t) + last->m_len, 0, padlen);
4986         last->m_len += padlen;
4987         m->m_pkthdr.len += padlen;
4988
4989         return (0);
4990 }
4991
4992 static struct mbuf *
4993 bge_check_short_dma(struct mbuf *m)
4994 {
4995         struct mbuf *n;
4996         int found;
4997
4998         /*
4999          * If device receive two back-to-back send BDs with less than
5000          * or equal to 8 total bytes then the device may hang.  The two
5001          * back-to-back send BDs must in the same frame for this failure
5002          * to occur.  Scan mbuf chains and see whether two back-to-back
5003          * send BDs are there. If this is the case, allocate new mbuf
5004          * and copy the frame to workaround the silicon bug.
5005          */
5006         for (n = m, found = 0; n != NULL; n = n->m_next) {
5007                 if (n->m_len < 8) {
5008                         found++;
5009                         if (found > 1)
5010                                 break;
5011                         continue;
5012                 }
5013                 found = 0;
5014         }
5015
5016         if (found > 1) {
5017                 n = m_defrag(m, M_NOWAIT);
5018                 if (n == NULL)
5019                         m_freem(m);
5020         } else
5021                 n = m;
5022         return (n);
5023 }
5024
5025 static struct mbuf *
5026 bge_setup_tso(struct bge_softc *sc, struct mbuf *m, uint16_t *mss,
5027     uint16_t *flags)
5028 {
5029         struct ip *ip;
5030         struct tcphdr *tcp;
5031         struct mbuf *n;
5032         uint16_t hlen;
5033         uint32_t poff;
5034
5035         if (M_WRITABLE(m) == 0) {
5036                 /* Get a writable copy. */
5037                 n = m_dup(m, M_NOWAIT);
5038                 m_freem(m);
5039                 if (n == NULL)
5040                         return (NULL);
5041                 m = n;
5042         }
5043         m = m_pullup(m, sizeof(struct ether_header) + sizeof(struct ip));
5044         if (m == NULL)
5045                 return (NULL);
5046         ip = (struct ip *)(mtod(m, char *) + sizeof(struct ether_header));
5047         poff = sizeof(struct ether_header) + (ip->ip_hl << 2);
5048         m = m_pullup(m, poff + sizeof(struct tcphdr));
5049         if (m == NULL)
5050                 return (NULL);
5051         tcp = (struct tcphdr *)(mtod(m, char *) + poff);
5052         m = m_pullup(m, poff + (tcp->th_off << 2));
5053         if (m == NULL)
5054                 return (NULL);
5055         /*
5056          * It seems controller doesn't modify IP length and TCP pseudo
5057          * checksum. These checksum computed by upper stack should be 0.
5058          */
5059         *mss = m->m_pkthdr.tso_segsz;
5060         ip = (struct ip *)(mtod(m, char *) + sizeof(struct ether_header));
5061         ip->ip_sum = 0;
5062         ip->ip_len = htons(*mss + (ip->ip_hl << 2) + (tcp->th_off << 2));
5063         /* Clear pseudo checksum computed by TCP stack. */
5064         tcp = (struct tcphdr *)(mtod(m, char *) + poff);
5065         tcp->th_sum = 0;
5066         /*
5067          * Broadcom controllers uses different descriptor format for
5068          * TSO depending on ASIC revision. Due to TSO-capable firmware
5069          * license issue and lower performance of firmware based TSO
5070          * we only support hardware based TSO.
5071          */
5072         /* Calculate header length, incl. TCP/IP options, in 32 bit units. */
5073         hlen = ((ip->ip_hl << 2) + (tcp->th_off << 2)) >> 2;
5074         if (sc->bge_flags & BGE_FLAG_TSO3) {
5075                 /*
5076                  * For BCM5717 and newer controllers, hardware based TSO
5077                  * uses the 14 lower bits of the bge_mss field to store the
5078                  * MSS and the upper 2 bits to store the lowest 2 bits of
5079                  * the IP/TCP header length.  The upper 6 bits of the header
5080                  * length are stored in the bge_flags[14:10,4] field.  Jumbo
5081                  * frames are supported.
5082                  */
5083                 *mss |= ((hlen & 0x3) << 14);
5084                 *flags |= ((hlen & 0xF8) << 7) | ((hlen & 0x4) << 2);
5085         } else {
5086                 /*
5087                  * For BCM5755 and newer controllers, hardware based TSO uses
5088                  * the lower 11 bits to store the MSS and the upper 5 bits to
5089                  * store the IP/TCP header length. Jumbo frames are not
5090                  * supported.
5091                  */
5092                 *mss |= (hlen << 11);
5093         }
5094         return (m);
5095 }
5096
5097 /*
5098  * Encapsulate an mbuf chain in the tx ring  by coupling the mbuf data
5099  * pointers to descriptors.
5100  */
5101 static int
5102 bge_encap(struct bge_softc *sc, struct mbuf **m_head, uint32_t *txidx)
5103 {
5104         bus_dma_segment_t       segs[BGE_NSEG_NEW];
5105         bus_dmamap_t            map;
5106         struct bge_tx_bd        *d;
5107         struct mbuf             *m = *m_head;
5108         uint32_t                idx = *txidx;
5109         uint16_t                csum_flags, mss, vlan_tag;
5110         int                     nsegs, i, error;
5111
5112         csum_flags = 0;
5113         mss = 0;
5114         vlan_tag = 0;
5115         if ((sc->bge_flags & BGE_FLAG_SHORT_DMA_BUG) != 0 &&
5116             m->m_next != NULL) {
5117                 *m_head = bge_check_short_dma(m);
5118                 if (*m_head == NULL)
5119                         return (ENOBUFS);
5120                 m = *m_head;
5121         }
5122         if ((m->m_pkthdr.csum_flags & CSUM_TSO) != 0) {
5123                 *m_head = m = bge_setup_tso(sc, m, &mss, &csum_flags);
5124                 if (*m_head == NULL)
5125                         return (ENOBUFS);
5126                 csum_flags |= BGE_TXBDFLAG_CPU_PRE_DMA |
5127                     BGE_TXBDFLAG_CPU_POST_DMA;
5128         } else if ((m->m_pkthdr.csum_flags & sc->bge_csum_features) != 0) {
5129                 if (m->m_pkthdr.csum_flags & CSUM_IP)
5130                         csum_flags |= BGE_TXBDFLAG_IP_CSUM;
5131                 if (m->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP)) {
5132                         csum_flags |= BGE_TXBDFLAG_TCP_UDP_CSUM;
5133                         if (m->m_pkthdr.len < ETHER_MIN_NOPAD &&
5134                             (error = bge_cksum_pad(m)) != 0) {
5135                                 m_freem(m);
5136                                 *m_head = NULL;
5137                                 return (error);
5138                         }
5139                 }
5140         }
5141
5142         if ((m->m_pkthdr.csum_flags & CSUM_TSO) == 0) {
5143                 if (sc->bge_flags & BGE_FLAG_JUMBO_FRAME &&
5144                     m->m_pkthdr.len > ETHER_MAX_LEN)
5145                         csum_flags |= BGE_TXBDFLAG_JUMBO_FRAME;
5146                 if (sc->bge_forced_collapse > 0 &&
5147                     (sc->bge_flags & BGE_FLAG_PCIE) != 0 && m->m_next != NULL) {
5148                         /*
5149                          * Forcedly collapse mbuf chains to overcome hardware
5150                          * limitation which only support a single outstanding
5151                          * DMA read operation.
5152                          */
5153                         if (sc->bge_forced_collapse == 1)
5154                                 m = m_defrag(m, M_NOWAIT);
5155                         else
5156                                 m = m_collapse(m, M_NOWAIT,
5157                                     sc->bge_forced_collapse);
5158                         if (m == NULL)
5159                                 m = *m_head;
5160                         *m_head = m;
5161                 }
5162         }
5163
5164         map = sc->bge_cdata.bge_tx_dmamap[idx];
5165         error = bus_dmamap_load_mbuf_sg(sc->bge_cdata.bge_tx_mtag, map, m, segs,
5166             &nsegs, BUS_DMA_NOWAIT);
5167         if (error == EFBIG) {
5168                 m = m_collapse(m, M_NOWAIT, BGE_NSEG_NEW);
5169                 if (m == NULL) {
5170                         m_freem(*m_head);
5171                         *m_head = NULL;
5172                         return (ENOBUFS);
5173                 }
5174                 *m_head = m;
5175                 error = bus_dmamap_load_mbuf_sg(sc->bge_cdata.bge_tx_mtag, map,
5176                     m, segs, &nsegs, BUS_DMA_NOWAIT);
5177                 if (error) {
5178                         m_freem(m);
5179                         *m_head = NULL;
5180                         return (error);
5181                 }
5182         } else if (error != 0)
5183                 return (error);
5184
5185         /* Check if we have enough free send BDs. */
5186         if (sc->bge_txcnt + nsegs >= BGE_TX_RING_CNT) {
5187                 bus_dmamap_unload(sc->bge_cdata.bge_tx_mtag, map);
5188                 return (ENOBUFS);
5189         }
5190
5191         bus_dmamap_sync(sc->bge_cdata.bge_tx_mtag, map, BUS_DMASYNC_PREWRITE);
5192
5193         if (m->m_flags & M_VLANTAG) {
5194                 csum_flags |= BGE_TXBDFLAG_VLAN_TAG;
5195                 vlan_tag = m->m_pkthdr.ether_vtag;
5196         }
5197         for (i = 0; ; i++) {
5198                 d = &sc->bge_ldata.bge_tx_ring[idx];
5199                 d->bge_addr.bge_addr_lo = BGE_ADDR_LO(segs[i].ds_addr);
5200                 d->bge_addr.bge_addr_hi = BGE_ADDR_HI(segs[i].ds_addr);
5201                 d->bge_len = segs[i].ds_len;
5202                 d->bge_flags = csum_flags;
5203                 d->bge_vlan_tag = vlan_tag;
5204                 d->bge_mss = mss;
5205                 if (i == nsegs - 1)
5206                         break;
5207                 BGE_INC(idx, BGE_TX_RING_CNT);
5208         }
5209
5210         /* Mark the last segment as end of packet... */
5211         d->bge_flags |= BGE_TXBDFLAG_END;
5212
5213         /*
5214          * Insure that the map for this transmission
5215          * is placed at the array index of the last descriptor
5216          * in this chain.
5217          */
5218         sc->bge_cdata.bge_tx_dmamap[*txidx] = sc->bge_cdata.bge_tx_dmamap[idx];
5219         sc->bge_cdata.bge_tx_dmamap[idx] = map;
5220         sc->bge_cdata.bge_tx_chain[idx] = m;
5221         sc->bge_txcnt += nsegs;
5222
5223         BGE_INC(idx, BGE_TX_RING_CNT);
5224         *txidx = idx;
5225
5226         return (0);
5227 }
5228
5229 /*
5230  * Main transmit routine. To avoid having to do mbuf copies, we put pointers
5231  * to the mbuf data regions directly in the transmit descriptors.
5232  */
5233 static void
5234 bge_start_locked(struct ifnet *ifp)
5235 {
5236         struct bge_softc *sc;
5237         struct mbuf *m_head;
5238         uint32_t prodidx;
5239         int count;
5240
5241         sc = ifp->if_softc;
5242         BGE_LOCK_ASSERT(sc);
5243
5244         if (!sc->bge_link ||
5245             (ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
5246             IFF_DRV_RUNNING)
5247                 return;
5248
5249         prodidx = sc->bge_tx_prodidx;
5250
5251         for (count = 0; !IFQ_DRV_IS_EMPTY(&ifp->if_snd);) {
5252                 if (sc->bge_txcnt > BGE_TX_RING_CNT - 16) {
5253                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
5254                         break;
5255                 }
5256                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
5257                 if (m_head == NULL)
5258                         break;
5259
5260                 /*
5261                  * Pack the data into the transmit ring. If we
5262                  * don't have room, set the OACTIVE flag and wait
5263                  * for the NIC to drain the ring.
5264                  */
5265                 if (bge_encap(sc, &m_head, &prodidx)) {
5266                         if (m_head == NULL)
5267                                 break;
5268                         IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
5269                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
5270                         break;
5271                 }
5272                 ++count;
5273
5274                 /*
5275                  * If there's a BPF listener, bounce a copy of this frame
5276                  * to him.
5277                  */
5278 #ifdef ETHER_BPF_MTAP
5279                 ETHER_BPF_MTAP(ifp, m_head);
5280 #else
5281                 BPF_MTAP(ifp, m_head);
5282 #endif
5283         }
5284
5285         if (count > 0) {
5286                 bus_dmamap_sync(sc->bge_cdata.bge_tx_ring_tag,
5287                     sc->bge_cdata.bge_tx_ring_map, BUS_DMASYNC_PREWRITE);
5288                 /* Transmit. */
5289                 bge_writembx(sc, BGE_MBX_TX_HOST_PROD0_LO, prodidx);
5290                 /* 5700 b2 errata */
5291                 if (sc->bge_chiprev == BGE_CHIPREV_5700_BX)
5292                         bge_writembx(sc, BGE_MBX_TX_HOST_PROD0_LO, prodidx);
5293
5294                 sc->bge_tx_prodidx = prodidx;
5295
5296                 /*
5297                  * Set a timeout in case the chip goes out to lunch.
5298                  */
5299                 sc->bge_timer = BGE_TX_TIMEOUT;
5300         }
5301 }
5302
5303 /*
5304  * Main transmit routine. To avoid having to do mbuf copies, we put pointers
5305  * to the mbuf data regions directly in the transmit descriptors.
5306  */
5307 static void
5308 bge_start(struct ifnet *ifp)
5309 {
5310         struct bge_softc *sc;
5311
5312         sc = ifp->if_softc;
5313         BGE_LOCK(sc);
5314         bge_start_locked(ifp);
5315         BGE_UNLOCK(sc);
5316 }
5317
5318 static void
5319 bge_init_locked(struct bge_softc *sc)
5320 {
5321         struct ifnet *ifp;
5322         uint16_t *m;
5323         uint32_t mode;
5324
5325         BGE_LOCK_ASSERT(sc);
5326
5327         ifp = sc->bge_ifp;
5328
5329         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
5330                 return;
5331
5332         /* Cancel pending I/O and flush buffers. */
5333         bge_stop(sc);
5334
5335         bge_stop_fw(sc);
5336         bge_sig_pre_reset(sc, BGE_RESET_START);
5337         bge_reset(sc);
5338         bge_sig_legacy(sc, BGE_RESET_START);
5339         bge_sig_post_reset(sc, BGE_RESET_START);
5340
5341         bge_chipinit(sc);
5342
5343         /*
5344          * Init the various state machines, ring
5345          * control blocks and firmware.
5346          */
5347         if (bge_blockinit(sc)) {
5348                 device_printf(sc->bge_dev, "initialization failure\n");
5349                 return;
5350         }
5351
5352         ifp = sc->bge_ifp;
5353
5354         /* Specify MTU. */
5355         CSR_WRITE_4(sc, BGE_RX_MTU, ifp->if_mtu +
5356             ETHER_HDR_LEN + ETHER_CRC_LEN +
5357             (ifp->if_capenable & IFCAP_VLAN_MTU ? ETHER_VLAN_ENCAP_LEN : 0));
5358
5359         /* Load our MAC address. */
5360         m = (uint16_t *)IF_LLADDR(sc->bge_ifp);
5361         CSR_WRITE_4(sc, BGE_MAC_ADDR1_LO, htons(m[0]));
5362         CSR_WRITE_4(sc, BGE_MAC_ADDR1_HI, (htons(m[1]) << 16) | htons(m[2]));
5363
5364         /* Program promiscuous mode. */
5365         bge_setpromisc(sc);
5366
5367         /* Program multicast filter. */
5368         bge_setmulti(sc);
5369
5370         /* Program VLAN tag stripping. */
5371         bge_setvlan(sc);
5372
5373         /* Override UDP checksum offloading. */
5374         if (sc->bge_forced_udpcsum == 0)
5375                 sc->bge_csum_features &= ~CSUM_UDP;
5376         else
5377                 sc->bge_csum_features |= CSUM_UDP;
5378         if (ifp->if_capabilities & IFCAP_TXCSUM &&
5379             ifp->if_capenable & IFCAP_TXCSUM) {
5380                 ifp->if_hwassist &= ~(BGE_CSUM_FEATURES | CSUM_UDP);
5381                 ifp->if_hwassist |= sc->bge_csum_features;
5382         }
5383
5384         /* Init RX ring. */
5385         if (bge_init_rx_ring_std(sc) != 0) {
5386                 device_printf(sc->bge_dev, "no memory for std Rx buffers.\n");
5387                 bge_stop(sc);
5388                 return;
5389         }
5390
5391         /*
5392          * Workaround for a bug in 5705 ASIC rev A0. Poll the NIC's
5393          * memory to insure that the chip has in fact read the first
5394          * entry of the ring.
5395          */
5396         if (sc->bge_chipid == BGE_CHIPID_BCM5705_A0) {
5397                 uint32_t                v, i;
5398                 for (i = 0; i < 10; i++) {
5399                         DELAY(20);
5400                         v = bge_readmem_ind(sc, BGE_STD_RX_RINGS + 8);
5401                         if (v == (MCLBYTES - ETHER_ALIGN))
5402                                 break;
5403                 }
5404                 if (i == 10)
5405                         device_printf (sc->bge_dev,
5406                             "5705 A0 chip failed to load RX ring\n");
5407         }
5408
5409         /* Init jumbo RX ring. */
5410         if (BGE_IS_JUMBO_CAPABLE(sc) &&
5411             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN + ETHER_VLAN_ENCAP_LEN >
5412             (MCLBYTES - ETHER_ALIGN)) {
5413                 if (bge_init_rx_ring_jumbo(sc) != 0) {
5414                         device_printf(sc->bge_dev,
5415                             "no memory for jumbo Rx buffers.\n");
5416                         bge_stop(sc);
5417                         return;
5418                 }
5419         }
5420
5421         /* Init our RX return ring index. */
5422         sc->bge_rx_saved_considx = 0;
5423
5424         /* Init our RX/TX stat counters. */
5425         sc->bge_rx_discards = sc->bge_tx_discards = sc->bge_tx_collisions = 0;
5426
5427         /* Init TX ring. */
5428         bge_init_tx_ring(sc);
5429
5430         /* Enable TX MAC state machine lockup fix. */
5431         mode = CSR_READ_4(sc, BGE_TX_MODE);
5432         if (BGE_IS_5755_PLUS(sc) || sc->bge_asicrev == BGE_ASICREV_BCM5906)
5433                 mode |= BGE_TXMODE_MBUF_LOCKUP_FIX;
5434         if (sc->bge_asicrev == BGE_ASICREV_BCM5720) {
5435                 mode &= ~(BGE_TXMODE_JMB_FRM_LEN | BGE_TXMODE_CNT_DN_MODE);
5436                 mode |= CSR_READ_4(sc, BGE_TX_MODE) &
5437                     (BGE_TXMODE_JMB_FRM_LEN | BGE_TXMODE_CNT_DN_MODE);
5438         }
5439         /* Turn on transmitter. */
5440         CSR_WRITE_4(sc, BGE_TX_MODE, mode | BGE_TXMODE_ENABLE);
5441         DELAY(100);
5442
5443         /* Turn on receiver. */
5444         mode = CSR_READ_4(sc, BGE_RX_MODE);
5445         if (BGE_IS_5755_PLUS(sc))
5446                 mode |= BGE_RXMODE_IPV6_ENABLE;
5447         CSR_WRITE_4(sc,BGE_RX_MODE, mode | BGE_RXMODE_ENABLE);
5448         DELAY(10);
5449
5450         /*
5451          * Set the number of good frames to receive after RX MBUF
5452          * Low Watermark has been reached. After the RX MAC receives
5453          * this number of frames, it will drop subsequent incoming
5454          * frames until the MBUF High Watermark is reached.
5455          */
5456         if (BGE_IS_57765_PLUS(sc))
5457                 CSR_WRITE_4(sc, BGE_MAX_RX_FRAME_LOWAT, 1);
5458         else
5459                 CSR_WRITE_4(sc, BGE_MAX_RX_FRAME_LOWAT, 2);
5460
5461         /* Clear MAC statistics. */
5462         if (BGE_IS_5705_PLUS(sc))
5463                 bge_stats_clear_regs(sc);
5464
5465         /* Tell firmware we're alive. */
5466         BGE_SETBIT(sc, BGE_MODE_CTL, BGE_MODECTL_STACKUP);
5467
5468 #ifdef DEVICE_POLLING
5469         /* Disable interrupts if we are polling. */
5470         if (ifp->if_capenable & IFCAP_POLLING) {
5471                 BGE_SETBIT(sc, BGE_PCI_MISC_CTL,
5472                     BGE_PCIMISCCTL_MASK_PCI_INTR);
5473                 bge_writembx(sc, BGE_MBX_IRQ0_LO, 1);
5474         } else
5475 #endif
5476
5477         /* Enable host interrupts. */
5478         {
5479         BGE_SETBIT(sc, BGE_PCI_MISC_CTL, BGE_PCIMISCCTL_CLEAR_INTA);
5480         BGE_CLRBIT(sc, BGE_PCI_MISC_CTL, BGE_PCIMISCCTL_MASK_PCI_INTR);
5481         bge_writembx(sc, BGE_MBX_IRQ0_LO, 0);
5482         }
5483
5484         ifp->if_drv_flags |= IFF_DRV_RUNNING;
5485         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
5486
5487         bge_ifmedia_upd_locked(ifp);
5488
5489         callout_reset(&sc->bge_stat_ch, hz, bge_tick, sc);
5490 }
5491
5492 static void
5493 bge_init(void *xsc)
5494 {
5495         struct bge_softc *sc = xsc;
5496
5497         BGE_LOCK(sc);
5498         bge_init_locked(sc);
5499         BGE_UNLOCK(sc);
5500 }
5501
5502 /*
5503  * Set media options.
5504  */
5505 static int
5506 bge_ifmedia_upd(struct ifnet *ifp)
5507 {
5508         struct bge_softc *sc = ifp->if_softc;
5509         int res;
5510
5511         BGE_LOCK(sc);
5512         res = bge_ifmedia_upd_locked(ifp);
5513         BGE_UNLOCK(sc);
5514
5515         return (res);
5516 }
5517
5518 static int
5519 bge_ifmedia_upd_locked(struct ifnet *ifp)
5520 {
5521         struct bge_softc *sc = ifp->if_softc;
5522         struct mii_data *mii;
5523         struct mii_softc *miisc;
5524         struct ifmedia *ifm;
5525
5526         BGE_LOCK_ASSERT(sc);
5527
5528         ifm = &sc->bge_ifmedia;
5529
5530         /* If this is a 1000baseX NIC, enable the TBI port. */
5531         if (sc->bge_flags & BGE_FLAG_TBI) {
5532                 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
5533                         return (EINVAL);
5534                 switch(IFM_SUBTYPE(ifm->ifm_media)) {
5535                 case IFM_AUTO:
5536                         /*
5537                          * The BCM5704 ASIC appears to have a special
5538                          * mechanism for programming the autoneg
5539                          * advertisement registers in TBI mode.
5540                          */
5541                         if (sc->bge_asicrev == BGE_ASICREV_BCM5704) {
5542                                 uint32_t sgdig;
5543                                 sgdig = CSR_READ_4(sc, BGE_SGDIG_STS);
5544                                 if (sgdig & BGE_SGDIGSTS_DONE) {
5545                                         CSR_WRITE_4(sc, BGE_TX_TBI_AUTONEG, 0);
5546                                         sgdig = CSR_READ_4(sc, BGE_SGDIG_CFG);
5547                                         sgdig |= BGE_SGDIGCFG_AUTO |
5548                                             BGE_SGDIGCFG_PAUSE_CAP |
5549                                             BGE_SGDIGCFG_ASYM_PAUSE;
5550                                         CSR_WRITE_4(sc, BGE_SGDIG_CFG,
5551                                             sgdig | BGE_SGDIGCFG_SEND);
5552                                         DELAY(5);
5553                                         CSR_WRITE_4(sc, BGE_SGDIG_CFG, sgdig);
5554                                 }
5555                         }
5556                         break;
5557                 case IFM_1000_SX:
5558                         if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) {
5559                                 BGE_CLRBIT(sc, BGE_MAC_MODE,
5560                                     BGE_MACMODE_HALF_DUPLEX);
5561                         } else {
5562                                 BGE_SETBIT(sc, BGE_MAC_MODE,
5563                                     BGE_MACMODE_HALF_DUPLEX);
5564                         }
5565                         DELAY(40);
5566                         break;
5567                 default:
5568                         return (EINVAL);
5569                 }
5570                 return (0);
5571         }
5572
5573         sc->bge_link_evt++;
5574         mii = device_get_softc(sc->bge_miibus);
5575         LIST_FOREACH(miisc, &mii->mii_phys, mii_list)
5576                 PHY_RESET(miisc);
5577         mii_mediachg(mii);
5578
5579         /*
5580          * Force an interrupt so that we will call bge_link_upd
5581          * if needed and clear any pending link state attention.
5582          * Without this we are not getting any further interrupts
5583          * for link state changes and thus will not UP the link and
5584          * not be able to send in bge_start_locked. The only
5585          * way to get things working was to receive a packet and
5586          * get an RX intr.
5587          * bge_tick should help for fiber cards and we might not
5588          * need to do this here if BGE_FLAG_TBI is set but as
5589          * we poll for fiber anyway it should not harm.
5590          */
5591         if (sc->bge_asicrev == BGE_ASICREV_BCM5700 ||
5592             sc->bge_flags & BGE_FLAG_5788)
5593                 BGE_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_INTR_SET);
5594         else
5595                 BGE_SETBIT(sc, BGE_HCC_MODE, BGE_HCCMODE_COAL_NOW);
5596
5597         return (0);
5598 }
5599
5600 /*
5601  * Report current media status.
5602  */
5603 static void
5604 bge_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
5605 {
5606         struct bge_softc *sc = ifp->if_softc;
5607         struct mii_data *mii;
5608
5609         BGE_LOCK(sc);
5610
5611         if ((ifp->if_flags & IFF_UP) == 0) {
5612                 BGE_UNLOCK(sc);
5613                 return;
5614         }
5615         if (sc->bge_flags & BGE_FLAG_TBI) {
5616                 ifmr->ifm_status = IFM_AVALID;
5617                 ifmr->ifm_active = IFM_ETHER;
5618                 if (CSR_READ_4(sc, BGE_MAC_STS) &
5619                     BGE_MACSTAT_TBI_PCS_SYNCHED)
5620                         ifmr->ifm_status |= IFM_ACTIVE;
5621                 else {
5622                         ifmr->ifm_active |= IFM_NONE;
5623                         BGE_UNLOCK(sc);
5624                         return;
5625                 }
5626                 ifmr->ifm_active |= IFM_1000_SX;
5627                 if (CSR_READ_4(sc, BGE_MAC_MODE) & BGE_MACMODE_HALF_DUPLEX)
5628                         ifmr->ifm_active |= IFM_HDX;
5629                 else
5630                         ifmr->ifm_active |= IFM_FDX;
5631                 BGE_UNLOCK(sc);
5632                 return;
5633         }
5634
5635         mii = device_get_softc(sc->bge_miibus);
5636         mii_pollstat(mii);
5637         ifmr->ifm_active = mii->mii_media_active;
5638         ifmr->ifm_status = mii->mii_media_status;
5639
5640         BGE_UNLOCK(sc);
5641 }
5642
5643 static int
5644 bge_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
5645 {
5646         struct bge_softc *sc = ifp->if_softc;
5647         struct ifreq *ifr = (struct ifreq *) data;
5648         struct mii_data *mii;
5649         int flags, mask, error = 0;
5650
5651         switch (command) {
5652         case SIOCSIFMTU:
5653                 if (BGE_IS_JUMBO_CAPABLE(sc) ||
5654                     (sc->bge_flags & BGE_FLAG_JUMBO_STD)) {
5655                         if (ifr->ifr_mtu < ETHERMIN ||
5656                             ifr->ifr_mtu > BGE_JUMBO_MTU) {
5657                                 error = EINVAL;
5658                                 break;
5659                         }
5660                 } else if (ifr->ifr_mtu < ETHERMIN || ifr->ifr_mtu > ETHERMTU) {
5661                         error = EINVAL;
5662                         break;
5663                 }
5664                 BGE_LOCK(sc);
5665                 if (ifp->if_mtu != ifr->ifr_mtu) {
5666                         ifp->if_mtu = ifr->ifr_mtu;
5667                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
5668                                 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
5669                                 bge_init_locked(sc);
5670                         }
5671                 }
5672                 BGE_UNLOCK(sc);
5673                 break;
5674         case SIOCSIFFLAGS:
5675                 BGE_LOCK(sc);
5676                 if (ifp->if_flags & IFF_UP) {
5677                         /*
5678                          * If only the state of the PROMISC flag changed,
5679                          * then just use the 'set promisc mode' command
5680                          * instead of reinitializing the entire NIC. Doing
5681                          * a full re-init means reloading the firmware and
5682                          * waiting for it to start up, which may take a
5683                          * second or two.  Similarly for ALLMULTI.
5684                          */
5685                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
5686                                 flags = ifp->if_flags ^ sc->bge_if_flags;
5687                                 if (flags & IFF_PROMISC)
5688                                         bge_setpromisc(sc);
5689                                 if (flags & IFF_ALLMULTI)
5690                                         bge_setmulti(sc);
5691                         } else
5692                                 bge_init_locked(sc);
5693                 } else {
5694                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
5695                                 bge_stop(sc);
5696                         }
5697                 }
5698                 sc->bge_if_flags = ifp->if_flags;
5699                 BGE_UNLOCK(sc);
5700                 error = 0;
5701                 break;
5702         case SIOCADDMULTI:
5703         case SIOCDELMULTI:
5704                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
5705                         BGE_LOCK(sc);
5706                         bge_setmulti(sc);
5707                         BGE_UNLOCK(sc);
5708                         error = 0;
5709                 }
5710                 break;
5711         case SIOCSIFMEDIA:
5712         case SIOCGIFMEDIA:
5713                 if (sc->bge_flags & BGE_FLAG_TBI) {
5714                         error = ifmedia_ioctl(ifp, ifr,
5715                             &sc->bge_ifmedia, command);
5716                 } else {
5717                         mii = device_get_softc(sc->bge_miibus);
5718                         error = ifmedia_ioctl(ifp, ifr,
5719                             &mii->mii_media, command);
5720                 }
5721                 break;
5722         case SIOCSIFCAP:
5723                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
5724 #ifdef DEVICE_POLLING
5725                 if (mask & IFCAP_POLLING) {
5726                         if (ifr->ifr_reqcap & IFCAP_POLLING) {
5727                                 error = ether_poll_register(bge_poll, ifp);
5728                                 if (error)
5729                                         return (error);
5730                                 BGE_LOCK(sc);
5731                                 BGE_SETBIT(sc, BGE_PCI_MISC_CTL,
5732                                     BGE_PCIMISCCTL_MASK_PCI_INTR);
5733                                 bge_writembx(sc, BGE_MBX_IRQ0_LO, 1);
5734                                 ifp->if_capenable |= IFCAP_POLLING;
5735                                 BGE_UNLOCK(sc);
5736                         } else {
5737                                 error = ether_poll_deregister(ifp);
5738                                 /* Enable interrupt even in error case */
5739                                 BGE_LOCK(sc);
5740                                 BGE_CLRBIT(sc, BGE_PCI_MISC_CTL,
5741                                     BGE_PCIMISCCTL_MASK_PCI_INTR);
5742                                 bge_writembx(sc, BGE_MBX_IRQ0_LO, 0);
5743                                 ifp->if_capenable &= ~IFCAP_POLLING;
5744                                 BGE_UNLOCK(sc);
5745                         }
5746                 }
5747 #endif
5748                 if ((mask & IFCAP_TXCSUM) != 0 &&
5749                     (ifp->if_capabilities & IFCAP_TXCSUM) != 0) {
5750                         ifp->if_capenable ^= IFCAP_TXCSUM;
5751                         if ((ifp->if_capenable & IFCAP_TXCSUM) != 0)
5752                                 ifp->if_hwassist |= sc->bge_csum_features;
5753                         else
5754                                 ifp->if_hwassist &= ~sc->bge_csum_features;
5755                 }
5756
5757                 if ((mask & IFCAP_RXCSUM) != 0 &&
5758                     (ifp->if_capabilities & IFCAP_RXCSUM) != 0)
5759                         ifp->if_capenable ^= IFCAP_RXCSUM;
5760
5761                 if ((mask & IFCAP_TSO4) != 0 &&
5762                     (ifp->if_capabilities & IFCAP_TSO4) != 0) {
5763                         ifp->if_capenable ^= IFCAP_TSO4;
5764                         if ((ifp->if_capenable & IFCAP_TSO4) != 0)
5765                                 ifp->if_hwassist |= CSUM_TSO;
5766                         else
5767                                 ifp->if_hwassist &= ~CSUM_TSO;
5768                 }
5769
5770                 if (mask & IFCAP_VLAN_MTU) {
5771                         ifp->if_capenable ^= IFCAP_VLAN_MTU;
5772                         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
5773                         bge_init(sc);
5774                 }
5775
5776                 if ((mask & IFCAP_VLAN_HWTSO) != 0 &&
5777                     (ifp->if_capabilities & IFCAP_VLAN_HWTSO) != 0)
5778                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
5779                 if ((mask & IFCAP_VLAN_HWTAGGING) != 0 &&
5780                     (ifp->if_capabilities & IFCAP_VLAN_HWTAGGING) != 0) {
5781                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
5782                         if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) == 0)
5783                                 ifp->if_capenable &= ~IFCAP_VLAN_HWTSO;
5784                         BGE_LOCK(sc);
5785                         bge_setvlan(sc);
5786                         BGE_UNLOCK(sc);
5787                 }
5788 #ifdef VLAN_CAPABILITIES
5789                 VLAN_CAPABILITIES(ifp);
5790 #endif
5791                 break;
5792         default:
5793                 error = ether_ioctl(ifp, command, data);
5794                 break;
5795         }
5796
5797         return (error);
5798 }
5799
5800 static void
5801 bge_watchdog(struct bge_softc *sc)
5802 {
5803         struct ifnet *ifp;
5804         uint32_t status;
5805
5806         BGE_LOCK_ASSERT(sc);
5807
5808         if (sc->bge_timer == 0 || --sc->bge_timer)
5809                 return;
5810
5811         /* If pause frames are active then don't reset the hardware. */
5812         if ((CSR_READ_4(sc, BGE_RX_MODE) & BGE_RXMODE_FLOWCTL_ENABLE) != 0) {
5813                 status = CSR_READ_4(sc, BGE_RX_STS);
5814                 if ((status & BGE_RXSTAT_REMOTE_XOFFED) != 0) {
5815                         /*
5816                          * If link partner has us in XOFF state then wait for
5817                          * the condition to clear.
5818                          */
5819                         CSR_WRITE_4(sc, BGE_RX_STS, status);
5820                         sc->bge_timer = BGE_TX_TIMEOUT;
5821                         return;
5822                 } else if ((status & BGE_RXSTAT_RCVD_XOFF) != 0 &&
5823                     (status & BGE_RXSTAT_RCVD_XON) != 0) {
5824                         /*
5825                          * If link partner has us in XOFF state then wait for
5826                          * the condition to clear.
5827                          */
5828                         CSR_WRITE_4(sc, BGE_RX_STS, status);
5829                         sc->bge_timer = BGE_TX_TIMEOUT;
5830                         return;
5831                 }
5832                 /*
5833                  * Any other condition is unexpected and the controller
5834                  * should be reset.
5835                  */
5836         }
5837
5838         ifp = sc->bge_ifp;
5839
5840         if_printf(ifp, "watchdog timeout -- resetting\n");
5841
5842         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
5843         bge_init_locked(sc);
5844
5845         ifp->if_oerrors++;
5846 }
5847
5848 static void
5849 bge_stop_block(struct bge_softc *sc, bus_size_t reg, uint32_t bit)
5850 {
5851         int i;
5852
5853         BGE_CLRBIT(sc, reg, bit);
5854
5855         for (i = 0; i < BGE_TIMEOUT; i++) {
5856                 if ((CSR_READ_4(sc, reg) & bit) == 0)
5857                         return;
5858                 DELAY(100);
5859         }
5860 }
5861
5862 /*
5863  * Stop the adapter and free any mbufs allocated to the
5864  * RX and TX lists.
5865  */
5866 static void
5867 bge_stop(struct bge_softc *sc)
5868 {
5869         struct ifnet *ifp;
5870
5871         BGE_LOCK_ASSERT(sc);
5872
5873         ifp = sc->bge_ifp;
5874
5875         callout_stop(&sc->bge_stat_ch);
5876
5877         /* Disable host interrupts. */
5878         BGE_SETBIT(sc, BGE_PCI_MISC_CTL, BGE_PCIMISCCTL_MASK_PCI_INTR);
5879         bge_writembx(sc, BGE_MBX_IRQ0_LO, 1);
5880
5881         /*
5882          * Tell firmware we're shutting down.
5883          */
5884         bge_stop_fw(sc);
5885         bge_sig_pre_reset(sc, BGE_RESET_SHUTDOWN);
5886
5887         /*
5888          * Disable all of the receiver blocks.
5889          */
5890         bge_stop_block(sc, BGE_RX_MODE, BGE_RXMODE_ENABLE);
5891         bge_stop_block(sc, BGE_RBDI_MODE, BGE_RBDIMODE_ENABLE);
5892         bge_stop_block(sc, BGE_RXLP_MODE, BGE_RXLPMODE_ENABLE);
5893         if (BGE_IS_5700_FAMILY(sc))
5894                 bge_stop_block(sc, BGE_RXLS_MODE, BGE_RXLSMODE_ENABLE);
5895         bge_stop_block(sc, BGE_RDBDI_MODE, BGE_RBDIMODE_ENABLE);
5896         bge_stop_block(sc, BGE_RDC_MODE, BGE_RDCMODE_ENABLE);
5897         bge_stop_block(sc, BGE_RBDC_MODE, BGE_RBDCMODE_ENABLE);
5898
5899         /*
5900          * Disable all of the transmit blocks.
5901          */
5902         bge_stop_block(sc, BGE_SRS_MODE, BGE_SRSMODE_ENABLE);
5903         bge_stop_block(sc, BGE_SBDI_MODE, BGE_SBDIMODE_ENABLE);
5904         bge_stop_block(sc, BGE_SDI_MODE, BGE_SDIMODE_ENABLE);
5905         bge_stop_block(sc, BGE_RDMA_MODE, BGE_RDMAMODE_ENABLE);
5906         bge_stop_block(sc, BGE_SDC_MODE, BGE_SDCMODE_ENABLE);
5907         if (BGE_IS_5700_FAMILY(sc))
5908                 bge_stop_block(sc, BGE_DMAC_MODE, BGE_DMACMODE_ENABLE);
5909         bge_stop_block(sc, BGE_SBDC_MODE, BGE_SBDCMODE_ENABLE);
5910
5911         /*
5912          * Shut down all of the memory managers and related
5913          * state machines.
5914          */
5915         bge_stop_block(sc, BGE_HCC_MODE, BGE_HCCMODE_ENABLE);
5916         bge_stop_block(sc, BGE_WDMA_MODE, BGE_WDMAMODE_ENABLE);
5917         if (BGE_IS_5700_FAMILY(sc))
5918                 bge_stop_block(sc, BGE_MBCF_MODE, BGE_MBCFMODE_ENABLE);
5919
5920         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0xFFFFFFFF);
5921         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0);
5922         if (!(BGE_IS_5705_PLUS(sc))) {
5923                 BGE_CLRBIT(sc, BGE_BMAN_MODE, BGE_BMANMODE_ENABLE);
5924                 BGE_CLRBIT(sc, BGE_MARB_MODE, BGE_MARBMODE_ENABLE);
5925         }
5926         /* Update MAC statistics. */
5927         if (BGE_IS_5705_PLUS(sc))
5928                 bge_stats_update_regs(sc);
5929
5930         bge_reset(sc);
5931         bge_sig_legacy(sc, BGE_RESET_SHUTDOWN);
5932         bge_sig_post_reset(sc, BGE_RESET_SHUTDOWN);
5933
5934         /*
5935          * Keep the ASF firmware running if up.
5936          */
5937         if (sc->bge_asf_mode & ASF_STACKUP)
5938                 BGE_SETBIT(sc, BGE_MODE_CTL, BGE_MODECTL_STACKUP);
5939         else
5940                 BGE_CLRBIT(sc, BGE_MODE_CTL, BGE_MODECTL_STACKUP);
5941
5942         /* Free the RX lists. */
5943         bge_free_rx_ring_std(sc);
5944
5945         /* Free jumbo RX list. */
5946         if (BGE_IS_JUMBO_CAPABLE(sc))
5947                 bge_free_rx_ring_jumbo(sc);
5948
5949         /* Free TX buffers. */
5950         bge_free_tx_ring(sc);
5951
5952         sc->bge_tx_saved_considx = BGE_TXCONS_UNSET;
5953
5954         /* Clear MAC's link state (PHY may still have link UP). */
5955         if (bootverbose && sc->bge_link)
5956                 if_printf(sc->bge_ifp, "link DOWN\n");
5957         sc->bge_link = 0;
5958
5959         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
5960 }
5961
5962 /*
5963  * Stop all chip I/O so that the kernel's probe routines don't
5964  * get confused by errant DMAs when rebooting.
5965  */
5966 static int
5967 bge_shutdown(device_t dev)
5968 {
5969         struct bge_softc *sc;
5970
5971         sc = device_get_softc(dev);
5972         BGE_LOCK(sc);
5973         bge_stop(sc);
5974         BGE_UNLOCK(sc);
5975
5976         return (0);
5977 }
5978
5979 static int
5980 bge_suspend(device_t dev)
5981 {
5982         struct bge_softc *sc;
5983
5984         sc = device_get_softc(dev);
5985         BGE_LOCK(sc);
5986         bge_stop(sc);
5987         BGE_UNLOCK(sc);
5988
5989         return (0);
5990 }
5991
5992 static int
5993 bge_resume(device_t dev)
5994 {
5995         struct bge_softc *sc;
5996         struct ifnet *ifp;
5997
5998         sc = device_get_softc(dev);
5999         BGE_LOCK(sc);
6000         ifp = sc->bge_ifp;
6001         if (ifp->if_flags & IFF_UP) {
6002                 bge_init_locked(sc);
6003                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
6004                         bge_start_locked(ifp);
6005         }
6006         BGE_UNLOCK(sc);
6007
6008         return (0);
6009 }
6010
6011 static void
6012 bge_link_upd(struct bge_softc *sc)
6013 {
6014         struct mii_data *mii;
6015         uint32_t link, status;
6016
6017         BGE_LOCK_ASSERT(sc);
6018
6019         /* Clear 'pending link event' flag. */
6020         sc->bge_link_evt = 0;
6021
6022         /*
6023          * Process link state changes.
6024          * Grrr. The link status word in the status block does
6025          * not work correctly on the BCM5700 rev AX and BX chips,
6026          * according to all available information. Hence, we have
6027          * to enable MII interrupts in order to properly obtain
6028          * async link changes. Unfortunately, this also means that
6029          * we have to read the MAC status register to detect link
6030          * changes, thereby adding an additional register access to
6031          * the interrupt handler.
6032          *
6033          * XXX: perhaps link state detection procedure used for
6034          * BGE_CHIPID_BCM5700_B2 can be used for others BCM5700 revisions.
6035          */
6036
6037         if (sc->bge_asicrev == BGE_ASICREV_BCM5700 &&
6038             sc->bge_chipid != BGE_CHIPID_BCM5700_B2) {
6039                 status = CSR_READ_4(sc, BGE_MAC_STS);
6040                 if (status & BGE_MACSTAT_MI_INTERRUPT) {
6041                         mii = device_get_softc(sc->bge_miibus);
6042                         mii_pollstat(mii);
6043                         if (!sc->bge_link &&
6044                             mii->mii_media_status & IFM_ACTIVE &&
6045                             IFM_SUBTYPE(mii->mii_media_active) != IFM_NONE) {
6046                                 sc->bge_link++;
6047                                 if (bootverbose)
6048                                         if_printf(sc->bge_ifp, "link UP\n");
6049                         } else if (sc->bge_link &&
6050                             (!(mii->mii_media_status & IFM_ACTIVE) ||
6051                             IFM_SUBTYPE(mii->mii_media_active) == IFM_NONE)) {
6052                                 sc->bge_link = 0;
6053                                 if (bootverbose)
6054                                         if_printf(sc->bge_ifp, "link DOWN\n");
6055                         }
6056
6057                         /* Clear the interrupt. */
6058                         CSR_WRITE_4(sc, BGE_MAC_EVT_ENB,
6059                             BGE_EVTENB_MI_INTERRUPT);
6060                         bge_miibus_readreg(sc->bge_dev, sc->bge_phy_addr,
6061                             BRGPHY_MII_ISR);
6062                         bge_miibus_writereg(sc->bge_dev, sc->bge_phy_addr,
6063                             BRGPHY_MII_IMR, BRGPHY_INTRS);
6064                 }
6065                 return;
6066         }
6067
6068         if (sc->bge_flags & BGE_FLAG_TBI) {
6069                 status = CSR_READ_4(sc, BGE_MAC_STS);
6070                 if (status & BGE_MACSTAT_TBI_PCS_SYNCHED) {
6071                         if (!sc->bge_link) {
6072                                 sc->bge_link++;
6073                                 if (sc->bge_asicrev == BGE_ASICREV_BCM5704) {
6074                                         BGE_CLRBIT(sc, BGE_MAC_MODE,
6075                                             BGE_MACMODE_TBI_SEND_CFGS);
6076                                         DELAY(40);
6077                                 }
6078                                 CSR_WRITE_4(sc, BGE_MAC_STS, 0xFFFFFFFF);
6079                                 if (bootverbose)
6080                                         if_printf(sc->bge_ifp, "link UP\n");
6081                                 if_link_state_change(sc->bge_ifp,
6082                                     LINK_STATE_UP);
6083                         }
6084                 } else if (sc->bge_link) {
6085                         sc->bge_link = 0;
6086                         if (bootverbose)
6087                                 if_printf(sc->bge_ifp, "link DOWN\n");
6088                         if_link_state_change(sc->bge_ifp, LINK_STATE_DOWN);
6089                 }
6090         } else if ((sc->bge_mi_mode & BGE_MIMODE_AUTOPOLL) != 0) {
6091                 /*
6092                  * Some broken BCM chips have BGE_STATFLAG_LINKSTATE_CHANGED bit
6093                  * in status word always set. Workaround this bug by reading
6094                  * PHY link status directly.
6095                  */
6096                 link = (CSR_READ_4(sc, BGE_MI_STS) & BGE_MISTS_LINK) ? 1 : 0;
6097
6098                 if (link != sc->bge_link ||
6099                     sc->bge_asicrev == BGE_ASICREV_BCM5700) {
6100                         mii = device_get_softc(sc->bge_miibus);
6101                         mii_pollstat(mii);
6102                         if (!sc->bge_link &&
6103                             mii->mii_media_status & IFM_ACTIVE &&
6104                             IFM_SUBTYPE(mii->mii_media_active) != IFM_NONE) {
6105                                 sc->bge_link++;
6106                                 if (bootverbose)
6107                                         if_printf(sc->bge_ifp, "link UP\n");
6108                         } else if (sc->bge_link &&
6109                             (!(mii->mii_media_status & IFM_ACTIVE) ||
6110                             IFM_SUBTYPE(mii->mii_media_active) == IFM_NONE)) {
6111                                 sc->bge_link = 0;
6112                                 if (bootverbose)
6113                                         if_printf(sc->bge_ifp, "link DOWN\n");
6114                         }
6115                 }
6116         } else {
6117                 /*
6118                  * For controllers that call mii_tick, we have to poll
6119                  * link status.
6120                  */
6121                 mii = device_get_softc(sc->bge_miibus);
6122                 mii_pollstat(mii);
6123                 bge_miibus_statchg(sc->bge_dev);
6124         }
6125
6126         /* Disable MAC attention when link is up. */
6127         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED |
6128             BGE_MACSTAT_CFG_CHANGED | BGE_MACSTAT_MI_COMPLETE |
6129             BGE_MACSTAT_LINK_CHANGED);
6130 }
6131
6132 static void
6133 bge_add_sysctls(struct bge_softc *sc)
6134 {
6135         struct sysctl_ctx_list *ctx;
6136         struct sysctl_oid_list *children;
6137         char tn[32];
6138         int unit;
6139
6140         ctx = device_get_sysctl_ctx(sc->bge_dev);
6141         children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->bge_dev));
6142
6143 #ifdef BGE_REGISTER_DEBUG
6144         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "debug_info",
6145             CTLTYPE_INT | CTLFLAG_RW, sc, 0, bge_sysctl_debug_info, "I",
6146             "Debug Information");
6147
6148         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "reg_read",
6149             CTLTYPE_INT | CTLFLAG_RW, sc, 0, bge_sysctl_reg_read, "I",
6150             "MAC Register Read");
6151
6152         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ape_read",
6153             CTLTYPE_INT | CTLFLAG_RW, sc, 0, bge_sysctl_ape_read, "I",
6154             "APE Register Read");
6155
6156         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mem_read",
6157             CTLTYPE_INT | CTLFLAG_RW, sc, 0, bge_sysctl_mem_read, "I",
6158             "Memory Read");
6159
6160 #endif
6161
6162         unit = device_get_unit(sc->bge_dev);
6163         /*
6164          * A common design characteristic for many Broadcom client controllers
6165          * is that they only support a single outstanding DMA read operation
6166          * on the PCIe bus. This means that it will take twice as long to fetch
6167          * a TX frame that is split into header and payload buffers as it does
6168          * to fetch a single, contiguous TX frame (2 reads vs. 1 read). For
6169          * these controllers, coalescing buffers to reduce the number of memory
6170          * reads is effective way to get maximum performance(about 940Mbps).
6171          * Without collapsing TX buffers the maximum TCP bulk transfer
6172          * performance is about 850Mbps. However forcing coalescing mbufs
6173          * consumes a lot of CPU cycles, so leave it off by default.
6174          */
6175         sc->bge_forced_collapse = 0;
6176         snprintf(tn, sizeof(tn), "dev.bge.%d.forced_collapse", unit);
6177         TUNABLE_INT_FETCH(tn, &sc->bge_forced_collapse);
6178         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "forced_collapse",
6179             CTLFLAG_RW, &sc->bge_forced_collapse, 0,
6180             "Number of fragmented TX buffers of a frame allowed before "
6181             "forced collapsing");
6182
6183         sc->bge_msi = 1;
6184         snprintf(tn, sizeof(tn), "dev.bge.%d.msi", unit);
6185         TUNABLE_INT_FETCH(tn, &sc->bge_msi);
6186         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "msi",
6187             CTLFLAG_RD, &sc->bge_msi, 0, "Enable MSI");
6188
6189         /*
6190          * It seems all Broadcom controllers have a bug that can generate UDP
6191          * datagrams with checksum value 0 when TX UDP checksum offloading is
6192          * enabled.  Generating UDP checksum value 0 is RFC 768 violation.
6193          * Even though the probability of generating such UDP datagrams is
6194          * low, I don't want to see FreeBSD boxes to inject such datagrams
6195          * into network so disable UDP checksum offloading by default.  Users
6196          * still override this behavior by setting a sysctl variable,
6197          * dev.bge.0.forced_udpcsum.
6198          */
6199         sc->bge_forced_udpcsum = 0;
6200         snprintf(tn, sizeof(tn), "dev.bge.%d.bge_forced_udpcsum", unit);
6201         TUNABLE_INT_FETCH(tn, &sc->bge_forced_udpcsum);
6202         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "forced_udpcsum",
6203             CTLFLAG_RW, &sc->bge_forced_udpcsum, 0,
6204             "Enable UDP checksum offloading even if controller can "
6205             "generate UDP checksum value 0");
6206
6207         if (BGE_IS_5705_PLUS(sc))
6208                 bge_add_sysctl_stats_regs(sc, ctx, children);
6209         else
6210                 bge_add_sysctl_stats(sc, ctx, children);
6211 }
6212
6213 #define BGE_SYSCTL_STAT(sc, ctx, desc, parent, node, oid) \
6214         SYSCTL_ADD_PROC(ctx, parent, OID_AUTO, oid, CTLTYPE_UINT|CTLFLAG_RD, \
6215             sc, offsetof(struct bge_stats, node), bge_sysctl_stats, "IU", \
6216             desc)
6217
6218 static void
6219 bge_add_sysctl_stats(struct bge_softc *sc, struct sysctl_ctx_list *ctx,
6220     struct sysctl_oid_list *parent)
6221 {
6222         struct sysctl_oid *tree;
6223         struct sysctl_oid_list *children, *schildren;
6224
6225         tree = SYSCTL_ADD_NODE(ctx, parent, OID_AUTO, "stats", CTLFLAG_RD,
6226             NULL, "BGE Statistics");
6227         schildren = children = SYSCTL_CHILDREN(tree);
6228         BGE_SYSCTL_STAT(sc, ctx, "Frames Dropped Due To Filters",
6229             children, COSFramesDroppedDueToFilters,
6230             "FramesDroppedDueToFilters");
6231         BGE_SYSCTL_STAT(sc, ctx, "NIC DMA Write Queue Full",
6232             children, nicDmaWriteQueueFull, "DmaWriteQueueFull");
6233         BGE_SYSCTL_STAT(sc, ctx, "NIC DMA Write High Priority Queue Full",
6234             children, nicDmaWriteHighPriQueueFull, "DmaWriteHighPriQueueFull");
6235         BGE_SYSCTL_STAT(sc, ctx, "NIC No More RX Buffer Descriptors",
6236             children, nicNoMoreRxBDs, "NoMoreRxBDs");
6237         BGE_SYSCTL_STAT(sc, ctx, "Discarded Input Frames",
6238             children, ifInDiscards, "InputDiscards");
6239         BGE_SYSCTL_STAT(sc, ctx, "Input Errors",
6240             children, ifInErrors, "InputErrors");
6241         BGE_SYSCTL_STAT(sc, ctx, "NIC Recv Threshold Hit",
6242             children, nicRecvThresholdHit, "RecvThresholdHit");
6243         BGE_SYSCTL_STAT(sc, ctx, "NIC DMA Read Queue Full",
6244             children, nicDmaReadQueueFull, "DmaReadQueueFull");
6245         BGE_SYSCTL_STAT(sc, ctx, "NIC DMA Read High Priority Queue Full",
6246             children, nicDmaReadHighPriQueueFull, "DmaReadHighPriQueueFull");
6247         BGE_SYSCTL_STAT(sc, ctx, "NIC Send Data Complete Queue Full",
6248             children, nicSendDataCompQueueFull, "SendDataCompQueueFull");
6249         BGE_SYSCTL_STAT(sc, ctx, "NIC Ring Set Send Producer Index",
6250             children, nicRingSetSendProdIndex, "RingSetSendProdIndex");
6251         BGE_SYSCTL_STAT(sc, ctx, "NIC Ring Status Update",
6252             children, nicRingStatusUpdate, "RingStatusUpdate");
6253         BGE_SYSCTL_STAT(sc, ctx, "NIC Interrupts",
6254             children, nicInterrupts, "Interrupts");
6255         BGE_SYSCTL_STAT(sc, ctx, "NIC Avoided Interrupts",
6256             children, nicAvoidedInterrupts, "AvoidedInterrupts");
6257         BGE_SYSCTL_STAT(sc, ctx, "NIC Send Threshold Hit",
6258             children, nicSendThresholdHit, "SendThresholdHit");
6259
6260         tree = SYSCTL_ADD_NODE(ctx, schildren, OID_AUTO, "rx", CTLFLAG_RD,
6261             NULL, "BGE RX Statistics");
6262         children = SYSCTL_CHILDREN(tree);
6263         BGE_SYSCTL_STAT(sc, ctx, "Inbound Octets",
6264             children, rxstats.ifHCInOctets, "ifHCInOctets");
6265         BGE_SYSCTL_STAT(sc, ctx, "Fragments",
6266             children, rxstats.etherStatsFragments, "Fragments");
6267         BGE_SYSCTL_STAT(sc, ctx, "Inbound Unicast Packets",
6268             children, rxstats.ifHCInUcastPkts, "UnicastPkts");
6269         BGE_SYSCTL_STAT(sc, ctx, "Inbound Multicast Packets",
6270             children, rxstats.ifHCInMulticastPkts, "MulticastPkts");
6271         BGE_SYSCTL_STAT(sc, ctx, "FCS Errors",
6272             children, rxstats.dot3StatsFCSErrors, "FCSErrors");
6273         BGE_SYSCTL_STAT(sc, ctx, "Alignment Errors",
6274             children, rxstats.dot3StatsAlignmentErrors, "AlignmentErrors");
6275         BGE_SYSCTL_STAT(sc, ctx, "XON Pause Frames Received",
6276             children, rxstats.xonPauseFramesReceived, "xonPauseFramesReceived");
6277         BGE_SYSCTL_STAT(sc, ctx, "XOFF Pause Frames Received",
6278             children, rxstats.xoffPauseFramesReceived,
6279             "xoffPauseFramesReceived");
6280         BGE_SYSCTL_STAT(sc, ctx, "MAC Control Frames Received",
6281             children, rxstats.macControlFramesReceived,
6282             "ControlFramesReceived");
6283         BGE_SYSCTL_STAT(sc, ctx, "XOFF State Entered",
6284             children, rxstats.xoffStateEntered, "xoffStateEntered");
6285         BGE_SYSCTL_STAT(sc, ctx, "Frames Too Long",
6286             children, rxstats.dot3StatsFramesTooLong, "FramesTooLong");
6287         BGE_SYSCTL_STAT(sc, ctx, "Jabbers",
6288             children, rxstats.etherStatsJabbers, "Jabbers");
6289         BGE_SYSCTL_STAT(sc, ctx, "Undersized Packets",
6290             children, rxstats.etherStatsUndersizePkts, "UndersizePkts");
6291         BGE_SYSCTL_STAT(sc, ctx, "Inbound Range Length Errors",
6292             children, rxstats.inRangeLengthError, "inRangeLengthError");
6293         BGE_SYSCTL_STAT(sc, ctx, "Outbound Range Length Errors",
6294             children, rxstats.outRangeLengthError, "outRangeLengthError");
6295
6296         tree = SYSCTL_ADD_NODE(ctx, schildren, OID_AUTO, "tx", CTLFLAG_RD,
6297             NULL, "BGE TX Statistics");
6298         children = SYSCTL_CHILDREN(tree);
6299         BGE_SYSCTL_STAT(sc, ctx, "Outbound Octets",
6300             children, txstats.ifHCOutOctets, "ifHCOutOctets");
6301         BGE_SYSCTL_STAT(sc, ctx, "TX Collisions",
6302             children, txstats.etherStatsCollisions, "Collisions");
6303         BGE_SYSCTL_STAT(sc, ctx, "XON Sent",
6304             children, txstats.outXonSent, "XonSent");
6305         BGE_SYSCTL_STAT(sc, ctx, "XOFF Sent",
6306             children, txstats.outXoffSent, "XoffSent");
6307         BGE_SYSCTL_STAT(sc, ctx, "Flow Control Done",
6308             children, txstats.flowControlDone, "flowControlDone");
6309         BGE_SYSCTL_STAT(sc, ctx, "Internal MAC TX errors",
6310             children, txstats.dot3StatsInternalMacTransmitErrors,
6311             "InternalMacTransmitErrors");
6312         BGE_SYSCTL_STAT(sc, ctx, "Single Collision Frames",
6313             children, txstats.dot3StatsSingleCollisionFrames,
6314             "SingleCollisionFrames");
6315         BGE_SYSCTL_STAT(sc, ctx, "Multiple Collision Frames",
6316             children, txstats.dot3StatsMultipleCollisionFrames,
6317             "MultipleCollisionFrames");
6318         BGE_SYSCTL_STAT(sc, ctx, "Deferred Transmissions",
6319             children, txstats.dot3StatsDeferredTransmissions,
6320             "DeferredTransmissions");
6321         BGE_SYSCTL_STAT(sc, ctx, "Excessive Collisions",
6322             children, txstats.dot3StatsExcessiveCollisions,
6323             "ExcessiveCollisions");
6324         BGE_SYSCTL_STAT(sc, ctx, "Late Collisions",
6325             children, txstats.dot3StatsLateCollisions,
6326             "LateCollisions");
6327         BGE_SYSCTL_STAT(sc, ctx, "Outbound Unicast Packets",
6328             children, txstats.ifHCOutUcastPkts, "UnicastPkts");
6329         BGE_SYSCTL_STAT(sc, ctx, "Outbound Multicast Packets",
6330             children, txstats.ifHCOutMulticastPkts, "MulticastPkts");
6331         BGE_SYSCTL_STAT(sc, ctx, "Outbound Broadcast Packets",
6332             children, txstats.ifHCOutBroadcastPkts, "BroadcastPkts");
6333         BGE_SYSCTL_STAT(sc, ctx, "Carrier Sense Errors",
6334             children, txstats.dot3StatsCarrierSenseErrors,
6335             "CarrierSenseErrors");
6336         BGE_SYSCTL_STAT(sc, ctx, "Outbound Discards",
6337             children, txstats.ifOutDiscards, "Discards");
6338         BGE_SYSCTL_STAT(sc, ctx, "Outbound Errors",
6339             children, txstats.ifOutErrors, "Errors");
6340 }
6341
6342 #undef BGE_SYSCTL_STAT
6343
6344 #define BGE_SYSCTL_STAT_ADD64(c, h, n, p, d)    \
6345             SYSCTL_ADD_UQUAD(c, h, OID_AUTO, n, CTLFLAG_RD, p, d)
6346
6347 static void
6348 bge_add_sysctl_stats_regs(struct bge_softc *sc, struct sysctl_ctx_list *ctx,
6349     struct sysctl_oid_list *parent)
6350 {
6351         struct sysctl_oid *tree;
6352         struct sysctl_oid_list *child, *schild;
6353         struct bge_mac_stats *stats;
6354
6355         stats = &sc->bge_mac_stats;
6356         tree = SYSCTL_ADD_NODE(ctx, parent, OID_AUTO, "stats", CTLFLAG_RD,
6357             NULL, "BGE Statistics");
6358         schild = child = SYSCTL_CHILDREN(tree);
6359         BGE_SYSCTL_STAT_ADD64(ctx, child, "FramesDroppedDueToFilters",
6360             &stats->FramesDroppedDueToFilters, "Frames Dropped Due to Filters");
6361         BGE_SYSCTL_STAT_ADD64(ctx, child, "DmaWriteQueueFull",
6362             &stats->DmaWriteQueueFull, "NIC DMA Write Queue Full");
6363         BGE_SYSCTL_STAT_ADD64(ctx, child, "DmaWriteHighPriQueueFull",
6364             &stats->DmaWriteHighPriQueueFull,
6365             "NIC DMA Write High Priority Queue Full");
6366         BGE_SYSCTL_STAT_ADD64(ctx, child, "NoMoreRxBDs",
6367             &stats->NoMoreRxBDs, "NIC No More RX Buffer Descriptors");
6368         BGE_SYSCTL_STAT_ADD64(ctx, child, "InputDiscards",
6369             &stats->InputDiscards, "Discarded Input Frames");
6370         BGE_SYSCTL_STAT_ADD64(ctx, child, "InputErrors",
6371             &stats->InputErrors, "Input Errors");
6372         BGE_SYSCTL_STAT_ADD64(ctx, child, "RecvThresholdHit",
6373             &stats->RecvThresholdHit, "NIC Recv Threshold Hit");
6374
6375         tree = SYSCTL_ADD_NODE(ctx, schild, OID_AUTO, "rx", CTLFLAG_RD,
6376             NULL, "BGE RX Statistics");
6377         child = SYSCTL_CHILDREN(tree);
6378         BGE_SYSCTL_STAT_ADD64(ctx, child, "ifHCInOctets",
6379             &stats->ifHCInOctets, "Inbound Octets");
6380         BGE_SYSCTL_STAT_ADD64(ctx, child, "Fragments",
6381             &stats->etherStatsFragments, "Fragments");
6382         BGE_SYSCTL_STAT_ADD64(ctx, child, "UnicastPkts",
6383             &stats->ifHCInUcastPkts, "Inbound Unicast Packets");
6384         BGE_SYSCTL_STAT_ADD64(ctx, child, "MulticastPkts",
6385             &stats->ifHCInMulticastPkts, "Inbound Multicast Packets");
6386         BGE_SYSCTL_STAT_ADD64(ctx, child, "BroadcastPkts",
6387             &stats->ifHCInBroadcastPkts, "Inbound Broadcast Packets");
6388         BGE_SYSCTL_STAT_ADD64(ctx, child, "FCSErrors",
6389             &stats->dot3StatsFCSErrors, "FCS Errors");
6390         BGE_SYSCTL_STAT_ADD64(ctx, child, "AlignmentErrors",
6391             &stats->dot3StatsAlignmentErrors, "Alignment Errors");
6392         BGE_SYSCTL_STAT_ADD64(ctx, child, "xonPauseFramesReceived",
6393             &stats->xonPauseFramesReceived, "XON Pause Frames Received");
6394         BGE_SYSCTL_STAT_ADD64(ctx, child, "xoffPauseFramesReceived",
6395             &stats->xoffPauseFramesReceived, "XOFF Pause Frames Received");
6396         BGE_SYSCTL_STAT_ADD64(ctx, child, "ControlFramesReceived",
6397             &stats->macControlFramesReceived, "MAC Control Frames Received");
6398         BGE_SYSCTL_STAT_ADD64(ctx, child, "xoffStateEntered",
6399             &stats->xoffStateEntered, "XOFF State Entered");
6400         BGE_SYSCTL_STAT_ADD64(ctx, child, "FramesTooLong",
6401             &stats->dot3StatsFramesTooLong, "Frames Too Long");
6402         BGE_SYSCTL_STAT_ADD64(ctx, child, "Jabbers",
6403             &stats->etherStatsJabbers, "Jabbers");
6404         BGE_SYSCTL_STAT_ADD64(ctx, child, "UndersizePkts",
6405             &stats->etherStatsUndersizePkts, "Undersized Packets");
6406
6407         tree = SYSCTL_ADD_NODE(ctx, schild, OID_AUTO, "tx", CTLFLAG_RD,
6408             NULL, "BGE TX Statistics");
6409         child = SYSCTL_CHILDREN(tree);
6410         BGE_SYSCTL_STAT_ADD64(ctx, child, "ifHCOutOctets",
6411             &stats->ifHCOutOctets, "Outbound Octets");
6412         BGE_SYSCTL_STAT_ADD64(ctx, child, "Collisions",
6413             &stats->etherStatsCollisions, "TX Collisions");
6414         BGE_SYSCTL_STAT_ADD64(ctx, child, "XonSent",
6415             &stats->outXonSent, "XON Sent");
6416         BGE_SYSCTL_STAT_ADD64(ctx, child, "XoffSent",
6417             &stats->outXoffSent, "XOFF Sent");
6418         BGE_SYSCTL_STAT_ADD64(ctx, child, "InternalMacTransmitErrors",
6419             &stats->dot3StatsInternalMacTransmitErrors,
6420             "Internal MAC TX Errors");
6421         BGE_SYSCTL_STAT_ADD64(ctx, child, "SingleCollisionFrames",
6422             &stats->dot3StatsSingleCollisionFrames, "Single Collision Frames");
6423         BGE_SYSCTL_STAT_ADD64(ctx, child, "MultipleCollisionFrames",
6424             &stats->dot3StatsMultipleCollisionFrames,
6425             "Multiple Collision Frames");
6426         BGE_SYSCTL_STAT_ADD64(ctx, child, "DeferredTransmissions",
6427             &stats->dot3StatsDeferredTransmissions, "Deferred Transmissions");
6428         BGE_SYSCTL_STAT_ADD64(ctx, child, "ExcessiveCollisions",
6429             &stats->dot3StatsExcessiveCollisions, "Excessive Collisions");
6430         BGE_SYSCTL_STAT_ADD64(ctx, child, "LateCollisions",
6431             &stats->dot3StatsLateCollisions, "Late Collisions");
6432         BGE_SYSCTL_STAT_ADD64(ctx, child, "UnicastPkts",
6433             &stats->ifHCOutUcastPkts, "Outbound Unicast Packets");
6434         BGE_SYSCTL_STAT_ADD64(ctx, child, "MulticastPkts",
6435             &stats->ifHCOutMulticastPkts, "Outbound Multicast Packets");
6436         BGE_SYSCTL_STAT_ADD64(ctx, child, "BroadcastPkts",
6437             &stats->ifHCOutBroadcastPkts, "Outbound Broadcast Packets");
6438 }
6439
6440 #undef  BGE_SYSCTL_STAT_ADD64
6441
6442 static int
6443 bge_sysctl_stats(SYSCTL_HANDLER_ARGS)
6444 {
6445         struct bge_softc *sc;
6446         uint32_t result;
6447         int offset;
6448
6449         sc = (struct bge_softc *)arg1;
6450         offset = arg2;
6451         result = CSR_READ_4(sc, BGE_MEMWIN_START + BGE_STATS_BLOCK + offset +
6452             offsetof(bge_hostaddr, bge_addr_lo));
6453         return (sysctl_handle_int(oidp, &result, 0, req));
6454 }
6455
6456 #ifdef BGE_REGISTER_DEBUG
6457 static int
6458 bge_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
6459 {
6460         struct bge_softc *sc;
6461         uint16_t *sbdata;
6462         int error, result, sbsz;
6463         int i, j;
6464
6465         result = -1;
6466         error = sysctl_handle_int(oidp, &result, 0, req);
6467         if (error || (req->newptr == NULL))
6468                 return (error);
6469
6470         if (result == 1) {
6471                 sc = (struct bge_softc *)arg1;
6472
6473                 if (sc->bge_asicrev == BGE_ASICREV_BCM5700 &&
6474                     sc->bge_chipid != BGE_CHIPID_BCM5700_C0)
6475                         sbsz = BGE_STATUS_BLK_SZ;
6476                 else
6477                         sbsz = 32;
6478                 sbdata = (uint16_t *)sc->bge_ldata.bge_status_block;
6479                 printf("Status Block:\n");
6480                 BGE_LOCK(sc);
6481                 bus_dmamap_sync(sc->bge_cdata.bge_status_tag,
6482                     sc->bge_cdata.bge_status_map,
6483                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
6484                 for (i = 0x0; i < sbsz / sizeof(uint16_t); ) {
6485                         printf("%06x:", i);
6486                         for (j = 0; j < 8; j++)
6487                                 printf(" %04x", sbdata[i++]);
6488                         printf("\n");
6489                 }
6490
6491                 printf("Registers:\n");
6492                 for (i = 0x800; i < 0xA00; ) {
6493                         printf("%06x:", i);
6494                         for (j = 0; j < 8; j++) {
6495                                 printf(" %08x", CSR_READ_4(sc, i));
6496                                 i += 4;
6497                         }
6498                         printf("\n");
6499                 }
6500                 BGE_UNLOCK(sc);
6501
6502                 printf("Hardware Flags:\n");
6503                 if (BGE_IS_5717_PLUS(sc))
6504                         printf(" - 5717 Plus\n");
6505                 if (BGE_IS_5755_PLUS(sc))
6506                         printf(" - 5755 Plus\n");
6507                 if (BGE_IS_575X_PLUS(sc))
6508                         printf(" - 575X Plus\n");
6509                 if (BGE_IS_5705_PLUS(sc))
6510                         printf(" - 5705 Plus\n");
6511                 if (BGE_IS_5714_FAMILY(sc))
6512                         printf(" - 5714 Family\n");
6513                 if (BGE_IS_5700_FAMILY(sc))
6514                         printf(" - 5700 Family\n");
6515                 if (sc->bge_flags & BGE_FLAG_JUMBO)
6516                         printf(" - Supports Jumbo Frames\n");
6517                 if (sc->bge_flags & BGE_FLAG_PCIX)
6518                         printf(" - PCI-X Bus\n");
6519                 if (sc->bge_flags & BGE_FLAG_PCIE)
6520                         printf(" - PCI Express Bus\n");
6521                 if (sc->bge_phy_flags & BGE_PHY_NO_3LED)
6522                         printf(" - No 3 LEDs\n");
6523                 if (sc->bge_flags & BGE_FLAG_RX_ALIGNBUG)
6524                         printf(" - RX Alignment Bug\n");
6525         }
6526
6527         return (error);
6528 }
6529
6530 static int
6531 bge_sysctl_reg_read(SYSCTL_HANDLER_ARGS)
6532 {
6533         struct bge_softc *sc;
6534         int error;
6535         uint16_t result;
6536         uint32_t val;
6537
6538         result = -1;
6539         error = sysctl_handle_int(oidp, &result, 0, req);
6540         if (error || (req->newptr == NULL))
6541                 return (error);
6542
6543         if (result < 0x8000) {
6544                 sc = (struct bge_softc *)arg1;
6545                 val = CSR_READ_4(sc, result);
6546                 printf("reg 0x%06X = 0x%08X\n", result, val);
6547         }
6548
6549         return (error);
6550 }
6551
6552 static int
6553 bge_sysctl_ape_read(SYSCTL_HANDLER_ARGS)
6554 {
6555         struct bge_softc *sc;
6556         int error;
6557         uint16_t result;
6558         uint32_t val;
6559
6560         result = -1;
6561         error = sysctl_handle_int(oidp, &result, 0, req);
6562         if (error || (req->newptr == NULL))
6563                 return (error);
6564
6565         if (result < 0x8000) {
6566                 sc = (struct bge_softc *)arg1;
6567                 val = APE_READ_4(sc, result);
6568                 printf("reg 0x%06X = 0x%08X\n", result, val);
6569         }
6570
6571         return (error);
6572 }
6573
6574 static int
6575 bge_sysctl_mem_read(SYSCTL_HANDLER_ARGS)
6576 {
6577         struct bge_softc *sc;
6578         int error;
6579         uint16_t result;
6580         uint32_t val;
6581
6582         result = -1;
6583         error = sysctl_handle_int(oidp, &result, 0, req);
6584         if (error || (req->newptr == NULL))
6585                 return (error);
6586
6587         if (result < 0x8000) {
6588                 sc = (struct bge_softc *)arg1;
6589                 val = bge_readmem_ind(sc, result);
6590                 printf("mem 0x%06X = 0x%08X\n", result, val);
6591         }
6592
6593         return (error);
6594 }
6595 #endif
6596
6597 static int
6598 bge_get_eaddr_fw(struct bge_softc *sc, uint8_t ether_addr[])
6599 {
6600
6601         if (sc->bge_flags & BGE_FLAG_EADDR)
6602                 return (1);
6603
6604 #ifdef __sparc64__
6605         OF_getetheraddr(sc->bge_dev, ether_addr);
6606         return (0);
6607 #endif
6608         return (1);
6609 }
6610
6611 static int
6612 bge_get_eaddr_mem(struct bge_softc *sc, uint8_t ether_addr[])
6613 {
6614         uint32_t mac_addr;
6615
6616         mac_addr = bge_readmem_ind(sc, BGE_SRAM_MAC_ADDR_HIGH_MB);
6617         if ((mac_addr >> 16) == 0x484b) {
6618                 ether_addr[0] = (uint8_t)(mac_addr >> 8);
6619                 ether_addr[1] = (uint8_t)mac_addr;
6620                 mac_addr = bge_readmem_ind(sc, BGE_SRAM_MAC_ADDR_LOW_MB);
6621                 ether_addr[2] = (uint8_t)(mac_addr >> 24);
6622                 ether_addr[3] = (uint8_t)(mac_addr >> 16);
6623                 ether_addr[4] = (uint8_t)(mac_addr >> 8);
6624                 ether_addr[5] = (uint8_t)mac_addr;
6625                 return (0);
6626         }
6627         return (1);
6628 }
6629
6630 static int
6631 bge_get_eaddr_nvram(struct bge_softc *sc, uint8_t ether_addr[])
6632 {
6633         int mac_offset = BGE_EE_MAC_OFFSET;
6634
6635         if (sc->bge_asicrev == BGE_ASICREV_BCM5906)
6636                 mac_offset = BGE_EE_MAC_OFFSET_5906;
6637
6638         return (bge_read_nvram(sc, ether_addr, mac_offset + 2,
6639             ETHER_ADDR_LEN));
6640 }
6641
6642 static int
6643 bge_get_eaddr_eeprom(struct bge_softc *sc, uint8_t ether_addr[])
6644 {
6645
6646         if (sc->bge_asicrev == BGE_ASICREV_BCM5906)
6647                 return (1);
6648
6649         return (bge_read_eeprom(sc, ether_addr, BGE_EE_MAC_OFFSET + 2,
6650            ETHER_ADDR_LEN));
6651 }
6652
6653 static int
6654 bge_get_eaddr(struct bge_softc *sc, uint8_t eaddr[])
6655 {
6656         static const bge_eaddr_fcn_t bge_eaddr_funcs[] = {
6657                 /* NOTE: Order is critical */
6658                 bge_get_eaddr_fw,
6659                 bge_get_eaddr_mem,
6660                 bge_get_eaddr_nvram,
6661                 bge_get_eaddr_eeprom,
6662                 NULL
6663         };
6664         const bge_eaddr_fcn_t *func;
6665
6666         for (func = bge_eaddr_funcs; *func != NULL; ++func) {
6667                 if ((*func)(sc, eaddr) == 0)
6668                         break;
6669         }
6670         return (*func == NULL ? ENXIO : 0);
6671 }