sys/dev/re/if_re.c

   1 /*-
   2  * Copyright (c) 1997, 1998-2003
   3  *      Bill Paul <wpaul@windriver.com>.  All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  * 3. All advertising materials mentioning features or use of this software
  14  *    must display the following acknowledgement:
  15  *      This product includes software developed by Bill Paul.
  16  * 4. Neither the name of the author nor the names of any co-contributors
  17  *    may be used to endorse or promote products derived from this software
  18  *    without specific prior written permission.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND
  21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23  * ARE DISCLAIMED.  IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD
  24  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  30  * THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <sys/cdefs.h>
  34 __FBSDID("$FreeBSD$");
  35
  36 /*
  37  * RealTek 8139C+/8169/8169S/8110S/8168/8111/8101E PCI NIC driver
  38  *
  39  * Written by Bill Paul <wpaul@windriver.com>
  40  * Senior Networking Software Engineer
  41  * Wind River Systems
  42  */
  43
  44 /*
  45  * This driver is designed to support RealTek's next generation of
  46  * 10/100 and 10/100/1000 PCI ethernet controllers. There are currently
  47  * seven devices in this family: the RTL8139C+, the RTL8169, the RTL8169S,
  48  * RTL8110S, the RTL8168, the RTL8111 and the RTL8101E.
  49  *
  50  * The 8139C+ is a 10/100 ethernet chip. It is backwards compatible
  51  * with the older 8139 family, however it also supports a special
  52  * C+ mode of operation that provides several new performance enhancing
  53  * features. These include:
  54  *
  55  *      o Descriptor based DMA mechanism. Each descriptor represents
  56  *        a single packet fragment. Data buffers may be aligned on
  57  *        any byte boundary.
  58  *
  59  *      o 64-bit DMA
  60  *
  61  *      o TCP/IP checksum offload for both RX and TX
  62  *
  63  *      o High and normal priority transmit DMA rings
  64  *
  65  *      o VLAN tag insertion and extraction
  66  *
  67  *      o TCP large send (segmentation offload)
  68  *
  69  * Like the 8139, the 8139C+ also has a built-in 10/100 PHY. The C+
  70  * programming API is fairly straightforward. The RX filtering, EEPROM
  71  * access and PHY access is the same as it is on the older 8139 series
  72  * chips.
  73  *
  74  * The 8169 is a 64-bit 10/100/1000 gigabit ethernet MAC. It has almost the
  75  * same programming API and feature set as the 8139C+ with the following
  76  * differences and additions:
  77  *
  78  *      o 1000Mbps mode
  79  *
  80  *      o Jumbo frames
  81  *
  82  *      o GMII and TBI ports/registers for interfacing with copper
  83  *        or fiber PHYs
  84  *
  85  *      o RX and TX DMA rings can have up to 1024 descriptors
  86  *        (the 8139C+ allows a maximum of 64)
  87  *
  88  *      o Slight differences in register layout from the 8139C+
  89  *
  90  * The TX start and timer interrupt registers are at different locations
  91  * on the 8169 than they are on the 8139C+. Also, the status word in the
  92  * RX descriptor has a slightly different bit layout. The 8169 does not
  93  * have a built-in PHY. Most reference boards use a Marvell 88E1000 'Alaska'
  94  * copper gigE PHY.
  95  *
  96  * The 8169S/8110S 10/100/1000 devices have built-in copper gigE PHYs
  97  * (the 'S' stands for 'single-chip'). These devices have the same
  98  * programming API as the older 8169, but also have some vendor-specific
  99  * registers for the on-board PHY. The 8110S is a LAN-on-motherboard
 100  * part designed to be pin-compatible with the RealTek 8100 10/100 chip.
 101  *
 102  * This driver takes advantage of the RX and TX checksum offload and
 103  * VLAN tag insertion/extraction features. It also implements TX
 104  * interrupt moderation using the timer interrupt registers, which
 105  * significantly reduces TX interrupt load. There is also support
 106  * for jumbo frames, however the 8169/8169S/8110S can not transmit
 107  * jumbo frames larger than 7440, so the max MTU possible with this
 108  * driver is 7422 bytes.
 109  */
 110
 111 #ifdef HAVE_KERNEL_OPTION_HEADERS
 112 #include "opt_device_polling.h"
 113 #endif
 114
 115 #include <sys/param.h>
 116 #include <sys/endian.h>
 117 #include <sys/systm.h>
 118 #include <sys/sockio.h>
 119 #include <sys/mbuf.h>
 120 #include <sys/malloc.h>
 121 #include <sys/module.h>
 122 #include <sys/kernel.h>
 123 #include <sys/socket.h>
 124 #include <sys/lock.h>
 125 #include <sys/mutex.h>
 126 #include <sys/sysctl.h>
 127 #include <sys/taskqueue.h>
 128
 129 #include <net/if.h>
 130 #include <net/if_var.h>
 131 #include <net/if_arp.h>
 132 #include <net/ethernet.h>
 133 #include <net/if_dl.h>
 134 #include <net/if_media.h>
 135 #include <net/if_types.h>
 136 #include <net/if_vlan_var.h>
 137
 138 #include <net/bpf.h>
 139
 140 #include <machine/bus.h>
 141 #include <machine/resource.h>
 142 #include <sys/bus.h>
 143 #include <sys/rman.h>
 144
 145 #include <dev/mii/mii.h>
 146 #include <dev/mii/miivar.h>
 147
 148 #include <dev/pci/pcireg.h>
 149 #include <dev/pci/pcivar.h>
 150
 151 #include <dev/rl/if_rlreg.h>
 152
 153 MODULE_DEPEND(re, pci, 1, 1, 1);
 154 MODULE_DEPEND(re, ether, 1, 1, 1);
 155 MODULE_DEPEND(re, miibus, 1, 1, 1);
 156
 157 /* "device miibus" required.  See GENERIC if you get errors here. */
 158 #include "miibus_if.h"
 159
 160 /* Tunables. */
 161 static int intr_filter = 0;
 162 TUNABLE_INT("hw.re.intr_filter", &intr_filter);
 163 static int msi_disable = 0;
 164 TUNABLE_INT("hw.re.msi_disable", &msi_disable);
 165 static int msix_disable = 0;
 166 TUNABLE_INT("hw.re.msix_disable", &msix_disable);
 167 static int prefer_iomap = 0;
 168 TUNABLE_INT("hw.re.prefer_iomap", &prefer_iomap);
 169
 170 #define RE_CSUM_FEATURES    (CSUM_IP | CSUM_TCP | CSUM_UDP)
 171
 172 /*
 173  * Various supported device vendors/types and their names.
 174  */
 175 static const struct rl_type re_devs[] = {
 176         { DLINK_VENDORID, DLINK_DEVICEID_528T, 0,
 177             "D-Link DGE-528(T) Gigabit Ethernet Adapter" },
 178         { DLINK_VENDORID, DLINK_DEVICEID_530T_REVC, 0,
 179             "D-Link DGE-530(T) Gigabit Ethernet Adapter" },
 180         { RT_VENDORID, RT_DEVICEID_8139, 0,
 181             "RealTek 8139C+ 10/100BaseTX" },
 182         { RT_VENDORID, RT_DEVICEID_8101E, 0,
 183             "RealTek 810xE PCIe 10/100baseTX" },
 184         { RT_VENDORID, RT_DEVICEID_8168, 0,
 185             "RealTek 8168/8111 B/C/CP/D/DP/E/F/G PCIe Gigabit Ethernet" },
 186         { RT_VENDORID, RT_DEVICEID_8169, 0,
 187             "RealTek 8169/8169S/8169SB(L)/8110S/8110SB(L) Gigabit Ethernet" },
 188         { RT_VENDORID, RT_DEVICEID_8169SC, 0,
 189             "RealTek 8169SC/8110SC Single-chip Gigabit Ethernet" },
 190         { COREGA_VENDORID, COREGA_DEVICEID_CGLAPCIGT, 0,
 191             "Corega CG-LAPCIGT (RTL8169S) Gigabit Ethernet" },
 192         { LINKSYS_VENDORID, LINKSYS_DEVICEID_EG1032, 0,
 193             "Linksys EG1032 (RTL8169S) Gigabit Ethernet" },
 194         { USR_VENDORID, USR_DEVICEID_997902, 0,
 195             "US Robotics 997902 (RTL8169S) Gigabit Ethernet" }
 196 };
 197
 198 static const struct rl_hwrev re_hwrevs[] = {
 199         { RL_HWREV_8139, RL_8139, "", RL_MTU },
 200         { RL_HWREV_8139A, RL_8139, "A", RL_MTU },
 201         { RL_HWREV_8139AG, RL_8139, "A-G", RL_MTU },
 202         { RL_HWREV_8139B, RL_8139, "B", RL_MTU },
 203         { RL_HWREV_8130, RL_8139, "8130", RL_MTU },
 204         { RL_HWREV_8139C, RL_8139, "C", RL_MTU },
 205         { RL_HWREV_8139D, RL_8139, "8139D/8100B/8100C", RL_MTU },
 206         { RL_HWREV_8139CPLUS, RL_8139CPLUS, "C+", RL_MTU },
 207         { RL_HWREV_8168B_SPIN1, RL_8169, "8168", RL_JUMBO_MTU },
 208         { RL_HWREV_8169, RL_8169, "8169", RL_JUMBO_MTU },
 209         { RL_HWREV_8169S, RL_8169, "8169S", RL_JUMBO_MTU },
 210         { RL_HWREV_8110S, RL_8169, "8110S", RL_JUMBO_MTU },
 211         { RL_HWREV_8169_8110SB, RL_8169, "8169SB/8110SB", RL_JUMBO_MTU },
 212         { RL_HWREV_8169_8110SC, RL_8169, "8169SC/8110SC", RL_JUMBO_MTU },
 213         { RL_HWREV_8169_8110SBL, RL_8169, "8169SBL/8110SBL", RL_JUMBO_MTU },
 214         { RL_HWREV_8169_8110SCE, RL_8169, "8169SC/8110SC", RL_JUMBO_MTU },
 215         { RL_HWREV_8100, RL_8139, "8100", RL_MTU },
 216         { RL_HWREV_8101, RL_8139, "8101", RL_MTU },
 217         { RL_HWREV_8100E, RL_8169, "8100E", RL_MTU },
 218         { RL_HWREV_8101E, RL_8169, "8101E", RL_MTU },
 219         { RL_HWREV_8102E, RL_8169, "8102E", RL_MTU },
 220         { RL_HWREV_8102EL, RL_8169, "8102EL", RL_MTU },
 221         { RL_HWREV_8102EL_SPIN1, RL_8169, "8102EL", RL_MTU },
 222         { RL_HWREV_8103E, RL_8169, "8103E", RL_MTU },
 223         { RL_HWREV_8401E, RL_8169, "8401E", RL_MTU },
 224         { RL_HWREV_8402, RL_8169, "8402", RL_MTU },
 225         { RL_HWREV_8105E, RL_8169, "8105E", RL_MTU },
 226         { RL_HWREV_8105E_SPIN1, RL_8169, "8105E", RL_MTU },
 227         { RL_HWREV_8106E, RL_8169, "8106E", RL_MTU },
 228         { RL_HWREV_8168B_SPIN2, RL_8169, "8168", RL_JUMBO_MTU },
 229         { RL_HWREV_8168B_SPIN3, RL_8169, "8168", RL_JUMBO_MTU },
 230         { RL_HWREV_8168C, RL_8169, "8168C/8111C", RL_JUMBO_MTU_6K },
 231         { RL_HWREV_8168C_SPIN2, RL_8169, "8168C/8111C", RL_JUMBO_MTU_6K },
 232         { RL_HWREV_8168CP, RL_8169, "8168CP/8111CP", RL_JUMBO_MTU_6K },
 233         { RL_HWREV_8168D, RL_8169, "8168D/8111D", RL_JUMBO_MTU_9K },
 234         { RL_HWREV_8168DP, RL_8169, "8168DP/8111DP", RL_JUMBO_MTU_9K },
 235         { RL_HWREV_8168E, RL_8169, "8168E/8111E", RL_JUMBO_MTU_9K},
 236         { RL_HWREV_8168E_VL, RL_8169, "8168E/8111E-VL", RL_JUMBO_MTU_6K},
 237         { RL_HWREV_8168EP, RL_8169, "8168EP/8111EP", RL_JUMBO_MTU_9K},
 238         { RL_HWREV_8168F, RL_8169, "8168F/8111F", RL_JUMBO_MTU_9K},
 239         { RL_HWREV_8168G, RL_8169, "8168G/8111G", RL_JUMBO_MTU_9K},
 240         { RL_HWREV_8168GU, RL_8169, "8168GU/8111GU", RL_JUMBO_MTU_9K},
 241         { RL_HWREV_8411, RL_8169, "8411", RL_JUMBO_MTU_9K},
 242         { RL_HWREV_8411B, RL_8169, "8411B", RL_JUMBO_MTU_9K},
 243         { 0, 0, NULL, 0 }
 244 };
 245
 246 static int re_probe             (device_t);
 247 static int re_attach            (device_t);
 248 static int re_detach            (device_t);
 249
 250 static int re_encap             (struct rl_softc *, struct mbuf **);
 251
 252 static void re_dma_map_addr     (void *, bus_dma_segment_t *, int, int);
 253 static int re_allocmem          (device_t, struct rl_softc *);
 254 static __inline void re_discard_rxbuf
 255                                 (struct rl_softc *, int);
 256 static int re_newbuf            (struct rl_softc *, int);
 257 static int re_jumbo_newbuf      (struct rl_softc *, int);
 258 static int re_rx_list_init      (struct rl_softc *);
 259 static int re_jrx_list_init     (struct rl_softc *);
 260 static int re_tx_list_init      (struct rl_softc *);
 261 #ifdef RE_FIXUP_RX
 262 static __inline void re_fixup_rx
 263                                 (struct mbuf *);
 264 #endif
 265 static int re_rxeof             (struct rl_softc *, int *);
 266 static void re_txeof            (struct rl_softc *);
 267 #ifdef DEVICE_POLLING
 268 static int re_poll              (struct ifnet *, enum poll_cmd, int);
 269 static int re_poll_locked       (struct ifnet *, enum poll_cmd, int);
 270 #endif
 271 static int re_intr              (void *);
 272 static void re_intr_msi         (void *);
 273 static void re_tick             (void *);
 274 static void re_int_task         (void *, int);
 275 static void re_start            (struct ifnet *);
 276 static void re_start_locked     (struct ifnet *);
 277 static int re_ioctl             (struct ifnet *, u_long, caddr_t);
 278 static void re_init             (void *);
 279 static void re_init_locked      (struct rl_softc *);
 280 static void re_stop             (struct rl_softc *);
 281 static void re_watchdog         (struct rl_softc *);
 282 static int re_suspend           (device_t);
 283 static int re_resume            (device_t);
 284 static int re_shutdown          (device_t);
 285 static int re_ifmedia_upd       (struct ifnet *);
 286 static void re_ifmedia_sts      (struct ifnet *, struct ifmediareq *);
 287
 288 static void re_eeprom_putbyte   (struct rl_softc *, int);
 289 static void re_eeprom_getword   (struct rl_softc *, int, u_int16_t *);
 290 static void re_read_eeprom      (struct rl_softc *, caddr_t, int, int);
 291 static int re_gmii_readreg      (device_t, int, int);
 292 static int re_gmii_writereg     (device_t, int, int, int);
 293
 294 static int re_miibus_readreg    (device_t, int, int);
 295 static int re_miibus_writereg   (device_t, int, int, int);
 296 static void re_miibus_statchg   (device_t);
 297
 298 static void re_set_jumbo        (struct rl_softc *, int);
 299 static void re_set_rxmode               (struct rl_softc *);
 300 static void re_reset            (struct rl_softc *);
 301 static void re_setwol           (struct rl_softc *);
 302 static void re_clrwol           (struct rl_softc *);
 303 static void re_set_linkspeed    (struct rl_softc *);
 304
 305 #ifdef DEV_NETMAP       /* see ixgbe.c for details */
 306 #include <dev/netmap/if_re_netmap.h>
 307 #endif /* !DEV_NETMAP */
 308
 309 #ifdef RE_DIAG
 310 static int re_diag              (struct rl_softc *);
 311 #endif
 312
 313 static void re_add_sysctls      (struct rl_softc *);
 314 static int re_sysctl_stats      (SYSCTL_HANDLER_ARGS);
 315 static int sysctl_int_range     (SYSCTL_HANDLER_ARGS, int, int);
 316 static int sysctl_hw_re_int_mod (SYSCTL_HANDLER_ARGS);
 317
 318 static device_method_t re_methods[] = {
 319         /* Device interface */
 320         DEVMETHOD(device_probe,         re_probe),
 321         DEVMETHOD(device_attach,        re_attach),
 322         DEVMETHOD(device_detach,        re_detach),
 323         DEVMETHOD(device_suspend,       re_suspend),
 324         DEVMETHOD(device_resume,        re_resume),
 325         DEVMETHOD(device_shutdown,      re_shutdown),
 326
 327         /* MII interface */
 328         DEVMETHOD(miibus_readreg,       re_miibus_readreg),
 329         DEVMETHOD(miibus_writereg,      re_miibus_writereg),
 330         DEVMETHOD(miibus_statchg,       re_miibus_statchg),
 331
 332         DEVMETHOD_END
 333 };
 334
 335 static driver_t re_driver = {
 336         "re",
 337         re_methods,
 338         sizeof(struct rl_softc)
 339 };
 340
 341 static devclass_t re_devclass;
 342
 343 DRIVER_MODULE(re, pci, re_driver, re_devclass, 0, 0);
 344 DRIVER_MODULE(miibus, re, miibus_driver, miibus_devclass, 0, 0);
 345
 346 #define EE_SET(x)                                       \
 347         CSR_WRITE_1(sc, RL_EECMD,                       \
 348                 CSR_READ_1(sc, RL_EECMD) | x)
 349
 350 #define EE_CLR(x)                                       \
 351         CSR_WRITE_1(sc, RL_EECMD,                       \
 352                 CSR_READ_1(sc, RL_EECMD) & ~x)
 353
 354 /*
 355  * Send a read command and address to the EEPROM, check for ACK.
 356  */
 357 static void
 358 re_eeprom_putbyte(struct rl_softc *sc, int addr)
 359 {
 360         int                     d, i;
 361
 362         d = addr | (RL_9346_READ << sc->rl_eewidth);
 363
 364         /*
 365          * Feed in each bit and strobe the clock.
 366          */
 367
 368         for (i = 1 << (sc->rl_eewidth + 3); i; i >>= 1) {
 369                 if (d & i) {
 370                         EE_SET(RL_EE_DATAIN);
 371                 } else {
 372                         EE_CLR(RL_EE_DATAIN);
 373                 }
 374                 DELAY(100);
 375                 EE_SET(RL_EE_CLK);
 376                 DELAY(150);
 377                 EE_CLR(RL_EE_CLK);
 378                 DELAY(100);
 379         }
 380 }
 381
 382 /*
 383  * Read a word of data stored in the EEPROM at address 'addr.'
 384  */
 385 static void
 386 re_eeprom_getword(struct rl_softc *sc, int addr, u_int16_t *dest)
 387 {
 388         int                     i;
 389         u_int16_t               word = 0;
 390
 391         /*
 392          * Send address of word we want to read.
 393          */
 394         re_eeprom_putbyte(sc, addr);
 395
 396         /*
 397          * Start reading bits from EEPROM.
 398          */
 399         for (i = 0x8000; i; i >>= 1) {
 400                 EE_SET(RL_EE_CLK);
 401                 DELAY(100);
 402                 if (CSR_READ_1(sc, RL_EECMD) & RL_EE_DATAOUT)
 403                         word |= i;
 404                 EE_CLR(RL_EE_CLK);
 405                 DELAY(100);
 406         }
 407
 408         *dest = word;
 409 }
 410
 411 /*
 412  * Read a sequence of words from the EEPROM.
 413  */
 414 static void
 415 re_read_eeprom(struct rl_softc *sc, caddr_t dest, int off, int cnt)
 416 {
 417         int                     i;
 418         u_int16_t               word = 0, *ptr;
 419
 420         CSR_SETBIT_1(sc, RL_EECMD, RL_EEMODE_PROGRAM);
 421
 422         DELAY(100);
 423
 424         for (i = 0; i < cnt; i++) {
 425                 CSR_SETBIT_1(sc, RL_EECMD, RL_EE_SEL);
 426                 re_eeprom_getword(sc, off + i, &word);
 427                 CSR_CLRBIT_1(sc, RL_EECMD, RL_EE_SEL);
 428                 ptr = (u_int16_t *)(dest + (i * 2));
 429                 *ptr = word;
 430         }
 431
 432         CSR_CLRBIT_1(sc, RL_EECMD, RL_EEMODE_PROGRAM);
 433 }
 434
 435 static int
 436 re_gmii_readreg(device_t dev, int phy, int reg)
 437 {
 438         struct rl_softc         *sc;
 439         u_int32_t               rval;
 440         int                     i;
 441
 442         sc = device_get_softc(dev);
 443
 444         /* Let the rgephy driver read the GMEDIASTAT register */
 445
 446         if (reg == RL_GMEDIASTAT) {
 447                 rval = CSR_READ_1(sc, RL_GMEDIASTAT);
 448                 return (rval);
 449         }
 450
 451         CSR_WRITE_4(sc, RL_PHYAR, reg << 16);
 452
 453         for (i = 0; i < RL_PHY_TIMEOUT; i++) {
 454                 rval = CSR_READ_4(sc, RL_PHYAR);
 455                 if (rval & RL_PHYAR_BUSY)
 456                         break;
 457                 DELAY(25);
 458         }
 459
 460         if (i == RL_PHY_TIMEOUT) {
 461                 device_printf(sc->rl_dev, "PHY read failed\n");
 462                 return (0);
 463         }
 464
 465         /*
 466          * Controller requires a 20us delay to process next MDIO request.
 467          */
 468         DELAY(20);
 469
 470         return (rval & RL_PHYAR_PHYDATA);
 471 }
 472
 473 static int
 474 re_gmii_writereg(device_t dev, int phy, int reg, int data)
 475 {
 476         struct rl_softc         *sc;
 477         u_int32_t               rval;
 478         int                     i;
 479
 480         sc = device_get_softc(dev);
 481
 482         CSR_WRITE_4(sc, RL_PHYAR, (reg << 16) |
 483             (data & RL_PHYAR_PHYDATA) | RL_PHYAR_BUSY);
 484
 485         for (i = 0; i < RL_PHY_TIMEOUT; i++) {
 486                 rval = CSR_READ_4(sc, RL_PHYAR);
 487                 if (!(rval & RL_PHYAR_BUSY))
 488                         break;
 489                 DELAY(25);
 490         }
 491
 492         if (i == RL_PHY_TIMEOUT) {
 493                 device_printf(sc->rl_dev, "PHY write failed\n");
 494                 return (0);
 495         }
 496
 497         /*
 498          * Controller requires a 20us delay to process next MDIO request.
 499          */
 500         DELAY(20);
 501
 502         return (0);
 503 }
 504
 505 static int
 506 re_miibus_readreg(device_t dev, int phy, int reg)
 507 {
 508         struct rl_softc         *sc;
 509         u_int16_t               rval = 0;
 510         u_int16_t               re8139_reg = 0;
 511
 512         sc = device_get_softc(dev);
 513
 514         if (sc->rl_type == RL_8169) {
 515                 rval = re_gmii_readreg(dev, phy, reg);
 516                 return (rval);
 517         }
 518
 519         switch (reg) {
 520         case MII_BMCR:
 521                 re8139_reg = RL_BMCR;
 522                 break;
 523         case MII_BMSR:
 524                 re8139_reg = RL_BMSR;
 525                 break;
 526         case MII_ANAR:
 527                 re8139_reg = RL_ANAR;
 528                 break;
 529         case MII_ANER:
 530                 re8139_reg = RL_ANER;
 531                 break;
 532         case MII_ANLPAR:
 533                 re8139_reg = RL_LPAR;
 534                 break;
 535         case MII_PHYIDR1:
 536         case MII_PHYIDR2:
 537                 return (0);
 538         /*
 539          * Allow the rlphy driver to read the media status
 540          * register. If we have a link partner which does not
 541          * support NWAY, this is the register which will tell
 542          * us the results of parallel detection.
 543          */
 544         case RL_MEDIASTAT:
 545                 rval = CSR_READ_1(sc, RL_MEDIASTAT);
 546                 return (rval);
 547         default:
 548                 device_printf(sc->rl_dev, "bad phy register\n");
 549                 return (0);
 550         }
 551         rval = CSR_READ_2(sc, re8139_reg);
 552         if (sc->rl_type == RL_8139CPLUS && re8139_reg == RL_BMCR) {
 553                 /* 8139C+ has different bit layout. */
 554                 rval &= ~(BMCR_LOOP | BMCR_ISO);
 555         }
 556         return (rval);
 557 }
 558
 559 static int
 560 re_miibus_writereg(device_t dev, int phy, int reg, int data)
 561 {
 562         struct rl_softc         *sc;
 563         u_int16_t               re8139_reg = 0;
 564         int                     rval = 0;
 565
 566         sc = device_get_softc(dev);
 567
 568         if (sc->rl_type == RL_8169) {
 569                 rval = re_gmii_writereg(dev, phy, reg, data);
 570                 return (rval);
 571         }
 572
 573         switch (reg) {
 574         case MII_BMCR:
 575                 re8139_reg = RL_BMCR;
 576                 if (sc->rl_type == RL_8139CPLUS) {
 577                         /* 8139C+ has different bit layout. */
 578                         data &= ~(BMCR_LOOP | BMCR_ISO);
 579                 }
 580                 break;
 581         case MII_BMSR:
 582                 re8139_reg = RL_BMSR;
 583                 break;
 584         case MII_ANAR:
 585                 re8139_reg = RL_ANAR;
 586                 break;
 587         case MII_ANER:
 588                 re8139_reg = RL_ANER;
 589                 break;
 590         case MII_ANLPAR:
 591                 re8139_reg = RL_LPAR;
 592                 break;
 593         case MII_PHYIDR1:
 594         case MII_PHYIDR2:
 595                 return (0);
 596                 break;
 597         default:
 598                 device_printf(sc->rl_dev, "bad phy register\n");
 599                 return (0);
 600         }
 601         CSR_WRITE_2(sc, re8139_reg, data);
 602         return (0);
 603 }
 604
 605 static void
 606 re_miibus_statchg(device_t dev)
 607 {
 608         struct rl_softc         *sc;
 609         struct ifnet            *ifp;
 610         struct mii_data         *mii;
 611
 612         sc = device_get_softc(dev);
 613         mii = device_get_softc(sc->rl_miibus);
 614         ifp = sc->rl_ifp;
 615         if (mii == NULL || ifp == NULL ||
 616             (ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 617                 return;
 618
 619         sc->rl_flags &= ~RL_FLAG_LINK;
 620         if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) ==
 621             (IFM_ACTIVE | IFM_AVALID)) {
 622                 switch (IFM_SUBTYPE(mii->mii_media_active)) {
 623                 case IFM_10_T:
 624                 case IFM_100_TX:
 625                         sc->rl_flags |= RL_FLAG_LINK;
 626                         break;
 627                 case IFM_1000_T:
 628                         if ((sc->rl_flags & RL_FLAG_FASTETHER) != 0)
 629                                 break;
 630                         sc->rl_flags |= RL_FLAG_LINK;
 631                         break;
 632                 default:
 633                         break;
 634                 }
 635         }
 636         /*
 637          * RealTek controllers does not provide any interface to
 638          * Tx/Rx MACs for resolved speed, duplex and flow-control
 639          * parameters.
 640          */
 641 }
 642
 643 /*
 644  * Set the RX configuration and 64-bit multicast hash filter.
 645  */
 646 static void
 647 re_set_rxmode(struct rl_softc *sc)
 648 {
 649         struct ifnet            *ifp;
 650         struct ifmultiaddr      *ifma;
 651         uint32_t                hashes[2] = { 0, 0 };
 652         uint32_t                h, rxfilt;
 653
 654         RL_LOCK_ASSERT(sc);
 655
 656         ifp = sc->rl_ifp;
 657
 658         rxfilt = RL_RXCFG_CONFIG | RL_RXCFG_RX_INDIV | RL_RXCFG_RX_BROAD;
 659         if ((sc->rl_flags & RL_FLAG_EARLYOFF) != 0)
 660                 rxfilt |= RL_RXCFG_EARLYOFF;
 661         else if ((sc->rl_flags & RL_FLAG_EARLYOFFV2) != 0)
 662                 rxfilt |= RL_RXCFG_EARLYOFFV2;
 663
 664         if (ifp->if_flags & (IFF_ALLMULTI | IFF_PROMISC)) {
 665                 if (ifp->if_flags & IFF_PROMISC)
 666                         rxfilt |= RL_RXCFG_RX_ALLPHYS;
 667                 /*
 668                  * Unlike other hardwares, we have to explicitly set
 669                  * RL_RXCFG_RX_MULTI to receive multicast frames in
 670                  * promiscuous mode.
 671                  */
 672                 rxfilt |= RL_RXCFG_RX_MULTI;
 673                 hashes[0] = hashes[1] = 0xffffffff;
 674                 goto done;
 675         }
 676
 677         if_maddr_rlock(ifp);
 678         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
 679                 if (ifma->ifma_addr->sa_family != AF_LINK)
 680                         continue;
 681                 h = ether_crc32_be(LLADDR((struct sockaddr_dl *)
 682                     ifma->ifma_addr), ETHER_ADDR_LEN) >> 26;
 683                 if (h < 32)
 684                         hashes[0] |= (1 << h);
 685                 else
 686                         hashes[1] |= (1 << (h - 32));
 687         }
 688         if_maddr_runlock(ifp);
 689
 690         if (hashes[0] != 0 || hashes[1] != 0) {
 691                 /*
 692                  * For some unfathomable reason, RealTek decided to
 693                  * reverse the order of the multicast hash registers
 694                  * in the PCI Express parts.  This means we have to
 695                  * write the hash pattern in reverse order for those
 696                  * devices.
 697                  */
 698                 if ((sc->rl_flags & RL_FLAG_PCIE) != 0) {
 699                         h = bswap32(hashes[0]);
 700                         hashes[0] = bswap32(hashes[1]);
 701                         hashes[1] = h;
 702                 }
 703                 rxfilt |= RL_RXCFG_RX_MULTI;
 704         }
 705
 706         if  (sc->rl_hwrev->rl_rev == RL_HWREV_8168F) {
 707                 /* Disable multicast filtering due to silicon bug. */
 708                 hashes[0] = 0xffffffff;
 709                 hashes[1] = 0xffffffff;
 710         }
 711
 712 done:
 713         CSR_WRITE_4(sc, RL_MAR0, hashes[0]);
 714         CSR_WRITE_4(sc, RL_MAR4, hashes[1]);
 715         CSR_WRITE_4(sc, RL_RXCFG, rxfilt);
 716 }
 717
 718 static void
 719 re_reset(struct rl_softc *sc)
 720 {
 721         int                     i;
 722
 723         RL_LOCK_ASSERT(sc);
 724
 725         CSR_WRITE_1(sc, RL_COMMAND, RL_CMD_RESET);
 726
 727         for (i = 0; i < RL_TIMEOUT; i++) {
 728                 DELAY(10);
 729                 if (!(CSR_READ_1(sc, RL_COMMAND) & RL_CMD_RESET))
 730                         break;
 731         }
 732         if (i == RL_TIMEOUT)
 733                 device_printf(sc->rl_dev, "reset never completed!\n");
 734
 735         if ((sc->rl_flags & RL_FLAG_MACRESET) != 0)
 736                 CSR_WRITE_1(sc, 0x82, 1);
 737         if (sc->rl_hwrev->rl_rev == RL_HWREV_8169S)
 738                 re_gmii_writereg(sc->rl_dev, 1, 0x0b, 0);
 739 }
 740
 741 #ifdef RE_DIAG
 742
 743 /*
 744  * The following routine is designed to test for a defect on some
 745  * 32-bit 8169 cards. Some of these NICs have the REQ64# and ACK64#
 746  * lines connected to the bus, however for a 32-bit only card, they
 747  * should be pulled high. The result of this defect is that the
 748  * NIC will not work right if you plug it into a 64-bit slot: DMA
 749  * operations will be done with 64-bit transfers, which will fail
 750  * because the 64-bit data lines aren't connected.
 751  *
 752  * There's no way to work around this (short of talking a soldering
 753  * iron to the board), however we can detect it. The method we use
 754  * here is to put the NIC into digital loopback mode, set the receiver
 755  * to promiscuous mode, and then try to send a frame. We then compare
 756  * the frame data we sent to what was received. If the data matches,
 757  * then the NIC is working correctly, otherwise we know the user has
 758  * a defective NIC which has been mistakenly plugged into a 64-bit PCI
 759  * slot. In the latter case, there's no way the NIC can work correctly,
 760  * so we print out a message on the console and abort the device attach.
 761  */
 762
 763 static int
 764 re_diag(struct rl_softc *sc)
 765 {
 766         struct ifnet            *ifp = sc->rl_ifp;
 767         struct mbuf             *m0;
 768         struct ether_header     *eh;
 769         struct rl_desc          *cur_rx;
 770         u_int16_t               status;
 771         u_int32_t               rxstat;
 772         int                     total_len, i, error = 0, phyaddr;
 773         u_int8_t                dst[] = { 0x00, 'h', 'e', 'l', 'l', 'o' };
 774         u_int8_t                src[] = { 0x00, 'w', 'o', 'r', 'l', 'd' };
 775
 776         /* Allocate a single mbuf */
 777         MGETHDR(m0, M_NOWAIT, MT_DATA);
 778         if (m0 == NULL)
 779                 return (ENOBUFS);
 780
 781         RL_LOCK(sc);
 782
 783         /*
 784          * Initialize the NIC in test mode. This sets the chip up
 785          * so that it can send and receive frames, but performs the
 786          * following special functions:
 787          * - Puts receiver in promiscuous mode
 788          * - Enables digital loopback mode
 789          * - Leaves interrupts turned off
 790          */
 791
 792         ifp->if_flags |= IFF_PROMISC;
 793         sc->rl_testmode = 1;
 794         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 795         re_init_locked(sc);
 796         sc->rl_flags |= RL_FLAG_LINK;
 797         if (sc->rl_type == RL_8169)
 798                 phyaddr = 1;
 799         else
 800                 phyaddr = 0;
 801
 802         re_miibus_writereg(sc->rl_dev, phyaddr, MII_BMCR, BMCR_RESET);
 803         for (i = 0; i < RL_TIMEOUT; i++) {
 804                 status = re_miibus_readreg(sc->rl_dev, phyaddr, MII_BMCR);
 805                 if (!(status & BMCR_RESET))
 806                         break;
 807         }
 808
 809         re_miibus_writereg(sc->rl_dev, phyaddr, MII_BMCR, BMCR_LOOP);
 810         CSR_WRITE_2(sc, RL_ISR, RL_INTRS);
 811
 812         DELAY(100000);
 813
 814         /* Put some data in the mbuf */
 815
 816         eh = mtod(m0, struct ether_header *);
 817         bcopy ((char *)&dst, eh->ether_dhost, ETHER_ADDR_LEN);
 818         bcopy ((char *)&src, eh->ether_shost, ETHER_ADDR_LEN);
 819         eh->ether_type = htons(ETHERTYPE_IP);
 820         m0->m_pkthdr.len = m0->m_len = ETHER_MIN_LEN - ETHER_CRC_LEN;
 821
 822         /*
 823          * Queue the packet, start transmission.
 824          * Note: IF_HANDOFF() ultimately calls re_start() for us.
 825          */
 826
 827         CSR_WRITE_2(sc, RL_ISR, 0xFFFF);
 828         RL_UNLOCK(sc);
 829         /* XXX: re_diag must not be called when in ALTQ mode */
 830         IF_HANDOFF(&ifp->if_snd, m0, ifp);
 831         RL_LOCK(sc);
 832         m0 = NULL;
 833
 834         /* Wait for it to propagate through the chip */
 835
 836         DELAY(100000);
 837         for (i = 0; i < RL_TIMEOUT; i++) {
 838                 status = CSR_READ_2(sc, RL_ISR);
 839                 CSR_WRITE_2(sc, RL_ISR, status);
 840                 if ((status & (RL_ISR_TIMEOUT_EXPIRED|RL_ISR_RX_OK)) ==
 841                     (RL_ISR_TIMEOUT_EXPIRED|RL_ISR_RX_OK))
 842                         break;
 843                 DELAY(10);
 844         }
 845
 846         if (i == RL_TIMEOUT) {
 847                 device_printf(sc->rl_dev,
 848                     "diagnostic failed, failed to receive packet in"
 849                     " loopback mode\n");
 850                 error = EIO;
 851                 goto done;
 852         }
 853
 854         /*
 855          * The packet should have been dumped into the first
 856          * entry in the RX DMA ring. Grab it from there.
 857          */
 858
 859         bus_dmamap_sync(sc->rl_ldata.rl_rx_list_tag,
 860             sc->rl_ldata.rl_rx_list_map,
 861             BUS_DMASYNC_POSTREAD);
 862         bus_dmamap_sync(sc->rl_ldata.rl_rx_mtag,
 863             sc->rl_ldata.rl_rx_desc[0].rx_dmamap,
 864             BUS_DMASYNC_POSTREAD);
 865         bus_dmamap_unload(sc->rl_ldata.rl_rx_mtag,
 866             sc->rl_ldata.rl_rx_desc[0].rx_dmamap);
 867
 868         m0 = sc->rl_ldata.rl_rx_desc[0].rx_m;
 869         sc->rl_ldata.rl_rx_desc[0].rx_m = NULL;
 870         eh = mtod(m0, struct ether_header *);
 871
 872         cur_rx = &sc->rl_ldata.rl_rx_list[0];
 873         total_len = RL_RXBYTES(cur_rx);
 874         rxstat = le32toh(cur_rx->rl_cmdstat);
 875
 876         if (total_len != ETHER_MIN_LEN) {
 877                 device_printf(sc->rl_dev,
 878                     "diagnostic failed, received short packet\n");
 879                 error = EIO;
 880                 goto done;
 881         }
 882
 883         /* Test that the received packet data matches what we sent. */
 884
 885         if (bcmp((char *)&eh->ether_dhost, (char *)&dst, ETHER_ADDR_LEN) ||
 886             bcmp((char *)&eh->ether_shost, (char *)&src, ETHER_ADDR_LEN) ||
 887             ntohs(eh->ether_type) != ETHERTYPE_IP) {
 888                 device_printf(sc->rl_dev, "WARNING, DMA FAILURE!\n");
 889                 device_printf(sc->rl_dev, "expected TX data: %6D/%6D/0x%x\n",
 890                     dst, ":", src, ":", ETHERTYPE_IP);
 891                 device_printf(sc->rl_dev, "received RX data: %6D/%6D/0x%x\n",
 892                     eh->ether_dhost, ":", eh->ether_shost, ":",
 893                     ntohs(eh->ether_type));
 894                 device_printf(sc->rl_dev, "You may have a defective 32-bit "
 895                     "NIC plugged into a 64-bit PCI slot.\n");
 896                 device_printf(sc->rl_dev, "Please re-install the NIC in a "
 897                     "32-bit slot for proper operation.\n");
 898                 device_printf(sc->rl_dev, "Read the re(4) man page for more "
 899                     "details.\n");
 900                 error = EIO;
 901         }
 902
 903 done:
 904         /* Turn interface off, release resources */
 905
 906         sc->rl_testmode = 0;
 907         sc->rl_flags &= ~RL_FLAG_LINK;
 908         ifp->if_flags &= ~IFF_PROMISC;
 909         re_stop(sc);
 910         if (m0 != NULL)
 911                 m_freem(m0);
 912
 913         RL_UNLOCK(sc);
 914
 915         return (error);
 916 }
 917
 918 #endif
 919
 920 /*
 921  * Probe for a RealTek 8139C+/8169/8110 chip. Check the PCI vendor and device
 922  * IDs against our list and return a device name if we find a match.
 923  */
 924 static int
 925 re_probe(device_t dev)
 926 {
 927         const struct rl_type    *t;
 928         uint16_t                devid, vendor;
 929         uint16_t                revid, sdevid;
 930         int                     i;
 931
 932         vendor = pci_get_vendor(dev);
 933         devid = pci_get_device(dev);
 934         revid = pci_get_revid(dev);
 935         sdevid = pci_get_subdevice(dev);
 936
 937         if (vendor == LINKSYS_VENDORID && devid == LINKSYS_DEVICEID_EG1032) {
 938                 if (sdevid != LINKSYS_SUBDEVICE_EG1032_REV3) {
 939                         /*
 940                          * Only attach to rev. 3 of the Linksys EG1032 adapter.
 941                          * Rev. 2 is supported by sk(4).
 942                          */
 943                         return (ENXIO);
 944                 }
 945         }
 946
 947         if (vendor == RT_VENDORID && devid == RT_DEVICEID_8139) {
 948                 if (revid != 0x20) {
 949                         /* 8139, let rl(4) take care of this device. */
 950                         return (ENXIO);
 951                 }
 952         }
 953
 954         t = re_devs;
 955         for (i = 0; i < sizeof(re_devs) / sizeof(re_devs[0]); i++, t++) {
 956                 if (vendor == t->rl_vid && devid == t->rl_did) {
 957                         device_set_desc(dev, t->rl_name);
 958                         return (BUS_PROBE_DEFAULT);
 959                 }
 960         }
 961
 962         return (ENXIO);
 963 }
 964
 965 /*
 966  * Map a single buffer address.
 967  */
 968
 969 static void
 970 re_dma_map_addr(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 971 {
 972         bus_addr_t              *addr;
 973
 974         if (error)
 975                 return;
 976
 977         KASSERT(nseg == 1, ("too many DMA segments, %d should be 1", nseg));
 978         addr = arg;
 979         *addr = segs->ds_addr;
 980 }
 981
 982 static int
 983 re_allocmem(device_t dev, struct rl_softc *sc)
 984 {
 985         bus_addr_t              lowaddr;
 986         bus_size_t              rx_list_size, tx_list_size;
 987         int                     error;
 988         int                     i;
 989
 990         rx_list_size = sc->rl_ldata.rl_rx_desc_cnt * sizeof(struct rl_desc);
 991         tx_list_size = sc->rl_ldata.rl_tx_desc_cnt * sizeof(struct rl_desc);
 992
 993         /*
 994          * Allocate the parent bus DMA tag appropriate for PCI.
 995          * In order to use DAC, RL_CPLUSCMD_PCI_DAC bit of RL_CPLUS_CMD
 996          * register should be set. However some RealTek chips are known
 997          * to be buggy on DAC handling, therefore disable DAC by limiting
 998          * DMA address space to 32bit. PCIe variants of RealTek chips
 999          * may not have the limitation.
1000          */
1001         lowaddr = BUS_SPACE_MAXADDR;
1002         if ((sc->rl_flags & RL_FLAG_PCIE) == 0)
1003                 lowaddr = BUS_SPACE_MAXADDR_32BIT;
1004         error = bus_dma_tag_create(bus_get_dma_tag(dev), 1, 0,
1005             lowaddr, BUS_SPACE_MAXADDR, NULL, NULL,
1006             BUS_SPACE_MAXSIZE_32BIT, 0, BUS_SPACE_MAXSIZE_32BIT, 0,
1007             NULL, NULL, &sc->rl_parent_tag);
1008         if (error) {
1009                 device_printf(dev, "could not allocate parent DMA tag\n");
1010                 return (error);
1011         }
1012
1013         /*
1014          * Allocate map for TX mbufs.
1015          */
1016         error = bus_dma_tag_create(sc->rl_parent_tag, 1, 0,
1017             BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL,
1018             NULL, MCLBYTES * RL_NTXSEGS, RL_NTXSEGS, 4096, 0,
1019             NULL, NULL, &sc->rl_ldata.rl_tx_mtag);
1020         if (error) {
1021                 device_printf(dev, "could not allocate TX DMA tag\n");
1022                 return (error);
1023         }
1024
1025         /*
1026          * Allocate map for RX mbufs.
1027          */
1028
1029         if ((sc->rl_flags & RL_FLAG_JUMBOV2) != 0) {
1030                 error = bus_dma_tag_create(sc->rl_parent_tag, sizeof(uint64_t),
1031                     0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
1032                     MJUM9BYTES, 1, MJUM9BYTES, 0, NULL, NULL,
1033                     &sc->rl_ldata.rl_jrx_mtag);
1034                 if (error) {
1035                         device_printf(dev,
1036                             "could not allocate jumbo RX DMA tag\n");
1037                         return (error);
1038                 }
1039         }
1040         error = bus_dma_tag_create(sc->rl_parent_tag, sizeof(uint64_t), 0,
1041             BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
1042             MCLBYTES, 1, MCLBYTES, 0, NULL, NULL, &sc->rl_ldata.rl_rx_mtag);
1043         if (error) {
1044                 device_printf(dev, "could not allocate RX DMA tag\n");
1045                 return (error);
1046         }
1047
1048         /*
1049          * Allocate map for TX descriptor list.
1050          */
1051         error = bus_dma_tag_create(sc->rl_parent_tag, RL_RING_ALIGN,
1052             0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL,
1053             NULL, tx_list_size, 1, tx_list_size, 0,
1054             NULL, NULL, &sc->rl_ldata.rl_tx_list_tag);
1055         if (error) {
1056                 device_printf(dev, "could not allocate TX DMA ring tag\n");
1057                 return (error);
1058         }
1059
1060         /* Allocate DMA'able memory for the TX ring */
1061
1062         error = bus_dmamem_alloc(sc->rl_ldata.rl_tx_list_tag,
1063             (void **)&sc->rl_ldata.rl_tx_list,
1064             BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO,
1065             &sc->rl_ldata.rl_tx_list_map);
1066         if (error) {
1067                 device_printf(dev, "could not allocate TX DMA ring\n");
1068                 return (error);
1069         }
1070
1071         /* Load the map for the TX ring. */
1072
1073         sc->rl_ldata.rl_tx_list_addr = 0;
1074         error = bus_dmamap_load(sc->rl_ldata.rl_tx_list_tag,
1075              sc->rl_ldata.rl_tx_list_map, sc->rl_ldata.rl_tx_list,
1076              tx_list_size, re_dma_map_addr,
1077              &sc->rl_ldata.rl_tx_list_addr, BUS_DMA_NOWAIT);
1078         if (error != 0 || sc->rl_ldata.rl_tx_list_addr == 0) {
1079                 device_printf(dev, "could not load TX DMA ring\n");
1080                 return (ENOMEM);
1081         }
1082
1083         /* Create DMA maps for TX buffers */
1084
1085         for (i = 0; i < sc->rl_ldata.rl_tx_desc_cnt; i++) {
1086                 error = bus_dmamap_create(sc->rl_ldata.rl_tx_mtag, 0,
1087                     &sc->rl_ldata.rl_tx_desc[i].tx_dmamap);
1088                 if (error) {
1089                         device_printf(dev, "could not create DMA map for TX\n");
1090                         return (error);
1091                 }
1092         }
1093
1094         /*
1095          * Allocate map for RX descriptor list.
1096          */
1097         error = bus_dma_tag_create(sc->rl_parent_tag, RL_RING_ALIGN,
1098             0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL,
1099             NULL, rx_list_size, 1, rx_list_size, 0,
1100             NULL, NULL, &sc->rl_ldata.rl_rx_list_tag);
1101         if (error) {
1102                 device_printf(dev, "could not create RX DMA ring tag\n");
1103                 return (error);
1104         }
1105
1106         /* Allocate DMA'able memory for the RX ring */
1107
1108         error = bus_dmamem_alloc(sc->rl_ldata.rl_rx_list_tag,
1109             (void **)&sc->rl_ldata.rl_rx_list,
1110             BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO,
1111             &sc->rl_ldata.rl_rx_list_map);
1112         if (error) {
1113                 device_printf(dev, "could not allocate RX DMA ring\n");
1114                 return (error);
1115         }
1116
1117         /* Load the map for the RX ring. */
1118
1119         sc->rl_ldata.rl_rx_list_addr = 0;
1120         error = bus_dmamap_load(sc->rl_ldata.rl_rx_list_tag,
1121              sc->rl_ldata.rl_rx_list_map, sc->rl_ldata.rl_rx_list,
1122              rx_list_size, re_dma_map_addr,
1123              &sc->rl_ldata.rl_rx_list_addr, BUS_DMA_NOWAIT);
1124         if (error != 0 || sc->rl_ldata.rl_rx_list_addr == 0) {
1125                 device_printf(dev, "could not load RX DMA ring\n");
1126                 return (ENOMEM);
1127         }
1128
1129         /* Create DMA maps for RX buffers */
1130
1131         if ((sc->rl_flags & RL_FLAG_JUMBOV2) != 0) {
1132                 error = bus_dmamap_create(sc->rl_ldata.rl_jrx_mtag, 0,
1133                     &sc->rl_ldata.rl_jrx_sparemap);
1134                 if (error) {
1135                         device_printf(dev,
1136                             "could not create spare DMA map for jumbo RX\n");
1137                         return (error);
1138                 }
1139                 for (i = 0; i < sc->rl_ldata.rl_rx_desc_cnt; i++) {
1140                         error = bus_dmamap_create(sc->rl_ldata.rl_jrx_mtag, 0,
1141                             &sc->rl_ldata.rl_jrx_desc[i].rx_dmamap);
1142                         if (error) {
1143                                 device_printf(dev,
1144                                     "could not create DMA map for jumbo RX\n");
1145                                 return (error);
1146                         }
1147                 }
1148         }
1149         error = bus_dmamap_create(sc->rl_ldata.rl_rx_mtag, 0,
1150             &sc->rl_ldata.rl_rx_sparemap);
1151         if (error) {
1152                 device_printf(dev, "could not create spare DMA map for RX\n");
1153                 return (error);
1154         }
1155         for (i = 0; i < sc->rl_ldata.rl_rx_desc_cnt; i++) {
1156                 error = bus_dmamap_create(sc->rl_ldata.rl_rx_mtag, 0,
1157                     &sc->rl_ldata.rl_rx_desc[i].rx_dmamap);
1158                 if (error) {
1159                         device_printf(dev, "could not create DMA map for RX\n");
1160                         return (error);
1161                 }
1162         }
1163
1164         /* Create DMA map for statistics. */
1165         error = bus_dma_tag_create(sc->rl_parent_tag, RL_DUMP_ALIGN, 0,
1166             BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
1167             sizeof(struct rl_stats), 1, sizeof(struct rl_stats), 0, NULL, NULL,
1168             &sc->rl_ldata.rl_stag);
1169         if (error) {
1170                 device_printf(dev, "could not create statistics DMA tag\n");
1171                 return (error);
1172         }
1173         /* Allocate DMA'able memory for statistics. */
1174         error = bus_dmamem_alloc(sc->rl_ldata.rl_stag,
1175             (void **)&sc->rl_ldata.rl_stats,
1176             BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO,
1177             &sc->rl_ldata.rl_smap);
1178         if (error) {
1179                 device_printf(dev,
1180                     "could not allocate statistics DMA memory\n");
1181                 return (error);
1182         }
1183         /* Load the map for statistics. */
1184         sc->rl_ldata.rl_stats_addr = 0;
1185         error = bus_dmamap_load(sc->rl_ldata.rl_stag, sc->rl_ldata.rl_smap,
1186             sc->rl_ldata.rl_stats, sizeof(struct rl_stats), re_dma_map_addr,
1187              &sc->rl_ldata.rl_stats_addr, BUS_DMA_NOWAIT);
1188         if (error != 0 || sc->rl_ldata.rl_stats_addr == 0) {
1189                 device_printf(dev, "could not load statistics DMA memory\n");
1190                 return (ENOMEM);
1191         }
1192
1193         return (0);
1194 }
1195
1196 /*
1197  * Attach the interface. Allocate softc structures, do ifmedia
1198  * setup and ethernet/BPF attach.
1199  */
1200 static int
1201 re_attach(device_t dev)
1202 {
1203         u_char                  eaddr[ETHER_ADDR_LEN];
1204         u_int16_t               as[ETHER_ADDR_LEN / 2];
1205         struct rl_softc         *sc;
1206         struct ifnet            *ifp;
1207         const struct rl_hwrev   *hw_rev;
1208         u_int32_t               cap, ctl;
1209         int                     hwrev;
1210         u_int16_t               devid, re_did = 0;
1211         int                     error = 0, i, phy, rid;
1212         int                     msic, msixc, reg;
1213         uint8_t                 cfg;
1214
1215         sc = device_get_softc(dev);
1216         sc->rl_dev = dev;
1217
1218         mtx_init(&sc->rl_mtx, device_get_nameunit(dev), MTX_NETWORK_LOCK,
1219             MTX_DEF);
1220         callout_init_mtx(&sc->rl_stat_callout, &sc->rl_mtx, 0);
1221
1222         /*
1223          * Map control/status registers.
1224          */
1225         pci_enable_busmaster(dev);
1226
1227         devid = pci_get_device(dev);
1228         /*
1229          * Prefer memory space register mapping over IO space.
1230          * Because RTL8169SC does not seem to work when memory mapping
1231          * is used always activate io mapping.
1232          */
1233         if (devid == RT_DEVICEID_8169SC)
1234                 prefer_iomap = 1;
1235         if (prefer_iomap == 0) {
1236                 sc->rl_res_id = PCIR_BAR(1);
1237                 sc->rl_res_type = SYS_RES_MEMORY;
1238                 /* RTL8168/8101E seems to use different BARs. */
1239                 if (devid == RT_DEVICEID_8168 || devid == RT_DEVICEID_8101E)
1240                         sc->rl_res_id = PCIR_BAR(2);
1241         } else {
1242                 sc->rl_res_id = PCIR_BAR(0);
1243                 sc->rl_res_type = SYS_RES_IOPORT;
1244         }
1245         sc->rl_res = bus_alloc_resource_any(dev, sc->rl_res_type,
1246             &sc->rl_res_id, RF_ACTIVE);
1247         if (sc->rl_res == NULL && prefer_iomap == 0) {
1248                 sc->rl_res_id = PCIR_BAR(0);
1249                 sc->rl_res_type = SYS_RES_IOPORT;
1250                 sc->rl_res = bus_alloc_resource_any(dev, sc->rl_res_type,
1251                     &sc->rl_res_id, RF_ACTIVE);
1252         }
1253         if (sc->rl_res == NULL) {
1254                 device_printf(dev, "couldn't map ports/memory\n");
1255                 error = ENXIO;
1256                 goto fail;
1257         }
1258
1259         sc->rl_btag = rman_get_bustag(sc->rl_res);
1260         sc->rl_bhandle = rman_get_bushandle(sc->rl_res);
1261
1262         msic = pci_msi_count(dev);
1263         msixc = pci_msix_count(dev);
1264         if (pci_find_cap(dev, PCIY_EXPRESS, &reg) == 0) {
1265                 sc->rl_flags |= RL_FLAG_PCIE;
1266                 sc->rl_expcap = reg;
1267         }
1268         if (bootverbose) {
1269                 device_printf(dev, "MSI count : %d\n", msic);
1270                 device_printf(dev, "MSI-X count : %d\n", msixc);
1271         }
1272         if (msix_disable > 0)
1273                 msixc = 0;
1274         if (msi_disable > 0)
1275                 msic = 0;
1276         /* Prefer MSI-X to MSI. */
1277         if (msixc > 0) {
1278                 msixc = RL_MSI_MESSAGES;
1279                 rid = PCIR_BAR(4);
1280                 sc->rl_res_pba = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
1281                     &rid, RF_ACTIVE);
1282                 if (sc->rl_res_pba == NULL) {
1283                         device_printf(sc->rl_dev,
1284                             "could not allocate MSI-X PBA resource\n");
1285                 }
1286                 if (sc->rl_res_pba != NULL &&
1287                     pci_alloc_msix(dev, &msixc) == 0) {
1288                         if (msixc == RL_MSI_MESSAGES) {
1289                                 device_printf(dev, "Using %d MSI-X message\n",
1290                                     msixc);
1291                                 sc->rl_flags |= RL_FLAG_MSIX;
1292                         } else
1293                                 pci_release_msi(dev);
1294                 }
1295                 if ((sc->rl_flags & RL_FLAG_MSIX) == 0) {
1296                         if (sc->rl_res_pba != NULL)
1297                                 bus_release_resource(dev, SYS_RES_MEMORY, rid,
1298                                     sc->rl_res_pba);
1299                         sc->rl_res_pba = NULL;
1300                         msixc = 0;
1301                 }
1302         }
1303         /* Prefer MSI to INTx. */
1304         if (msixc == 0 && msic > 0) {
1305                 msic = RL_MSI_MESSAGES;
1306                 if (pci_alloc_msi(dev, &msic) == 0) {
1307                         if (msic == RL_MSI_MESSAGES) {
1308                                 device_printf(dev, "Using %d MSI message\n",
1309                                     msic);
1310                                 sc->rl_flags |= RL_FLAG_MSI;
1311                                 /* Explicitly set MSI enable bit. */
1312                                 CSR_WRITE_1(sc, RL_EECMD, RL_EE_MODE);
1313                                 cfg = CSR_READ_1(sc, RL_CFG2);
1314                                 cfg |= RL_CFG2_MSI;
1315                                 CSR_WRITE_1(sc, RL_CFG2, cfg);
1316                                 CSR_WRITE_1(sc, RL_EECMD, RL_EEMODE_OFF);
1317                         } else
1318                                 pci_release_msi(dev);
1319                 }
1320                 if ((sc->rl_flags & RL_FLAG_MSI) == 0)
1321                         msic = 0;
1322         }
1323
1324         /* Allocate interrupt */
1325         if ((sc->rl_flags & (RL_FLAG_MSI | RL_FLAG_MSIX)) == 0) {
1326                 rid = 0;
1327                 sc->rl_irq[0] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
1328                     RF_SHAREABLE | RF_ACTIVE);
1329                 if (sc->rl_irq[0] == NULL) {
1330                         device_printf(dev, "couldn't allocate IRQ resources\n");
1331                         error = ENXIO;
1332                         goto fail;
1333                 }
1334         } else {
1335                 for (i = 0, rid = 1; i < RL_MSI_MESSAGES; i++, rid++) {
1336                         sc->rl_irq[i] = bus_alloc_resource_any(dev,
1337                             SYS_RES_IRQ, &rid, RF_ACTIVE);
1338                         if (sc->rl_irq[i] == NULL) {
1339                                 device_printf(dev,
1340                                     "couldn't allocate IRQ resources for "
1341                                     "message %d\n", rid);
1342                                 error = ENXIO;
1343                                 goto fail;
1344                         }
1345                 }
1346         }
1347
1348         if ((sc->rl_flags & RL_FLAG_MSI) == 0) {
1349                 CSR_WRITE_1(sc, RL_EECMD, RL_EE_MODE);
1350                 cfg = CSR_READ_1(sc, RL_CFG2);
1351                 if ((cfg & RL_CFG2_MSI) != 0) {
1352                         device_printf(dev, "turning off MSI enable bit.\n");
1353                         cfg &= ~RL_CFG2_MSI;
1354                         CSR_WRITE_1(sc, RL_CFG2, cfg);
1355                 }
1356                 CSR_WRITE_1(sc, RL_EECMD, RL_EEMODE_OFF);
1357         }
1358
1359         /* Disable ASPM L0S/L1. */
1360         if (sc->rl_expcap != 0) {
1361                 cap = pci_read_config(dev, sc->rl_expcap +
1362                     PCIER_LINK_CAP, 2);
1363                 if ((cap & PCIEM_LINK_CAP_ASPM) != 0) {
1364                         ctl = pci_read_config(dev, sc->rl_expcap +
1365                             PCIER_LINK_CTL, 2);
1366                         if ((ctl & PCIEM_LINK_CTL_ASPMC) != 0) {
1367                                 ctl &= ~PCIEM_LINK_CTL_ASPMC;
1368                                 pci_write_config(dev, sc->rl_expcap +
1369                                     PCIER_LINK_CTL, ctl, 2);
1370                                 device_printf(dev, "ASPM disabled\n");
1371                         }
1372                 } else
1373                         device_printf(dev, "no ASPM capability\n");
1374         }
1375
1376         hw_rev = re_hwrevs;
1377         hwrev = CSR_READ_4(sc, RL_TXCFG);
1378         switch (hwrev & 0x70000000) {
1379         case 0x00000000:
1380         case 0x10000000:
1381                 device_printf(dev, "Chip rev. 0x%08x\n", hwrev & 0xfc800000);
1382                 hwrev &= (RL_TXCFG_HWREV | 0x80000000);
1383                 break;
1384         default:
1385                 device_printf(dev, "Chip rev. 0x%08x\n", hwrev & 0x7c800000);
1386                 sc->rl_macrev = hwrev & 0x00700000;
1387                 hwrev &= RL_TXCFG_HWREV;
1388                 break;
1389         }
1390         device_printf(dev, "MAC rev. 0x%08x\n", sc->rl_macrev);
1391         while (hw_rev->rl_desc != NULL) {
1392                 if (hw_rev->rl_rev == hwrev) {
1393                         sc->rl_type = hw_rev->rl_type;
1394                         sc->rl_hwrev = hw_rev;
1395                         break;
1396                 }
1397                 hw_rev++;
1398         }
1399         if (hw_rev->rl_desc == NULL) {
1400                 device_printf(dev, "Unknown H/W revision: 0x%08x\n", hwrev);
1401                 error = ENXIO;
1402                 goto fail;
1403         }
1404
1405         switch (hw_rev->rl_rev) {
1406         case RL_HWREV_8139CPLUS:
1407                 sc->rl_flags |= RL_FLAG_FASTETHER | RL_FLAG_AUTOPAD;
1408                 break;
1409         case RL_HWREV_8100E:
1410         case RL_HWREV_8101E:
1411                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_FASTETHER;
1412                 break;
1413         case RL_HWREV_8102E:
1414         case RL_HWREV_8102EL:
1415         case RL_HWREV_8102EL_SPIN1:
1416                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PAR | RL_FLAG_DESCV2 |
1417                     RL_FLAG_MACSTAT | RL_FLAG_FASTETHER | RL_FLAG_CMDSTOP |
1418                     RL_FLAG_AUTOPAD;
1419                 break;
1420         case RL_HWREV_8103E:
1421                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PAR | RL_FLAG_DESCV2 |
1422                     RL_FLAG_MACSTAT | RL_FLAG_FASTETHER | RL_FLAG_CMDSTOP |
1423                     RL_FLAG_AUTOPAD | RL_FLAG_MACSLEEP;
1424                 break;
1425         case RL_HWREV_8401E:
1426         case RL_HWREV_8105E:
1427         case RL_HWREV_8105E_SPIN1:
1428         case RL_HWREV_8106E:
1429                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PHYWAKE_PM |
1430                     RL_FLAG_PAR | RL_FLAG_DESCV2 | RL_FLAG_MACSTAT |
1431                     RL_FLAG_FASTETHER | RL_FLAG_CMDSTOP | RL_FLAG_AUTOPAD;
1432                 break;
1433         case RL_HWREV_8402:
1434                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PHYWAKE_PM |
1435                     RL_FLAG_PAR | RL_FLAG_DESCV2 | RL_FLAG_MACSTAT |
1436                     RL_FLAG_FASTETHER | RL_FLAG_CMDSTOP | RL_FLAG_AUTOPAD |
1437                     RL_FLAG_CMDSTOP_WAIT_TXQ;
1438                 break;
1439         case RL_HWREV_8168B_SPIN1:
1440         case RL_HWREV_8168B_SPIN2:
1441                 sc->rl_flags |= RL_FLAG_WOLRXENB;
1442                 /* FALLTHROUGH */
1443         case RL_HWREV_8168B_SPIN3:
1444                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_MACSTAT;
1445                 break;
1446         case RL_HWREV_8168C_SPIN2:
1447                 sc->rl_flags |= RL_FLAG_MACSLEEP;
1448                 /* FALLTHROUGH */
1449         case RL_HWREV_8168C:
1450                 if (sc->rl_macrev == 0x00200000)
1451                         sc->rl_flags |= RL_FLAG_MACSLEEP;
1452                 /* FALLTHROUGH */
1453         case RL_HWREV_8168CP:
1454                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PAR |
1455                     RL_FLAG_DESCV2 | RL_FLAG_MACSTAT | RL_FLAG_CMDSTOP |
1456                     RL_FLAG_AUTOPAD | RL_FLAG_JUMBOV2 | RL_FLAG_WOL_MANLINK;
1457                 break;
1458         case RL_HWREV_8168D:
1459                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PHYWAKE_PM |
1460                     RL_FLAG_PAR | RL_FLAG_DESCV2 | RL_FLAG_MACSTAT |
1461                     RL_FLAG_CMDSTOP | RL_FLAG_AUTOPAD | RL_FLAG_JUMBOV2 |
1462                     RL_FLAG_WOL_MANLINK;
1463                 break;
1464         case RL_HWREV_8168DP:
1465                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PAR |
1466                     RL_FLAG_DESCV2 | RL_FLAG_MACSTAT | RL_FLAG_AUTOPAD |
1467                     RL_FLAG_JUMBOV2 | RL_FLAG_WAIT_TXPOLL | RL_FLAG_WOL_MANLINK;
1468                 break;
1469         case RL_HWREV_8168E:
1470                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PHYWAKE_PM |
1471                     RL_FLAG_PAR | RL_FLAG_DESCV2 | RL_FLAG_MACSTAT |
1472                     RL_FLAG_CMDSTOP | RL_FLAG_AUTOPAD | RL_FLAG_JUMBOV2 |
1473                     RL_FLAG_WOL_MANLINK;
1474                 break;
1475         case RL_HWREV_8168E_VL:
1476         case RL_HWREV_8168F:
1477                 sc->rl_flags |= RL_FLAG_EARLYOFF;
1478                 /* FALLTHROUGH */
1479         case RL_HWREV_8411:
1480                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PAR |
1481                     RL_FLAG_DESCV2 | RL_FLAG_MACSTAT | RL_FLAG_CMDSTOP |
1482                     RL_FLAG_AUTOPAD | RL_FLAG_JUMBOV2 |
1483                     RL_FLAG_CMDSTOP_WAIT_TXQ | RL_FLAG_WOL_MANLINK;
1484                 break;
1485         case RL_HWREV_8168EP:
1486         case RL_HWREV_8168G:
1487         case RL_HWREV_8411B:
1488                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PAR |
1489                     RL_FLAG_DESCV2 | RL_FLAG_MACSTAT | RL_FLAG_CMDSTOP |
1490                     RL_FLAG_AUTOPAD | RL_FLAG_JUMBOV2 |
1491                     RL_FLAG_CMDSTOP_WAIT_TXQ | RL_FLAG_WOL_MANLINK |
1492                     RL_FLAG_EARLYOFFV2 | RL_FLAG_RXDV_GATED;
1493                 break;
1494         case RL_HWREV_8168GU:
1495                 if (pci_get_device(dev) == RT_DEVICEID_8101E) {
1496                         /* RTL8106EUS */
1497                         sc->rl_flags |= RL_FLAG_FASTETHER;
1498                 } else
1499                         sc->rl_flags |= RL_FLAG_JUMBOV2 | RL_FLAG_WOL_MANLINK;
1500
1501                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PAR |
1502                     RL_FLAG_DESCV2 | RL_FLAG_MACSTAT | RL_FLAG_CMDSTOP |
1503                     RL_FLAG_AUTOPAD | RL_FLAG_CMDSTOP_WAIT_TXQ |
1504                     RL_FLAG_EARLYOFFV2 | RL_FLAG_RXDV_GATED;
1505                 break;
1506         case RL_HWREV_8169_8110SB:
1507         case RL_HWREV_8169_8110SBL:
1508         case RL_HWREV_8169_8110SC:
1509         case RL_HWREV_8169_8110SCE:
1510                 sc->rl_flags |= RL_FLAG_PHYWAKE;
1511                 /* FALLTHROUGH */
1512         case RL_HWREV_8169:
1513         case RL_HWREV_8169S:
1514         case RL_HWREV_8110S:
1515                 sc->rl_flags |= RL_FLAG_MACRESET;
1516                 break;
1517         default:
1518                 break;
1519         }
1520
1521         if (sc->rl_hwrev->rl_rev == RL_HWREV_8139CPLUS) {
1522                 sc->rl_cfg0 = RL_8139_CFG0;
1523                 sc->rl_cfg1 = RL_8139_CFG1;
1524                 sc->rl_cfg2 = 0;
1525                 sc->rl_cfg3 = RL_8139_CFG3;
1526                 sc->rl_cfg4 = RL_8139_CFG4;
1527                 sc->rl_cfg5 = RL_8139_CFG5;
1528         } else {
1529                 sc->rl_cfg0 = RL_CFG0;
1530                 sc->rl_cfg1 = RL_CFG1;
1531                 sc->rl_cfg2 = RL_CFG2;
1532                 sc->rl_cfg3 = RL_CFG3;
1533                 sc->rl_cfg4 = RL_CFG4;
1534                 sc->rl_cfg5 = RL_CFG5;
1535         }
1536
1537         /* Reset the adapter. */
1538         RL_LOCK(sc);
1539         re_reset(sc);
1540         RL_UNLOCK(sc);
1541
1542         /* Enable PME. */
1543         CSR_WRITE_1(sc, RL_EECMD, RL_EE_MODE);
1544         cfg = CSR_READ_1(sc, sc->rl_cfg1);
1545         cfg |= RL_CFG1_PME;
1546         CSR_WRITE_1(sc, sc->rl_cfg1, cfg);
1547         cfg = CSR_READ_1(sc, sc->rl_cfg5);
1548         cfg &= RL_CFG5_PME_STS;
1549         CSR_WRITE_1(sc, sc->rl_cfg5, cfg);
1550         CSR_WRITE_1(sc, RL_EECMD, RL_EEMODE_OFF);
1551
1552         if ((sc->rl_flags & RL_FLAG_PAR) != 0) {
1553                 /*
1554                  * XXX Should have a better way to extract station
1555                  * address from EEPROM.
1556                  */
1557                 for (i = 0; i < ETHER_ADDR_LEN; i++)
1558                         eaddr[i] = CSR_READ_1(sc, RL_IDR0 + i);
1559         } else {
1560                 sc->rl_eewidth = RL_9356_ADDR_LEN;
1561                 re_read_eeprom(sc, (caddr_t)&re_did, 0, 1);
1562                 if (re_did != 0x8129)
1563                         sc->rl_eewidth = RL_9346_ADDR_LEN;
1564
1565                 /*
1566                  * Get station address from the EEPROM.
1567                  */
1568                 re_read_eeprom(sc, (caddr_t)as, RL_EE_EADDR, 3);
1569                 for (i = 0; i < ETHER_ADDR_LEN / 2; i++)
1570                         as[i] = le16toh(as[i]);
1571                 bcopy(as, eaddr, ETHER_ADDR_LEN);
1572         }
1573
1574         if (sc->rl_type == RL_8169) {
1575                 /* Set RX length mask and number of descriptors. */
1576                 sc->rl_rxlenmask = RL_RDESC_STAT_GFRAGLEN;
1577                 sc->rl_txstart = RL_GTXSTART;
1578                 sc->rl_ldata.rl_tx_desc_cnt = RL_8169_TX_DESC_CNT;
1579                 sc->rl_ldata.rl_rx_desc_cnt = RL_8169_RX_DESC_CNT;
1580         } else {
1581                 /* Set RX length mask and number of descriptors. */
1582                 sc->rl_rxlenmask = RL_RDESC_STAT_FRAGLEN;
1583                 sc->rl_txstart = RL_TXSTART;
1584                 sc->rl_ldata.rl_tx_desc_cnt = RL_8139_TX_DESC_CNT;
1585                 sc->rl_ldata.rl_rx_desc_cnt = RL_8139_RX_DESC_CNT;
1586         }
1587
1588         error = re_allocmem(dev, sc);
1589         if (error)
1590                 goto fail;
1591         re_add_sysctls(sc);
1592
1593         ifp = sc->rl_ifp = if_alloc(IFT_ETHER);
1594         if (ifp == NULL) {
1595                 device_printf(dev, "can not if_alloc()\n");
1596                 error = ENOSPC;
1597                 goto fail;
1598         }
1599
1600         /* Take controller out of deep sleep mode. */
1601         if ((sc->rl_flags & RL_FLAG_MACSLEEP) != 0) {
1602                 if ((CSR_READ_1(sc, RL_MACDBG) & 0x80) == 0x80)
1603                         CSR_WRITE_1(sc, RL_GPIO,
1604                             CSR_READ_1(sc, RL_GPIO) | 0x01);
1605                 else
1606                         CSR_WRITE_1(sc, RL_GPIO,
1607                             CSR_READ_1(sc, RL_GPIO) & ~0x01);
1608         }
1609
1610         /* Take PHY out of power down mode. */
1611         if ((sc->rl_flags & RL_FLAG_PHYWAKE_PM) != 0) {
1612                 CSR_WRITE_1(sc, RL_PMCH, CSR_READ_1(sc, RL_PMCH) | 0x80);
1613                 if (hw_rev->rl_rev == RL_HWREV_8401E)
1614                         CSR_WRITE_1(sc, 0xD1, CSR_READ_1(sc, 0xD1) & ~0x08);
1615         }
1616         if ((sc->rl_flags & RL_FLAG_PHYWAKE) != 0) {
1617                 re_gmii_writereg(dev, 1, 0x1f, 0);
1618                 re_gmii_writereg(dev, 1, 0x0e, 0);
1619         }
1620
1621         ifp->if_softc = sc;
1622         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1623         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1624         ifp->if_ioctl = re_ioctl;
1625         ifp->if_start = re_start;
1626         /*
1627          * RTL8168/8111C generates wrong IP checksummed frame if the
1628          * packet has IP options so disable TX checksum offloading.
1629          */
1630         if (sc->rl_hwrev->rl_rev == RL_HWREV_8168C ||
1631             sc->rl_hwrev->rl_rev == RL_HWREV_8168C_SPIN2 ||
1632             sc->rl_hwrev->rl_rev == RL_HWREV_8168CP) {
1633                 ifp->if_hwassist = 0;
1634                 ifp->if_capabilities = IFCAP_RXCSUM | IFCAP_TSO4;
1635         } else {
1636                 ifp->if_hwassist = CSUM_IP | CSUM_TCP | CSUM_UDP;
1637                 ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_TSO4;
1638         }
1639         ifp->if_hwassist |= CSUM_TSO;
1640         ifp->if_capenable = ifp->if_capabilities;
1641         ifp->if_init = re_init;
1642         IFQ_SET_MAXLEN(&ifp->if_snd, RL_IFQ_MAXLEN);
1643         ifp->if_snd.ifq_drv_maxlen = RL_IFQ_MAXLEN;
1644         IFQ_SET_READY(&ifp->if_snd);
1645
1646         TASK_INIT(&sc->rl_inttask, 0, re_int_task, sc);
1647
1648 #define RE_PHYAD_INTERNAL        0
1649
1650         /* Do MII setup. */
1651         phy = RE_PHYAD_INTERNAL;
1652         if (sc->rl_type == RL_8169)
1653                 phy = 1;
1654         error = mii_attach(dev, &sc->rl_miibus, ifp, re_ifmedia_upd,
1655             re_ifmedia_sts, BMSR_DEFCAPMASK, phy, MII_OFFSET_ANY, MIIF_DOPAUSE);
1656         if (error != 0) {
1657                 device_printf(dev, "attaching PHYs failed\n");
1658                 goto fail;
1659         }
1660
1661         /*
1662          * Call MI attach routine.
1663          */
1664         ether_ifattach(ifp, eaddr);
1665
1666         /* VLAN capability setup */
1667         ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING;
1668         if (ifp->if_capabilities & IFCAP_HWCSUM)
1669                 ifp->if_capabilities |= IFCAP_VLAN_HWCSUM;
1670         /* Enable WOL if PM is supported. */
1671         if (pci_find_cap(sc->rl_dev, PCIY_PMG, &reg) == 0)
1672                 ifp->if_capabilities |= IFCAP_WOL;
1673         ifp->if_capenable = ifp->if_capabilities;
1674         ifp->if_capenable &= ~(IFCAP_WOL_UCAST | IFCAP_WOL_MCAST);
1675         /*
1676          * Don't enable TSO by default.  It is known to generate
1677          * corrupted TCP segments(bad TCP options) under certain
1678          * circumstances.
1679          */
1680         ifp->if_hwassist &= ~CSUM_TSO;
1681         ifp->if_capenable &= ~(IFCAP_TSO4 | IFCAP_VLAN_HWTSO);
1682 #ifdef DEVICE_POLLING
1683         ifp->if_capabilities |= IFCAP_POLLING;
1684 #endif
1685         /*
1686          * Tell the upper layer(s) we support long frames.
1687          * Must appear after the call to ether_ifattach() because
1688          * ether_ifattach() sets ifi_hdrlen to the default value.
1689          */
1690         ifp->if_hdrlen = sizeof(struct ether_vlan_header);
1691
1692 #ifdef DEV_NETMAP
1693         re_netmap_attach(sc);
1694 #endif /* DEV_NETMAP */
1695 #ifdef RE_DIAG
1696         /*
1697          * Perform hardware diagnostic on the original RTL8169.
1698          * Some 32-bit cards were incorrectly wired and would
1699          * malfunction if plugged into a 64-bit slot.
1700          */
1701
1702         if (hwrev == RL_HWREV_8169) {
1703                 error = re_diag(sc);
1704                 if (error) {
1705                         device_printf(dev,
1706                         "attach aborted due to hardware diag failure\n");
1707                         ether_ifdetach(ifp);
1708                         goto fail;
1709                 }
1710         }
1711 #endif
1712
1713 #ifdef RE_TX_MODERATION
1714         intr_filter = 1;
1715 #endif
1716         /* Hook interrupt last to avoid having to lock softc */
1717         if ((sc->rl_flags & (RL_FLAG_MSI | RL_FLAG_MSIX)) != 0 &&
1718             intr_filter == 0) {
1719                 error = bus_setup_intr(dev, sc->rl_irq[0],
1720                     INTR_TYPE_NET | INTR_MPSAFE, NULL, re_intr_msi, sc,
1721                     &sc->rl_intrhand[0]);
1722         } else {
1723                 error = bus_setup_intr(dev, sc->rl_irq[0],
1724                     INTR_TYPE_NET | INTR_MPSAFE, re_intr, NULL, sc,
1725                     &sc->rl_intrhand[0]);
1726         }
1727         if (error) {
1728                 device_printf(dev, "couldn't set up irq\n");
1729                 ether_ifdetach(ifp);
1730         }
1731
1732 fail:
1733
1734         if (error)
1735                 re_detach(dev);
1736
1737         return (error);
1738 }
1739
1740 /*
1741  * Shutdown hardware and free up resources. This can be called any
1742  * time after the mutex has been initialized. It is called in both
1743  * the error case in attach and the normal detach case so it needs
1744  * to be careful about only freeing resources that have actually been
1745  * allocated.
1746  */
1747 static int
1748 re_detach(device_t dev)
1749 {
1750         struct rl_softc         *sc;
1751         struct ifnet            *ifp;
1752         int                     i, rid;
1753
1754         sc = device_get_softc(dev);
1755         ifp = sc->rl_ifp;
1756         KASSERT(mtx_initialized(&sc->rl_mtx), ("re mutex not initialized"));
1757
1758         /* These should only be active if attach succeeded */
1759         if (device_is_attached(dev)) {
1760 #ifdef DEVICE_POLLING
1761                 if (ifp->if_capenable & IFCAP_POLLING)
1762                         ether_poll_deregister(ifp);
1763 #endif
1764                 RL_LOCK(sc);
1765 #if 0
1766                 sc->suspended = 1;
1767 #endif
1768                 re_stop(sc);
1769                 RL_UNLOCK(sc);
1770                 callout_drain(&sc->rl_stat_callout);
1771                 taskqueue_drain(taskqueue_fast, &sc->rl_inttask);
1772                 /*
1773                  * Force off the IFF_UP flag here, in case someone
1774                  * still had a BPF descriptor attached to this
1775                  * interface. If they do, ether_ifdetach() will cause
1776                  * the BPF code to try and clear the promisc mode
1777                  * flag, which will bubble down to re_ioctl(),
1778                  * which will try to call re_init() again. This will
1779                  * turn the NIC back on and restart the MII ticker,
1780                  * which will panic the system when the kernel tries
1781                  * to invoke the re_tick() function that isn't there
1782                  * anymore.
1783                  */
1784                 ifp->if_flags &= ~IFF_UP;
1785                 ether_ifdetach(ifp);
1786         }
1787         if (sc->rl_miibus)
1788                 device_delete_child(dev, sc->rl_miibus);
1789         bus_generic_detach(dev);
1790
1791         /*
1792          * The rest is resource deallocation, so we should already be
1793          * stopped here.
1794          */
1795
1796         if (sc->rl_intrhand[0] != NULL) {
1797                 bus_teardown_intr(dev, sc->rl_irq[0], sc->rl_intrhand[0]);
1798                 sc->rl_intrhand[0] = NULL;
1799         }
1800         if (ifp != NULL) {
1801 #ifdef DEV_NETMAP
1802                 netmap_detach(ifp);
1803 #endif /* DEV_NETMAP */
1804                 if_free(ifp);
1805         }
1806         if ((sc->rl_flags & (RL_FLAG_MSI | RL_FLAG_MSIX)) == 0)
1807                 rid = 0;
1808         else
1809                 rid = 1;
1810         if (sc->rl_irq[0] != NULL) {
1811                 bus_release_resource(dev, SYS_RES_IRQ, rid, sc->rl_irq[0]);
1812                 sc->rl_irq[0] = NULL;
1813         }
1814         if ((sc->rl_flags & (RL_FLAG_MSI | RL_FLAG_MSIX)) != 0)
1815                 pci_release_msi(dev);
1816         if (sc->rl_res_pba) {
1817                 rid = PCIR_BAR(4);
1818                 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->rl_res_pba);
1819         }
1820         if (sc->rl_res)
1821                 bus_release_resource(dev, sc->rl_res_type, sc->rl_res_id,
1822                     sc->rl_res);
1823
1824         /* Unload and free the RX DMA ring memory and map */
1825
1826         if (sc->rl_ldata.rl_rx_list_tag) {
1827                 if (sc->rl_ldata.rl_rx_list_addr)
1828                         bus_dmamap_unload(sc->rl_ldata.rl_rx_list_tag,
1829                             sc->rl_ldata.rl_rx_list_map);
1830                 if (sc->rl_ldata.rl_rx_list)
1831                         bus_dmamem_free(sc->rl_ldata.rl_rx_list_tag,
1832                             sc->rl_ldata.rl_rx_list,
1833                             sc->rl_ldata.rl_rx_list_map);
1834                 bus_dma_tag_destroy(sc->rl_ldata.rl_rx_list_tag);
1835         }
1836
1837         /* Unload and free the TX DMA ring memory and map */
1838
1839         if (sc->rl_ldata.rl_tx_list_tag) {
1840                 if (sc->rl_ldata.rl_tx_list_addr)
1841                         bus_dmamap_unload(sc->rl_ldata.rl_tx_list_tag,
1842                             sc->rl_ldata.rl_tx_list_map);
1843                 if (sc->rl_ldata.rl_tx_list)
1844                         bus_dmamem_free(sc->rl_ldata.rl_tx_list_tag,
1845                             sc->rl_ldata.rl_tx_list,
1846                             sc->rl_ldata.rl_tx_list_map);
1847                 bus_dma_tag_destroy(sc->rl_ldata.rl_tx_list_tag);
1848         }
1849
1850         /* Destroy all the RX and TX buffer maps */
1851
1852         if (sc->rl_ldata.rl_tx_mtag) {
1853                 for (i = 0; i < sc->rl_ldata.rl_tx_desc_cnt; i++) {
1854                         if (sc->rl_ldata.rl_tx_desc[i].tx_dmamap)
1855                                 bus_dmamap_destroy(sc->rl_ldata.rl_tx_mtag,
1856                                     sc->rl_ldata.rl_tx_desc[i].tx_dmamap);
1857                 }
1858                 bus_dma_tag_destroy(sc->rl_ldata.rl_tx_mtag);
1859         }
1860         if (sc->rl_ldata.rl_rx_mtag) {
1861                 for (i = 0; i < sc->rl_ldata.rl_rx_desc_cnt; i++) {
1862                         if (sc->rl_ldata.rl_rx_desc[i].rx_dmamap)
1863                                 bus_dmamap_destroy(sc->rl_ldata.rl_rx_mtag,
1864                                     sc->rl_ldata.rl_rx_desc[i].rx_dmamap);
1865                 }
1866                 if (sc->rl_ldata.rl_rx_sparemap)
1867                         bus_dmamap_destroy(sc->rl_ldata.rl_rx_mtag,
1868                             sc->rl_ldata.rl_rx_sparemap);
1869                 bus_dma_tag_destroy(sc->rl_ldata.rl_rx_mtag);
1870         }
1871         if (sc->rl_ldata.rl_jrx_mtag) {
1872                 for (i = 0; i < sc->rl_ldata.rl_rx_desc_cnt; i++) {
1873                         if (sc->rl_ldata.rl_jrx_desc[i].rx_dmamap)
1874                                 bus_dmamap_destroy(sc->rl_ldata.rl_jrx_mtag,
1875                                     sc->rl_ldata.rl_jrx_desc[i].rx_dmamap);
1876                 }
1877                 if (sc->rl_ldata.rl_jrx_sparemap)
1878                         bus_dmamap_destroy(sc->rl_ldata.rl_jrx_mtag,
1879                             sc->rl_ldata.rl_jrx_sparemap);
1880                 bus_dma_tag_destroy(sc->rl_ldata.rl_jrx_mtag);
1881         }
1882         /* Unload and free the stats buffer and map */
1883
1884         if (sc->rl_ldata.rl_stag) {
1885                 if (sc->rl_ldata.rl_stats_addr)
1886                         bus_dmamap_unload(sc->rl_ldata.rl_stag,
1887                             sc->rl_ldata.rl_smap);
1888                 if (sc->rl_ldata.rl_stats)
1889                         bus_dmamem_free(sc->rl_ldata.rl_stag,
1890                             sc->rl_ldata.rl_stats, sc->rl_ldata.rl_smap);
1891                 bus_dma_tag_destroy(sc->rl_ldata.rl_stag);
1892         }
1893
1894         if (sc->rl_parent_tag)
1895                 bus_dma_tag_destroy(sc->rl_parent_tag);
1896
1897         mtx_destroy(&sc->rl_mtx);
1898
1899         return (0);
1900 }
1901
1902 static __inline void
1903 re_discard_rxbuf(struct rl_softc *sc, int idx)
1904 {
1905         struct rl_desc          *desc;
1906         struct rl_rxdesc        *rxd;
1907         uint32_t                cmdstat;
1908
1909         if (sc->rl_ifp->if_mtu > RL_MTU &&
1910             (sc->rl_flags & RL_FLAG_JUMBOV2) != 0)
1911                 rxd = &sc->rl_ldata.rl_jrx_desc[idx];
1912         else
1913                 rxd = &sc->rl_ldata.rl_rx_desc[idx];
1914         desc = &sc->rl_ldata.rl_rx_list[idx];
1915         desc->rl_vlanctl = 0;
1916         cmdstat = rxd->rx_size;
1917         if (idx == sc->rl_ldata.rl_rx_desc_cnt - 1)
1918                 cmdstat |= RL_RDESC_CMD_EOR;
1919         desc->rl_cmdstat = htole32(cmdstat | RL_RDESC_CMD_OWN);
1920 }
1921
1922 static int
1923 re_newbuf(struct rl_softc *sc, int idx)
1924 {
1925         struct mbuf             *m;
1926         struct rl_rxdesc        *rxd;
1927         bus_dma_segment_t       segs[1];
1928         bus_dmamap_t            map;
1929         struct rl_desc          *desc;
1930         uint32_t                cmdstat;
1931         int                     error, nsegs;
1932
1933         m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
1934         if (m == NULL)
1935                 return (ENOBUFS);
1936
1937         m->m_len = m->m_pkthdr.len = MCLBYTES;
1938 #ifdef RE_FIXUP_RX
1939         /*
1940          * This is part of an evil trick to deal with non-x86 platforms.
1941          * The RealTek chip requires RX buffers to be aligned on 64-bit
1942          * boundaries, but that will hose non-x86 machines. To get around
1943          * this, we leave some empty space at the start of each buffer
1944          * and for non-x86 hosts, we copy the buffer back six bytes
1945          * to achieve word alignment. This is slightly more efficient
1946          * than allocating a new buffer, copying the contents, and
1947          * discarding the old buffer.
1948          */
1949         m_adj(m, RE_ETHER_ALIGN);
1950 #endif
1951         error = bus_dmamap_load_mbuf_sg(sc->rl_ldata.rl_rx_mtag,
1952             sc->rl_ldata.rl_rx_sparemap, m, segs, &nsegs, BUS_DMA_NOWAIT);
1953         if (error != 0) {
1954                 m_freem(m);
1955                 return (ENOBUFS);
1956         }
1957         KASSERT(nsegs == 1, ("%s: %d segment returned!", __func__, nsegs));
1958
1959         rxd = &sc->rl_ldata.rl_rx_desc[idx];
1960         if (rxd->rx_m != NULL) {
1961                 bus_dmamap_sync(sc->rl_ldata.rl_rx_mtag, rxd->rx_dmamap,
1962                     BUS_DMASYNC_POSTREAD);
1963                 bus_dmamap_unload(sc->rl_ldata.rl_rx_mtag, rxd->rx_dmamap);
1964         }
1965
1966         rxd->rx_m = m;
1967         map = rxd->rx_dmamap;
1968         rxd->rx_dmamap = sc->rl_ldata.rl_rx_sparemap;
1969         rxd->rx_size = segs[0].ds_len;
1970         sc->rl_ldata.rl_rx_sparemap = map;
1971         bus_dmamap_sync(sc->rl_ldata.rl_rx_mtag, rxd->rx_dmamap,
1972             BUS_DMASYNC_PREREAD);
1973
1974         desc = &sc->rl_ldata.rl_rx_list[idx];
1975         desc->rl_vlanctl = 0;
1976         desc->rl_bufaddr_lo = htole32(RL_ADDR_LO(segs[0].ds_addr));
1977         desc->rl_bufaddr_hi = htole32(RL_ADDR_HI(segs[0].ds_addr));
1978         cmdstat = segs[0].ds_len;
1979         if (idx == sc->rl_ldata.rl_rx_desc_cnt - 1)
1980                 cmdstat |= RL_RDESC_CMD_EOR;
1981         desc->rl_cmdstat = htole32(cmdstat | RL_RDESC_CMD_OWN);
1982
1983         return (0);
1984 }
1985
1986 static int
1987 re_jumbo_newbuf(struct rl_softc *sc, int idx)
1988 {
1989         struct mbuf             *m;
1990         struct rl_rxdesc        *rxd;
1991         bus_dma_segment_t       segs[1];
1992         bus_dmamap_t            map;
1993         struct rl_desc          *desc;
1994         uint32_t                cmdstat;
1995         int                     error, nsegs;
1996
1997         m = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, MJUM9BYTES);
1998         if (m == NULL)
1999                 return (ENOBUFS);
2000         m->m_len = m->m_pkthdr.len = MJUM9BYTES;
2001 #ifdef RE_FIXUP_RX
2002         m_adj(m, RE_ETHER_ALIGN);
2003 #endif
2004         error = bus_dmamap_load_mbuf_sg(sc->rl_ldata.rl_jrx_mtag,
2005             sc->rl_ldata.rl_jrx_sparemap, m, segs, &nsegs, BUS_DMA_NOWAIT);
2006         if (error != 0) {
2007                 m_freem(m);
2008                 return (ENOBUFS);
2009         }
2010         KASSERT(nsegs == 1, ("%s: %d segment returned!", __func__, nsegs));
2011
2012         rxd = &sc->rl_ldata.rl_jrx_desc[idx];
2013         if (rxd->rx_m != NULL) {
2014                 bus_dmamap_sync(sc->rl_ldata.rl_jrx_mtag, rxd->rx_dmamap,
2015                     BUS_DMASYNC_POSTREAD);
2016                 bus_dmamap_unload(sc->rl_ldata.rl_jrx_mtag, rxd->rx_dmamap);
2017         }
2018
2019         rxd->rx_m = m;
2020         map = rxd->rx_dmamap;
2021         rxd->rx_dmamap = sc->rl_ldata.rl_jrx_sparemap;
2022         rxd->rx_size = segs[0].ds_len;
2023         sc->rl_ldata.rl_jrx_sparemap = map;
2024         bus_dmamap_sync(sc->rl_ldata.rl_jrx_mtag, rxd->rx_dmamap,
2025             BUS_DMASYNC_PREREAD);
2026
2027         desc = &sc->rl_ldata.rl_rx_list[idx];
2028         desc->rl_vlanctl = 0;
2029         desc->rl_bufaddr_lo = htole32(RL_ADDR_LO(segs[0].ds_addr));
2030         desc->rl_bufaddr_hi = htole32(RL_ADDR_HI(segs[0].ds_addr));
2031         cmdstat = segs[0].ds_len;
2032         if (idx == sc->rl_ldata.rl_rx_desc_cnt - 1)
2033                 cmdstat |= RL_RDESC_CMD_EOR;
2034         desc->rl_cmdstat = htole32(cmdstat | RL_RDESC_CMD_OWN);
2035
2036         return (0);
2037 }
2038
2039 #ifdef RE_FIXUP_RX
2040 static __inline void
2041 re_fixup_rx(struct mbuf *m)
2042 {
2043         int                     i;
2044         uint16_t                *src, *dst;
2045
2046         src = mtod(m, uint16_t *);
2047         dst = src - (RE_ETHER_ALIGN - ETHER_ALIGN) / sizeof *src;
2048
2049         for (i = 0; i < (m->m_len / sizeof(uint16_t) + 1); i++)
2050                 *dst++ = *src++;
2051
2052         m->m_data -= RE_ETHER_ALIGN - ETHER_ALIGN;
2053 }
2054 #endif
2055
2056 static int
2057 re_tx_list_init(struct rl_softc *sc)
2058 {
2059         struct rl_desc          *desc;
2060         int                     i;
2061
2062         RL_LOCK_ASSERT(sc);
2063
2064         bzero(sc->rl_ldata.rl_tx_list,
2065             sc->rl_ldata.rl_tx_desc_cnt * sizeof(struct rl_desc));
2066         for (i = 0; i < sc->rl_ldata.rl_tx_desc_cnt; i++)
2067                 sc->rl_ldata.rl_tx_desc[i].tx_m = NULL;
2068 #ifdef DEV_NETMAP
2069         re_netmap_tx_init(sc);
2070 #endif /* DEV_NETMAP */
2071         /* Set EOR. */
2072         desc = &sc->rl_ldata.rl_tx_list[sc->rl_ldata.rl_tx_desc_cnt - 1];
2073         desc->rl_cmdstat |= htole32(RL_TDESC_CMD_EOR);
2074
2075         bus_dmamap_sync(sc->rl_ldata.rl_tx_list_tag,
2076             sc->rl_ldata.rl_tx_list_map,
2077             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2078
2079         sc->rl_ldata.rl_tx_prodidx = 0;
2080         sc->rl_ldata.rl_tx_considx = 0;
2081         sc->rl_ldata.rl_tx_free = sc->rl_ldata.rl_tx_desc_cnt;
2082
2083         return (0);
2084 }
2085
2086 static int
2087 re_rx_list_init(struct rl_softc *sc)
2088 {
2089         int                     error, i;
2090
2091         bzero(sc->rl_ldata.rl_rx_list,
2092             sc->rl_ldata.rl_rx_desc_cnt * sizeof(struct rl_desc));
2093         for (i = 0; i < sc->rl_ldata.rl_rx_desc_cnt; i++) {
2094                 sc->rl_ldata.rl_rx_desc[i].rx_m = NULL;
2095                 if ((error = re_newbuf(sc, i)) != 0)
2096                         return (error);
2097         }
2098 #ifdef DEV_NETMAP
2099         re_netmap_rx_init(sc);
2100 #endif /* DEV_NETMAP */
2101
2102         /* Flush the RX descriptors */
2103
2104         bus_dmamap_sync(sc->rl_ldata.rl_rx_list_tag,
2105             sc->rl_ldata.rl_rx_list_map,
2106             BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD);
2107
2108         sc->rl_ldata.rl_rx_prodidx = 0;
2109         sc->rl_head = sc->rl_tail = NULL;
2110         sc->rl_int_rx_act = 0;
2111
2112         return (0);
2113 }
2114
2115 static int
2116 re_jrx_list_init(struct rl_softc *sc)
2117 {
2118         int                     error, i;
2119
2120         bzero(sc->rl_ldata.rl_rx_list,
2121             sc->rl_ldata.rl_rx_desc_cnt * sizeof(struct rl_desc));
2122         for (i = 0; i < sc->rl_ldata.rl_rx_desc_cnt; i++) {
2123                 sc->rl_ldata.rl_jrx_desc[i].rx_m = NULL;
2124                 if ((error = re_jumbo_newbuf(sc, i)) != 0)
2125                         return (error);
2126         }
2127
2128         bus_dmamap_sync(sc->rl_ldata.rl_rx_list_tag,
2129             sc->rl_ldata.rl_rx_list_map,
2130             BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2131
2132         sc->rl_ldata.rl_rx_prodidx = 0;
2133         sc->rl_head = sc->rl_tail = NULL;
2134         sc->rl_int_rx_act = 0;
2135
2136         return (0);
2137 }
2138
2139 /*
2140  * RX handler for C+ and 8169. For the gigE chips, we support
2141  * the reception of jumbo frames that have been fragmented
2142  * across multiple 2K mbuf cluster buffers.
2143  */
2144 static int
2145 re_rxeof(struct rl_softc *sc, int *rx_npktsp)
2146 {
2147         struct mbuf             *m;
2148         struct ifnet            *ifp;
2149         int                     i, rxerr, total_len;
2150         struct rl_desc          *cur_rx;
2151         u_int32_t               rxstat, rxvlan;
2152         int                     jumbo, maxpkt = 16, rx_npkts = 0;
2153
2154         RL_LOCK_ASSERT(sc);
2155
2156         ifp = sc->rl_ifp;
2157 #ifdef DEV_NETMAP
2158         if (netmap_rx_irq(ifp, 0, &rx_npkts))
2159                 return 0;
2160 #endif /* DEV_NETMAP */
2161         if (ifp->if_mtu > RL_MTU && (sc->rl_flags & RL_FLAG_JUMBOV2) != 0)
2162                 jumbo = 1;
2163         else
2164                 jumbo = 0;
2165
2166         /* Invalidate the descriptor memory */
2167
2168         bus_dmamap_sync(sc->rl_ldata.rl_rx_list_tag,
2169             sc->rl_ldata.rl_rx_list_map,
2170             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2171
2172         for (i = sc->rl_ldata.rl_rx_prodidx; maxpkt > 0;
2173             i = RL_RX_DESC_NXT(sc, i)) {
2174                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
2175                         break;
2176                 cur_rx = &sc->rl_ldata.rl_rx_list[i];
2177                 rxstat = le32toh(cur_rx->rl_cmdstat);
2178                 if ((rxstat & RL_RDESC_STAT_OWN) != 0)
2179                         break;
2180                 total_len = rxstat & sc->rl_rxlenmask;
2181                 rxvlan = le32toh(cur_rx->rl_vlanctl);
2182                 if (jumbo != 0)
2183                         m = sc->rl_ldata.rl_jrx_desc[i].rx_m;
2184                 else
2185                         m = sc->rl_ldata.rl_rx_desc[i].rx_m;
2186
2187                 if ((sc->rl_flags & RL_FLAG_JUMBOV2) != 0 &&
2188                     (rxstat & (RL_RDESC_STAT_SOF | RL_RDESC_STAT_EOF)) !=
2189                     (RL_RDESC_STAT_SOF | RL_RDESC_STAT_EOF)) {
2190                         /*
2191                          * RTL8168C or later controllers do not
2192                          * support multi-fragment packet.
2193                          */
2194                         re_discard_rxbuf(sc, i);
2195                         continue;
2196                 } else if ((rxstat & RL_RDESC_STAT_EOF) == 0) {
2197                         if (re_newbuf(sc, i) != 0) {
2198                                 /*
2199                                  * If this is part of a multi-fragment packet,
2200                                  * discard all the pieces.
2201                                  */
2202                                 if (sc->rl_head != NULL) {
2203                                         m_freem(sc->rl_head);
2204                                         sc->rl_head = sc->rl_tail = NULL;
2205                                 }
2206                                 re_discard_rxbuf(sc, i);
2207                                 continue;
2208                         }
2209                         m->m_len = RE_RX_DESC_BUFLEN;
2210                         if (sc->rl_head == NULL)
2211                                 sc->rl_head = sc->rl_tail = m;
2212                         else {
2213                                 m->m_flags &= ~M_PKTHDR;
2214                                 sc->rl_tail->m_next = m;
2215                                 sc->rl_tail = m;
2216                         }
2217                         continue;
2218                 }
2219
2220                 /*
2221                  * NOTE: for the 8139C+, the frame length field
2222                  * is always 12 bits in size, but for the gigE chips,
2223                  * it is 13 bits (since the max RX frame length is 16K).
2224                  * Unfortunately, all 32 bits in the status word
2225                  * were already used, so to make room for the extra
2226                  * length bit, RealTek took out the 'frame alignment
2227                  * error' bit and shifted the other status bits
2228                  * over one slot. The OWN, EOR, FS and LS bits are
2229                  * still in the same places. We have already extracted
2230                  * the frame length and checked the OWN bit, so rather
2231                  * than using an alternate bit mapping, we shift the
2232                  * status bits one space to the right so we can evaluate
2233                  * them using the 8169 status as though it was in the
2234                  * same format as that of the 8139C+.
2235                  */
2236                 if (sc->rl_type == RL_8169)
2237                         rxstat >>= 1;
2238
2239                 /*
2240                  * if total_len > 2^13-1, both _RXERRSUM and _GIANT will be
2241                  * set, but if CRC is clear, it will still be a valid frame.
2242                  */
2243                 if ((rxstat & RL_RDESC_STAT_RXERRSUM) != 0) {
2244                         rxerr = 1;
2245                         if ((sc->rl_flags & RL_FLAG_JUMBOV2) == 0 &&
2246                             total_len > 8191 &&
2247                             (rxstat & RL_RDESC_STAT_ERRS) == RL_RDESC_STAT_GIANT)
2248                                 rxerr = 0;
2249                         if (rxerr != 0) {
2250                                 if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
2251                                 /*
2252                                  * If this is part of a multi-fragment packet,
2253                                  * discard all the pieces.
2254                                  */
2255                                 if (sc->rl_head != NULL) {
2256                                         m_freem(sc->rl_head);
2257                                         sc->rl_head = sc->rl_tail = NULL;
2258                                 }
2259                                 re_discard_rxbuf(sc, i);
2260                                 continue;
2261                         }
2262                 }
2263
2264                 /*
2265                  * If allocating a replacement mbuf fails,
2266                  * reload the current one.
2267                  */
2268                 if (jumbo != 0)
2269                         rxerr = re_jumbo_newbuf(sc, i);
2270                 else
2271                         rxerr = re_newbuf(sc, i);
2272                 if (rxerr != 0) {
2273                         if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1);
2274                         if (sc->rl_head != NULL) {
2275                                 m_freem(sc->rl_head);
2276                                 sc->rl_head = sc->rl_tail = NULL;
2277                         }
2278                         re_discard_rxbuf(sc, i);
2279                         continue;
2280                 }
2281
2282                 if (sc->rl_head != NULL) {
2283                         if (jumbo != 0)
2284                                 m->m_len = total_len;
2285                         else {
2286                                 m->m_len = total_len % RE_RX_DESC_BUFLEN;
2287                                 if (m->m_len == 0)
2288                                         m->m_len = RE_RX_DESC_BUFLEN;
2289                         }
2290                         /*
2291                          * Special case: if there's 4 bytes or less
2292                          * in this buffer, the mbuf can be discarded:
2293                          * the last 4 bytes is the CRC, which we don't
2294                          * care about anyway.
2295                          */
2296                         if (m->m_len <= ETHER_CRC_LEN) {
2297                                 sc->rl_tail->m_len -=
2298                                     (ETHER_CRC_LEN - m->m_len);
2299                                 m_freem(m);
2300                         } else {
2301                                 m->m_len -= ETHER_CRC_LEN;
2302                                 m->m_flags &= ~M_PKTHDR;
2303                                 sc->rl_tail->m_next = m;
2304                         }
2305                         m = sc->rl_head;
2306                         sc->rl_head = sc->rl_tail = NULL;
2307                         m->m_pkthdr.len = total_len - ETHER_CRC_LEN;
2308                 } else
2309                         m->m_pkthdr.len = m->m_len =
2310                             (total_len - ETHER_CRC_LEN);
2311
2312 #ifdef RE_FIXUP_RX
2313                 re_fixup_rx(m);
2314 #endif
2315                 if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);
2316                 m->m_pkthdr.rcvif = ifp;
2317
2318                 /* Do RX checksumming if enabled */
2319
2320                 if (ifp->if_capenable & IFCAP_RXCSUM) {
2321                         if ((sc->rl_flags & RL_FLAG_DESCV2) == 0) {
2322                                 /* Check IP header checksum */
2323                                 if (rxstat & RL_RDESC_STAT_PROTOID)
2324                                         m->m_pkthdr.csum_flags |=
2325                                             CSUM_IP_CHECKED;
2326                                 if (!(rxstat & RL_RDESC_STAT_IPSUMBAD))
2327                                         m->m_pkthdr.csum_flags |=
2328                                             CSUM_IP_VALID;
2329
2330                                 /* Check TCP/UDP checksum */
2331                                 if ((RL_TCPPKT(rxstat) &&
2332                                     !(rxstat & RL_RDESC_STAT_TCPSUMBAD)) ||
2333                                     (RL_UDPPKT(rxstat) &&
2334                                      !(rxstat & RL_RDESC_STAT_UDPSUMBAD))) {
2335                                         m->m_pkthdr.csum_flags |=
2336                                                 CSUM_DATA_VALID|CSUM_PSEUDO_HDR;
2337                                         m->m_pkthdr.csum_data = 0xffff;
2338                                 }
2339                         } else {
2340                                 /*
2341                                  * RTL8168C/RTL816CP/RTL8111C/RTL8111CP
2342                                  */
2343                                 if ((rxstat & RL_RDESC_STAT_PROTOID) &&
2344                                     (rxvlan & RL_RDESC_IPV4))
2345                                         m->m_pkthdr.csum_flags |=
2346                                             CSUM_IP_CHECKED;
2347                                 if (!(rxstat & RL_RDESC_STAT_IPSUMBAD) &&
2348                                     (rxvlan & RL_RDESC_IPV4))
2349                                         m->m_pkthdr.csum_flags |=
2350                                             CSUM_IP_VALID;
2351                                 if (((rxstat & RL_RDESC_STAT_TCP) &&
2352                                     !(rxstat & RL_RDESC_STAT_TCPSUMBAD)) ||
2353                                     ((rxstat & RL_RDESC_STAT_UDP) &&
2354                                     !(rxstat & RL_RDESC_STAT_UDPSUMBAD))) {
2355                                         m->m_pkthdr.csum_flags |=
2356                                                 CSUM_DATA_VALID|CSUM_PSEUDO_HDR;
2357                                         m->m_pkthdr.csum_data = 0xffff;
2358                                 }
2359                         }
2360                 }
2361                 maxpkt--;
2362                 if (rxvlan & RL_RDESC_VLANCTL_TAG) {
2363                         m->m_pkthdr.ether_vtag =
2364                             bswap16((rxvlan & RL_RDESC_VLANCTL_DATA));
2365                         m->m_flags |= M_VLANTAG;
2366                 }
2367                 RL_UNLOCK(sc);
2368                 (*ifp->if_input)(ifp, m);
2369                 RL_LOCK(sc);
2370                 rx_npkts++;
2371         }
2372
2373         /* Flush the RX DMA ring */
2374
2375         bus_dmamap_sync(sc->rl_ldata.rl_rx_list_tag,
2376             sc->rl_ldata.rl_rx_list_map,
2377             BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD);
2378
2379         sc->rl_ldata.rl_rx_prodidx = i;
2380
2381         if (rx_npktsp != NULL)
2382                 *rx_npktsp = rx_npkts;
2383         if (maxpkt)
2384                 return (EAGAIN);
2385
2386         return (0);
2387 }
2388
2389 static void
2390 re_txeof(struct rl_softc *sc)
2391 {
2392         struct ifnet            *ifp;
2393         struct rl_txdesc        *txd;
2394         u_int32_t               txstat;
2395         int                     cons;
2396
2397         cons = sc->rl_ldata.rl_tx_considx;
2398         if (cons == sc->rl_ldata.rl_tx_prodidx)
2399                 return;
2400
2401         ifp = sc->rl_ifp;
2402 #ifdef DEV_NETMAP
2403         if (netmap_tx_irq(ifp, 0))
2404                 return;
2405 #endif /* DEV_NETMAP */
2406         /* Invalidate the TX descriptor list */
2407         bus_dmamap_sync(sc->rl_ldata.rl_tx_list_tag,
2408             sc->rl_ldata.rl_tx_list_map,
2409             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2410
2411         for (; cons != sc->rl_ldata.rl_tx_prodidx;
2412             cons = RL_TX_DESC_NXT(sc, cons)) {
2413                 txstat = le32toh(sc->rl_ldata.rl_tx_list[cons].rl_cmdstat);
2414                 if (txstat & RL_TDESC_STAT_OWN)
2415                         break;
2416                 /*
2417                  * We only stash mbufs in the last descriptor
2418                  * in a fragment chain, which also happens to
2419                  * be the only place where the TX status bits
2420                  * are valid.
2421                  */
2422                 if (txstat & RL_TDESC_CMD_EOF) {
2423                         txd = &sc->rl_ldata.rl_tx_desc[cons];
2424                         bus_dmamap_sync(sc->rl_ldata.rl_tx_mtag,
2425                             txd->tx_dmamap, BUS_DMASYNC_POSTWRITE);
2426                         bus_dmamap_unload(sc->rl_ldata.rl_tx_mtag,
2427                             txd->tx_dmamap);
2428                         KASSERT(txd->tx_m != NULL,
2429                             ("%s: freeing NULL mbufs!", __func__));
2430                         m_freem(txd->tx_m);
2431                         txd->tx_m = NULL;
2432                         if (txstat & (RL_TDESC_STAT_EXCESSCOL|
2433                             RL_TDESC_STAT_COLCNT))
2434                                 if_inc_counter(ifp, IFCOUNTER_COLLISIONS, 1);
2435                         if (txstat & RL_TDESC_STAT_TXERRSUM)
2436                                 if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
2437                         else
2438                                 if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1);
2439                 }
2440                 sc->rl_ldata.rl_tx_free++;
2441                 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
2442         }
2443         sc->rl_ldata.rl_tx_considx = cons;
2444
2445         /* No changes made to the TX ring, so no flush needed */
2446
2447         if (sc->rl_ldata.rl_tx_free != sc->rl_ldata.rl_tx_desc_cnt) {
2448 #ifdef RE_TX_MODERATION
2449                 /*
2450                  * If not all descriptors have been reaped yet, reload
2451                  * the timer so that we will eventually get another
2452                  * interrupt that will cause us to re-enter this routine.
2453                  * This is done in case the transmitter has gone idle.
2454                  */
2455                 CSR_WRITE_4(sc, RL_TIMERCNT, 1);
2456 #endif
2457         } else
2458                 sc->rl_watchdog_timer = 0;
2459 }
2460
2461 static void
2462 re_tick(void *xsc)
2463 {
2464         struct rl_softc         *sc;
2465         struct mii_data         *mii;
2466
2467         sc = xsc;
2468
2469         RL_LOCK_ASSERT(sc);
2470
2471         mii = device_get_softc(sc->rl_miibus);
2472         mii_tick(mii);
2473         if ((sc->rl_flags & RL_FLAG_LINK) == 0)
2474                 re_miibus_statchg(sc->rl_dev);
2475         /*
2476          * Reclaim transmitted frames here. Technically it is not
2477          * necessary to do here but it ensures periodic reclamation
2478          * regardless of Tx completion interrupt which seems to be
2479          * lost on PCIe based controllers under certain situations.
2480          */
2481         re_txeof(sc);
2482         re_watchdog(sc);
2483         callout_reset(&sc->rl_stat_callout, hz, re_tick, sc);
2484 }
2485
2486 #ifdef DEVICE_POLLING
2487 static int
2488 re_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
2489 {
2490         struct rl_softc *sc = ifp->if_softc;
2491         int rx_npkts = 0;
2492
2493         RL_LOCK(sc);
2494         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
2495                 rx_npkts = re_poll_locked(ifp, cmd, count);
2496         RL_UNLOCK(sc);
2497         return (rx_npkts);
2498 }
2499
2500 static int
2501 re_poll_locked(struct ifnet *ifp, enum poll_cmd cmd, int count)
2502 {
2503         struct rl_softc *sc = ifp->if_softc;
2504         int rx_npkts;
2505
2506         RL_LOCK_ASSERT(sc);
2507
2508         sc->rxcycles = count;
2509         re_rxeof(sc, &rx_npkts);
2510         re_txeof(sc);
2511
2512         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
2513                 re_start_locked(ifp);
2514
2515         if (cmd == POLL_AND_CHECK_STATUS) { /* also check status register */
2516                 u_int16_t       status;
2517
2518                 status = CSR_READ_2(sc, RL_ISR);
2519                 if (status == 0xffff)
2520                         return (rx_npkts);
2521                 if (status)
2522                         CSR_WRITE_2(sc, RL_ISR, status);
2523                 if ((status & (RL_ISR_TX_OK | RL_ISR_TX_DESC_UNAVAIL)) &&
2524                     (sc->rl_flags & RL_FLAG_PCIE))
2525                         CSR_WRITE_1(sc, sc->rl_txstart, RL_TXSTART_START);
2526
2527                 /*
2528                  * XXX check behaviour on receiver stalls.
2529                  */
2530
2531                 if (status & RL_ISR_SYSTEM_ERR) {
2532                         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2533                         re_init_locked(sc);
2534                 }
2535         }
2536         return (rx_npkts);
2537 }
2538 #endif /* DEVICE_POLLING */
2539
2540 static int
2541 re_intr(void *arg)
2542 {
2543         struct rl_softc         *sc;
2544         uint16_t                status;
2545
2546         sc = arg;
2547
2548         status = CSR_READ_2(sc, RL_ISR);
2549         if (status == 0xFFFF || (status & RL_INTRS_CPLUS) == 0)
2550                 return (FILTER_STRAY);
2551         CSR_WRITE_2(sc, RL_IMR, 0);
2552
2553         taskqueue_enqueue_fast(taskqueue_fast, &sc->rl_inttask);
2554
2555         return (FILTER_HANDLED);
2556 }
2557
2558 static void
2559 re_int_task(void *arg, int npending)
2560 {
2561         struct rl_softc         *sc;
2562         struct ifnet            *ifp;
2563         u_int16_t               status;
2564         int                     rval = 0;
2565
2566         sc = arg;
2567         ifp = sc->rl_ifp;
2568
2569         RL_LOCK(sc);
2570
2571         status = CSR_READ_2(sc, RL_ISR);
2572         CSR_WRITE_2(sc, RL_ISR, status);
2573
2574         if (sc->suspended ||
2575             (ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
2576                 RL_UNLOCK(sc);
2577                 return;
2578         }
2579
2580 #ifdef DEVICE_POLLING
2581         if  (ifp->if_capenable & IFCAP_POLLING) {
2582                 RL_UNLOCK(sc);
2583                 return;
2584         }
2585 #endif
2586
2587         if (status & (RL_ISR_RX_OK|RL_ISR_RX_ERR|RL_ISR_FIFO_OFLOW))
2588                 rval = re_rxeof(sc, NULL);
2589
2590         /*
2591          * Some chips will ignore a second TX request issued
2592          * while an existing transmission is in progress. If
2593          * the transmitter goes idle but there are still
2594          * packets waiting to be sent, we need to restart the
2595          * channel here to flush them out. This only seems to
2596          * be required with the PCIe devices.
2597          */
2598         if ((status & (RL_ISR_TX_OK | RL_ISR_TX_DESC_UNAVAIL)) &&
2599             (sc->rl_flags & RL_FLAG_PCIE))
2600                 CSR_WRITE_1(sc, sc->rl_txstart, RL_TXSTART_START);
2601         if (status & (
2602 #ifdef RE_TX_MODERATION
2603             RL_ISR_TIMEOUT_EXPIRED|
2604 #else
2605             RL_ISR_TX_OK|
2606 #endif
2607             RL_ISR_TX_ERR|RL_ISR_TX_DESC_UNAVAIL))
2608                 re_txeof(sc);
2609
2610         if (status & RL_ISR_SYSTEM_ERR) {
2611                 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2612                 re_init_locked(sc);
2613         }
2614
2615         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
2616                 re_start_locked(ifp);
2617
2618         RL_UNLOCK(sc);
2619
2620         if ((CSR_READ_2(sc, RL_ISR) & RL_INTRS_CPLUS) || rval) {
2621                 taskqueue_enqueue_fast(taskqueue_fast, &sc->rl_inttask);
2622                 return;
2623         }
2624
2625         CSR_WRITE_2(sc, RL_IMR, RL_INTRS_CPLUS);
2626 }
2627
2628 static void
2629 re_intr_msi(void *xsc)
2630 {
2631         struct rl_softc         *sc;
2632         struct ifnet            *ifp;
2633         uint16_t                intrs, status;
2634
2635         sc = xsc;
2636         RL_LOCK(sc);
2637
2638         ifp = sc->rl_ifp;
2639 #ifdef DEVICE_POLLING
2640         if (ifp->if_capenable & IFCAP_POLLING) {
2641                 RL_UNLOCK(sc);
2642                 return;
2643         }
2644 #endif
2645         /* Disable interrupts. */
2646         CSR_WRITE_2(sc, RL_IMR, 0);
2647         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
2648                 RL_UNLOCK(sc);
2649                 return;
2650         }
2651
2652         intrs = RL_INTRS_CPLUS;
2653         status = CSR_READ_2(sc, RL_ISR);
2654         CSR_WRITE_2(sc, RL_ISR, status);
2655         if (sc->rl_int_rx_act > 0) {
2656                 intrs &= ~(RL_ISR_RX_OK | RL_ISR_RX_ERR | RL_ISR_FIFO_OFLOW |
2657                     RL_ISR_RX_OVERRUN);
2658                 status &= ~(RL_ISR_RX_OK | RL_ISR_RX_ERR | RL_ISR_FIFO_OFLOW |
2659                     RL_ISR_RX_OVERRUN);
2660         }
2661
2662         if (status & (RL_ISR_TIMEOUT_EXPIRED | RL_ISR_RX_OK | RL_ISR_RX_ERR |
2663             RL_ISR_FIFO_OFLOW | RL_ISR_RX_OVERRUN)) {
2664                 re_rxeof(sc, NULL);
2665                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) {
2666                         if (sc->rl_int_rx_mod != 0 &&
2667                             (status & (RL_ISR_RX_OK | RL_ISR_RX_ERR |
2668                             RL_ISR_FIFO_OFLOW | RL_ISR_RX_OVERRUN)) != 0) {
2669                                 /* Rearm one-shot timer. */
2670                                 CSR_WRITE_4(sc, RL_TIMERCNT, 1);
2671                                 intrs &= ~(RL_ISR_RX_OK | RL_ISR_RX_ERR |
2672                                     RL_ISR_FIFO_OFLOW | RL_ISR_RX_OVERRUN);
2673                                 sc->rl_int_rx_act = 1;
2674                         } else {
2675                                 intrs |= RL_ISR_RX_OK | RL_ISR_RX_ERR |
2676                                     RL_ISR_FIFO_OFLOW | RL_ISR_RX_OVERRUN;
2677                                 sc->rl_int_rx_act = 0;
2678                         }
2679                 }
2680         }
2681
2682         /*
2683          * Some chips will ignore a second TX request issued
2684          * while an existing transmission is in progress. If
2685          * the transmitter goes idle but there are still
2686          * packets waiting to be sent, we need to restart the
2687          * channel here to flush them out. This only seems to
2688          * be required with the PCIe devices.
2689          */
2690         if ((status & (RL_ISR_TX_OK | RL_ISR_TX_DESC_UNAVAIL)) &&
2691             (sc->rl_flags & RL_FLAG_PCIE))
2692                 CSR_WRITE_1(sc, sc->rl_txstart, RL_TXSTART_START);
2693         if (status & (RL_ISR_TX_OK | RL_ISR_TX_ERR | RL_ISR_TX_DESC_UNAVAIL))
2694                 re_txeof(sc);
2695
2696         if (status & RL_ISR_SYSTEM_ERR) {
2697                 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2698                 re_init_locked(sc);
2699         }
2700
2701         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) {
2702                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
2703                         re_start_locked(ifp);
2704                 CSR_WRITE_2(sc, RL_IMR, intrs);
2705         }
2706         RL_UNLOCK(sc);
2707 }
2708
2709 static int
2710 re_encap(struct rl_softc *sc, struct mbuf **m_head)
2711 {
2712         struct rl_txdesc        *txd, *txd_last;
2713         bus_dma_segment_t       segs[RL_NTXSEGS];
2714         bus_dmamap_t            map;
2715         struct mbuf             *m_new;
2716         struct rl_desc          *desc;
2717         int                     nsegs, prod;
2718         int                     i, error, ei, si;
2719         int                     padlen;
2720         uint32_t                cmdstat, csum_flags, vlanctl;
2721
2722         RL_LOCK_ASSERT(sc);
2723         M_ASSERTPKTHDR((*m_head));
2724
2725         /*
2726          * With some of the RealTek chips, using the checksum offload
2727          * support in conjunction with the autopadding feature results
2728          * in the transmission of corrupt frames. For example, if we
2729          * need to send a really small IP fragment that's less than 60
2730          * bytes in size, and IP header checksumming is enabled, the
2731          * resulting ethernet frame that appears on the wire will
2732          * have garbled payload. To work around this, if TX IP checksum
2733          * offload is enabled, we always manually pad short frames out
2734          * to the minimum ethernet frame size.
2735          */
2736         if ((sc->rl_flags & RL_FLAG_AUTOPAD) == 0 &&
2737             (*m_head)->m_pkthdr.len < RL_IP4CSUMTX_PADLEN &&
2738             ((*m_head)->m_pkthdr.csum_flags & CSUM_IP) != 0) {
2739                 padlen = RL_MIN_FRAMELEN - (*m_head)->m_pkthdr.len;
2740                 if (M_WRITABLE(*m_head) == 0) {
2741                         /* Get a writable copy. */
2742                         m_new = m_dup(*m_head, M_NOWAIT);
2743                         m_freem(*m_head);
2744                         if (m_new == NULL) {
2745                                 *m_head = NULL;
2746                                 return (ENOBUFS);
2747                         }
2748                         *m_head = m_new;
2749                 }
2750                 if ((*m_head)->m_next != NULL ||
2751                     M_TRAILINGSPACE(*m_head) < padlen) {
2752                         m_new = m_defrag(*m_head, M_NOWAIT);
2753                         if (m_new == NULL) {
2754                                 m_freem(*m_head);
2755                                 *m_head = NULL;
2756                                 return (ENOBUFS);
2757                         }
2758                 } else
2759                         m_new = *m_head;
2760
2761                 /*
2762                  * Manually pad short frames, and zero the pad space
2763                  * to avoid leaking data.
2764                  */
2765                 bzero(mtod(m_new, char *) + m_new->m_pkthdr.len, padlen);
2766                 m_new->m_pkthdr.len += padlen;
2767                 m_new->m_len = m_new->m_pkthdr.len;
2768                 *m_head = m_new;
2769         }
2770
2771         prod = sc->rl_ldata.rl_tx_prodidx;
2772         txd = &sc->rl_ldata.rl_tx_desc[prod];
2773         error = bus_dmamap_load_mbuf_sg(sc->rl_ldata.rl_tx_mtag, txd->tx_dmamap,
2774             *m_head, segs, &nsegs, BUS_DMA_NOWAIT);
2775         if (error == EFBIG) {
2776                 m_new = m_collapse(*m_head, M_NOWAIT, RL_NTXSEGS);
2777                 if (m_new == NULL) {
2778                         m_freem(*m_head);
2779                         *m_head = NULL;
2780                         return (ENOBUFS);
2781                 }
2782                 *m_head = m_new;
2783                 error = bus_dmamap_load_mbuf_sg(sc->rl_ldata.rl_tx_mtag,
2784                     txd->tx_dmamap, *m_head, segs, &nsegs, BUS_DMA_NOWAIT);
2785                 if (error != 0) {
2786                         m_freem(*m_head);
2787                         *m_head = NULL;
2788                         return (error);
2789                 }
2790         } else if (error != 0)
2791                 return (error);
2792         if (nsegs == 0) {
2793                 m_freem(*m_head);
2794                 *m_head = NULL;
2795                 return (EIO);
2796         }
2797
2798         /* Check for number of available descriptors. */
2799         if (sc->rl_ldata.rl_tx_free - nsegs <= 1) {
2800                 bus_dmamap_unload(sc->rl_ldata.rl_tx_mtag, txd->tx_dmamap);
2801                 return (ENOBUFS);
2802         }
2803
2804         bus_dmamap_sync(sc->rl_ldata.rl_tx_mtag, txd->tx_dmamap,
2805             BUS_DMASYNC_PREWRITE);
2806
2807         /*
2808          * Set up checksum offload. Note: checksum offload bits must
2809          * appear in all descriptors of a multi-descriptor transmit
2810          * attempt. This is according to testing done with an 8169
2811          * chip. This is a requirement.
2812          */
2813         vlanctl = 0;
2814         csum_flags = 0;
2815         if (((*m_head)->m_pkthdr.csum_flags & CSUM_TSO) != 0) {
2816                 if ((sc->rl_flags & RL_FLAG_DESCV2) != 0) {
2817                         csum_flags |= RL_TDESC_CMD_LGSEND;
2818                         vlanctl |= ((uint32_t)(*m_head)->m_pkthdr.tso_segsz <<
2819                             RL_TDESC_CMD_MSSVALV2_SHIFT);
2820                 } else {
2821                         csum_flags |= RL_TDESC_CMD_LGSEND |
2822                             ((uint32_t)(*m_head)->m_pkthdr.tso_segsz <<
2823                             RL_TDESC_CMD_MSSVAL_SHIFT);
2824                 }
2825         } else {
2826                 /*
2827                  * Unconditionally enable IP checksum if TCP or UDP
2828                  * checksum is required. Otherwise, TCP/UDP checksum
2829                  * doesn't make effects.
2830                  */
2831                 if (((*m_head)->m_pkthdr.csum_flags & RE_CSUM_FEATURES) != 0) {
2832                         if ((sc->rl_flags & RL_FLAG_DESCV2) == 0) {
2833                                 csum_flags |= RL_TDESC_CMD_IPCSUM;
2834                                 if (((*m_head)->m_pkthdr.csum_flags &
2835                                     CSUM_TCP) != 0)
2836                                         csum_flags |= RL_TDESC_CMD_TCPCSUM;
2837                                 if (((*m_head)->m_pkthdr.csum_flags &
2838                                     CSUM_UDP) != 0)
2839                                         csum_flags |= RL_TDESC_CMD_UDPCSUM;
2840                         } else {
2841                                 vlanctl |= RL_TDESC_CMD_IPCSUMV2;
2842                                 if (((*m_head)->m_pkthdr.csum_flags &
2843                                     CSUM_TCP) != 0)
2844                                         vlanctl |= RL_TDESC_CMD_TCPCSUMV2;
2845                                 if (((*m_head)->m_pkthdr.csum_flags &
2846                                     CSUM_UDP) != 0)
2847                                         vlanctl |= RL_TDESC_CMD_UDPCSUMV2;
2848                         }
2849                 }
2850         }
2851
2852         /*
2853          * Set up hardware VLAN tagging. Note: vlan tag info must
2854          * appear in all descriptors of a multi-descriptor
2855          * transmission attempt.
2856          */
2857         if ((*m_head)->m_flags & M_VLANTAG)
2858                 vlanctl |= bswap16((*m_head)->m_pkthdr.ether_vtag) |
2859                     RL_TDESC_VLANCTL_TAG;
2860
2861         si = prod;
2862         for (i = 0; i < nsegs; i++, prod = RL_TX_DESC_NXT(sc, prod)) {
2863                 desc = &sc->rl_ldata.rl_tx_list[prod];
2864                 desc->rl_vlanctl = htole32(vlanctl);
2865                 desc->rl_bufaddr_lo = htole32(RL_ADDR_LO(segs[i].ds_addr));
2866                 desc->rl_bufaddr_hi = htole32(RL_ADDR_HI(segs[i].ds_addr));
2867                 cmdstat = segs[i].ds_len;
2868                 if (i != 0)
2869                         cmdstat |= RL_TDESC_CMD_OWN;
2870                 if (prod == sc->rl_ldata.rl_tx_desc_cnt - 1)
2871                         cmdstat |= RL_TDESC_CMD_EOR;
2872                 desc->rl_cmdstat = htole32(cmdstat | csum_flags);
2873                 sc->rl_ldata.rl_tx_free--;
2874         }
2875         /* Update producer index. */
2876         sc->rl_ldata.rl_tx_prodidx = prod;
2877
2878         /* Set EOF on the last descriptor. */
2879         ei = RL_TX_DESC_PRV(sc, prod);
2880         desc = &sc->rl_ldata.rl_tx_list[ei];
2881         desc->rl_cmdstat |= htole32(RL_TDESC_CMD_EOF);
2882
2883         desc = &sc->rl_ldata.rl_tx_list[si];
2884         /* Set SOF and transfer ownership of packet to the chip. */
2885         desc->rl_cmdstat |= htole32(RL_TDESC_CMD_OWN | RL_TDESC_CMD_SOF);
2886
2887         /*
2888          * Insure that the map for this transmission
2889          * is placed at the array index of the last descriptor
2890          * in this chain.  (Swap last and first dmamaps.)
2891          */
2892         txd_last = &sc->rl_ldata.rl_tx_desc[ei];
2893         map = txd->tx_dmamap;
2894         txd->tx_dmamap = txd_last->tx_dmamap;
2895         txd_last->tx_dmamap = map;
2896         txd_last->tx_m = *m_head;
2897
2898         return (0);
2899 }
2900
2901 static void
2902 re_start(struct ifnet *ifp)
2903 {
2904         struct rl_softc         *sc;
2905
2906         sc = ifp->if_softc;
2907         RL_LOCK(sc);
2908         re_start_locked(ifp);
2909         RL_UNLOCK(sc);
2910 }
2911
2912 /*
2913  * Main transmit routine for C+ and gigE NICs.
2914  */
2915 static void
2916 re_start_locked(struct ifnet *ifp)
2917 {
2918         struct rl_softc         *sc;
2919         struct mbuf             *m_head;
2920         int                     queued;
2921
2922         sc = ifp->if_softc;
2923
2924 #ifdef DEV_NETMAP
2925         /* XXX is this necessary ? */
2926         if (ifp->if_capenable & IFCAP_NETMAP) {
2927                 struct netmap_kring *kring = &NA(ifp)->tx_rings[0];
2928                 if (sc->rl_ldata.rl_tx_prodidx != kring->nr_hwcur) {
2929                         /* kick the tx unit */
2930                         CSR_WRITE_1(sc, sc->rl_txstart, RL_TXSTART_START);
2931 #ifdef RE_TX_MODERATION
2932                         CSR_WRITE_4(sc, RL_TIMERCNT, 1);
2933 #endif
2934                         sc->rl_watchdog_timer = 5;
2935                 }
2936                 return;
2937         }
2938 #endif /* DEV_NETMAP */
2939         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
2940             IFF_DRV_RUNNING || (sc->rl_flags & RL_FLAG_LINK) == 0)
2941                 return;
2942
2943         for (queued = 0; !IFQ_DRV_IS_EMPTY(&ifp->if_snd) &&
2944             sc->rl_ldata.rl_tx_free > 1;) {
2945                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
2946                 if (m_head == NULL)
2947                         break;
2948
2949                 if (re_encap(sc, &m_head) != 0) {
2950                         if (m_head == NULL)
2951                                 break;
2952                         IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
2953                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
2954                         break;
2955                 }
2956
2957                 /*
2958                  * If there's a BPF listener, bounce a copy of this frame
2959                  * to him.
2960                  */
2961                 ETHER_BPF_MTAP(ifp, m_head);
2962
2963                 queued++;
2964         }
2965
2966         if (queued == 0) {
2967 #ifdef RE_TX_MODERATION
2968                 if (sc->rl_ldata.rl_tx_free != sc->rl_ldata.rl_tx_desc_cnt)
2969                         CSR_WRITE_4(sc, RL_TIMERCNT, 1);
2970 #endif
2971                 return;
2972         }
2973
2974         /* Flush the TX descriptors */
2975
2976         bus_dmamap_sync(sc->rl_ldata.rl_tx_list_tag,
2977             sc->rl_ldata.rl_tx_list_map,
2978             BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD);
2979
2980         CSR_WRITE_1(sc, sc->rl_txstart, RL_TXSTART_START);
2981
2982 #ifdef RE_TX_MODERATION
2983         /*
2984          * Use the countdown timer for interrupt moderation.
2985          * 'TX done' interrupts are disabled. Instead, we reset the
2986          * countdown timer, which will begin counting until it hits
2987          * the value in the TIMERINT register, and then trigger an
2988          * interrupt. Each time we write to the TIMERCNT register,
2989          * the timer count is reset to 0.
2990          */
2991         CSR_WRITE_4(sc, RL_TIMERCNT, 1);
2992 #endif
2993
2994         /*
2995          * Set a timeout in case the chip goes out to lunch.
2996          */
2997         sc->rl_watchdog_timer = 5;
2998 }
2999
3000 static void
3001 re_set_jumbo(struct rl_softc *sc, int jumbo)
3002 {
3003
3004         if (sc->rl_hwrev->rl_rev == RL_HWREV_8168E_VL) {
3005                 pci_set_max_read_req(sc->rl_dev, 4096);
3006                 return;
3007         }
3008
3009         CSR_WRITE_1(sc, RL_EECMD, RL_EEMODE_WRITECFG);
3010         if (jumbo != 0) {
3011                 CSR_WRITE_1(sc, sc->rl_cfg3, CSR_READ_1(sc, sc->rl_cfg3) |
3012                     RL_CFG3_JUMBO_EN0);
3013                 switch (sc->rl_hwrev->rl_rev) {
3014                 case RL_HWREV_8168DP:
3015                         break;
3016                 case RL_HWREV_8168E:
3017                         CSR_WRITE_1(sc, sc->rl_cfg4,
3018                             CSR_READ_1(sc, sc->rl_cfg4) | 0x01);
3019                         break;
3020                 default:
3021                         CSR_WRITE_1(sc, sc->rl_cfg4,
3022                             CSR_READ_1(sc, sc->rl_cfg4) | RL_CFG4_JUMBO_EN1);
3023                 }
3024         } else {
3025                 CSR_WRITE_1(sc, sc->rl_cfg3, CSR_READ_1(sc, sc->rl_cfg3) &
3026                     ~RL_CFG3_JUMBO_EN0);
3027                 switch (sc->rl_hwrev->rl_rev) {
3028                 case RL_HWREV_8168DP:
3029                         break;
3030                 case RL_HWREV_8168E:
3031                         CSR_WRITE_1(sc, sc->rl_cfg4,
3032                             CSR_READ_1(sc, sc->rl_cfg4) & ~0x01);
3033                         break;
3034                 default:
3035                         CSR_WRITE_1(sc, sc->rl_cfg4,
3036                             CSR_READ_1(sc, sc->rl_cfg4) & ~RL_CFG4_JUMBO_EN1);
3037                 }
3038         }
3039         CSR_WRITE_1(sc, RL_EECMD, RL_EEMODE_OFF);
3040
3041         switch (sc->rl_hwrev->rl_rev) {
3042         case RL_HWREV_8168DP:
3043                 pci_set_max_read_req(sc->rl_dev, 4096);
3044                 break;
3045         default:
3046                 if (jumbo != 0)
3047                         pci_set_max_read_req(sc->rl_dev, 512);
3048                 else
3049                         pci_set_max_read_req(sc->rl_dev, 4096);
3050         }
3051 }
3052
3053 static void
3054 re_init(void *xsc)
3055 {
3056         struct rl_softc         *sc = xsc;
3057
3058         RL_LOCK(sc);
3059         re_init_locked(sc);
3060         RL_UNLOCK(sc);
3061 }
3062
3063 static void
3064 re_init_locked(struct rl_softc *sc)
3065 {
3066         struct ifnet            *ifp = sc->rl_ifp;
3067         struct mii_data         *mii;
3068         uint32_t                reg;
3069         uint16_t                cfg;
3070         union {
3071                 uint32_t align_dummy;
3072                 u_char eaddr[ETHER_ADDR_LEN];
3073         } eaddr;
3074
3075         RL_LOCK_ASSERT(sc);
3076
3077         mii = device_get_softc(sc->rl_miibus);
3078
3079         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0)
3080                 return;
3081
3082         /*
3083          * Cancel pending I/O and free all RX/TX buffers.
3084          */
3085         re_stop(sc);
3086
3087         /* Put controller into known state. */
3088         re_reset(sc);
3089
3090         /*
3091          * For C+ mode, initialize the RX descriptors and mbufs.
3092          */
3093         if ((sc->rl_flags & RL_FLAG_JUMBOV2) != 0) {
3094                 if (ifp->if_mtu > RL_MTU) {
3095                         if (re_jrx_list_init(sc) != 0) {
3096                                 device_printf(sc->rl_dev,
3097                                     "no memory for jumbo RX buffers\n");
3098                                 re_stop(sc);
3099                                 return;
3100                         }
3101                         /* Disable checksum offloading for jumbo frames. */
3102                         ifp->if_capenable &= ~(IFCAP_HWCSUM | IFCAP_TSO4);
3103                         ifp->if_hwassist &= ~(RE_CSUM_FEATURES | CSUM_TSO);
3104                 } else {
3105                         if (re_rx_list_init(sc) != 0) {
3106                                 device_printf(sc->rl_dev,
3107                                     "no memory for RX buffers\n");
3108                                 re_stop(sc);
3109                                 return;
3110                         }
3111                 }
3112                 re_set_jumbo(sc, ifp->if_mtu > RL_MTU);
3113         } else {
3114                 if (re_rx_list_init(sc) != 0) {
3115                         device_printf(sc->rl_dev, "no memory for RX buffers\n");
3116                         re_stop(sc);
3117                         return;
3118                 }
3119                 if ((sc->rl_flags & RL_FLAG_PCIE) != 0 &&
3120                     pci_get_device(sc->rl_dev) != RT_DEVICEID_8101E) {
3121                         if (ifp->if_mtu > RL_MTU)
3122                                 pci_set_max_read_req(sc->rl_dev, 512);
3123                         else
3124                                 pci_set_max_read_req(sc->rl_dev, 4096);
3125                 }
3126         }
3127         re_tx_list_init(sc);
3128
3129         /*
3130          * Enable C+ RX and TX mode, as well as VLAN stripping and
3131          * RX checksum offload. We must configure the C+ register
3132          * before all others.
3133          */
3134         cfg = RL_CPLUSCMD_PCI_MRW;
3135         if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
3136                 cfg |= RL_CPLUSCMD_RXCSUM_ENB;
3137         if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0)
3138                 cfg |= RL_CPLUSCMD_VLANSTRIP;
3139         if ((sc->rl_flags & RL_FLAG_MACSTAT) != 0) {
3140                 cfg |= RL_CPLUSCMD_MACSTAT_DIS;
3141                 /* XXX magic. */
3142                 cfg |= 0x0001;
3143         } else
3144                 cfg |= RL_CPLUSCMD_RXENB | RL_CPLUSCMD_TXENB;
3145         CSR_WRITE_2(sc, RL_CPLUS_CMD, cfg);
3146         if (sc->rl_hwrev->rl_rev == RL_HWREV_8169_8110SC ||
3147             sc->rl_hwrev->rl_rev == RL_HWREV_8169_8110SCE) {
3148                 reg = 0x000fff00;
3149                 if ((CSR_READ_1(sc, sc->rl_cfg2) & RL_CFG2_PCI66MHZ) != 0)
3150                         reg |= 0x000000ff;
3151                 if (sc->rl_hwrev->rl_rev == RL_HWREV_8169_8110SCE)
3152                         reg |= 0x00f00000;
3153                 CSR_WRITE_4(sc, 0x7c, reg);
3154                 /* Disable interrupt mitigation. */
3155                 CSR_WRITE_2(sc, 0xe2, 0);
3156         }
3157         /*
3158          * Disable TSO if interface MTU size is greater than MSS
3159          * allowed in controller.
3160          */
3161         if (ifp->if_mtu > RL_TSO_MTU && (ifp->if_capenable & IFCAP_TSO4) != 0) {
3162                 ifp->if_capenable &= ~IFCAP_TSO4;
3163                 ifp->if_hwassist &= ~CSUM_TSO;
3164         }
3165
3166         /*
3167          * Init our MAC address.  Even though the chipset
3168          * documentation doesn't mention it, we need to enter "Config
3169          * register write enable" mode to modify the ID registers.
3170          */
3171         /* Copy MAC address on stack to align. */
3172         bcopy(IF_LLADDR(ifp), eaddr.eaddr, ETHER_ADDR_LEN);
3173         CSR_WRITE_1(sc, RL_EECMD, RL_EEMODE_WRITECFG);
3174         CSR_WRITE_4(sc, RL_IDR0,
3175             htole32(*(u_int32_t *)(&eaddr.eaddr[0])));
3176         CSR_WRITE_4(sc, RL_IDR4,
3177             htole32(*(u_int32_t *)(&eaddr.eaddr[4])));
3178         CSR_WRITE_1(sc, RL_EECMD, RL_EEMODE_OFF);
3179
3180         /*
3181          * Load the addresses of the RX and TX lists into the chip.
3182          */
3183
3184         CSR_WRITE_4(sc, RL_RXLIST_ADDR_HI,
3185             RL_ADDR_HI(sc->rl_ldata.rl_rx_list_addr));
3186         CSR_WRITE_4(sc, RL_RXLIST_ADDR_LO,
3187             RL_ADDR_LO(sc->rl_ldata.rl_rx_list_addr));
3188
3189         CSR_WRITE_4(sc, RL_TXLIST_ADDR_HI,
3190             RL_ADDR_HI(sc->rl_ldata.rl_tx_list_addr));
3191         CSR_WRITE_4(sc, RL_TXLIST_ADDR_LO,
3192             RL_ADDR_LO(sc->rl_ldata.rl_tx_list_addr));
3193
3194         if ((sc->rl_flags & RL_FLAG_RXDV_GATED) != 0)
3195                 CSR_WRITE_4(sc, RL_MISC, CSR_READ_4(sc, RL_MISC) &
3196                     ~0x00080000);
3197
3198         /*
3199          * Set the initial TX configuration.
3200          */
3201         if (sc->rl_testmode) {
3202                 if (sc->rl_type == RL_8169)
3203                         CSR_WRITE_4(sc, RL_TXCFG,
3204                             RL_TXCFG_CONFIG|RL_LOOPTEST_ON);
3205                 else
3206                         CSR_WRITE_4(sc, RL_TXCFG,
3207                             RL_TXCFG_CONFIG|RL_LOOPTEST_ON_CPLUS);
3208         } else
3209                 CSR_WRITE_4(sc, RL_TXCFG, RL_TXCFG_CONFIG);
3210
3211         CSR_WRITE_1(sc, RL_EARLY_TX_THRESH, 16);
3212
3213         /*
3214          * Set the initial RX configuration.
3215          */
3216         re_set_rxmode(sc);
3217
3218         /* Configure interrupt moderation. */
3219         if (sc->rl_type == RL_8169) {
3220                 /* Magic from vendor. */
3221                 CSR_WRITE_2(sc, RL_INTRMOD, 0x5100);
3222         }
3223
3224         /*
3225          * Enable transmit and receive.
3226          */
3227         CSR_WRITE_1(sc, RL_COMMAND, RL_CMD_TX_ENB | RL_CMD_RX_ENB);
3228
3229 #ifdef DEVICE_POLLING
3230         /*
3231          * Disable interrupts if we are polling.
3232          */
3233         if (ifp->if_capenable & IFCAP_POLLING)
3234                 CSR_WRITE_2(sc, RL_IMR, 0);
3235         else    /* otherwise ... */
3236 #endif
3237
3238         /*
3239          * Enable interrupts.
3240          */
3241         if (sc->rl_testmode)
3242                 CSR_WRITE_2(sc, RL_IMR, 0);
3243         else
3244                 CSR_WRITE_2(sc, RL_IMR, RL_INTRS_CPLUS);
3245         CSR_WRITE_2(sc, RL_ISR, RL_INTRS_CPLUS);
3246
3247         /* Set initial TX threshold */
3248         sc->rl_txthresh = RL_TX_THRESH_INIT;
3249
3250         /* Start RX/TX process. */
3251         CSR_WRITE_4(sc, RL_MISSEDPKT, 0);
3252
3253         /*
3254          * Initialize the timer interrupt register so that
3255          * a timer interrupt will be generated once the timer
3256          * reaches a certain number of ticks. The timer is
3257          * reloaded on each transmit.
3258          */
3259 #ifdef RE_TX_MODERATION
3260         /*
3261          * Use timer interrupt register to moderate TX interrupt
3262          * moderation, which dramatically improves TX frame rate.
3263          */
3264         if (sc->rl_type == RL_8169)
3265                 CSR_WRITE_4(sc, RL_TIMERINT_8169, 0x800);
3266         else
3267                 CSR_WRITE_4(sc, RL_TIMERINT, 0x400);
3268 #else
3269         /*
3270          * Use timer interrupt register to moderate RX interrupt
3271          * moderation.
3272          */
3273         if ((sc->rl_flags & (RL_FLAG_MSI | RL_FLAG_MSIX)) != 0 &&
3274             intr_filter == 0) {
3275                 if (sc->rl_type == RL_8169)
3276                         CSR_WRITE_4(sc, RL_TIMERINT_8169,
3277                             RL_USECS(sc->rl_int_rx_mod));
3278         } else {
3279                 if (sc->rl_type == RL_8169)
3280                         CSR_WRITE_4(sc, RL_TIMERINT_8169, RL_USECS(0));
3281         }
3282 #endif
3283
3284         /*
3285          * For 8169 gigE NICs, set the max allowed RX packet
3286          * size so we can receive jumbo frames.
3287          */
3288         if (sc->rl_type == RL_8169) {
3289                 if ((sc->rl_flags & RL_FLAG_JUMBOV2) != 0) {
3290                         /*
3291                          * For controllers that use new jumbo frame scheme,
3292                          * set maximum size of jumbo frame depending on
3293                          * controller revisions.
3294                          */
3295                         if (ifp->if_mtu > RL_MTU)
3296                                 CSR_WRITE_2(sc, RL_MAXRXPKTLEN,
3297                                     sc->rl_hwrev->rl_max_mtu +
3298                                     ETHER_VLAN_ENCAP_LEN + ETHER_HDR_LEN +
3299                                     ETHER_CRC_LEN);
3300                         else
3301                                 CSR_WRITE_2(sc, RL_MAXRXPKTLEN,
3302                                     RE_RX_DESC_BUFLEN);
3303                 } else if ((sc->rl_flags & RL_FLAG_PCIE) != 0 &&
3304                     sc->rl_hwrev->rl_max_mtu == RL_MTU) {
3305                         /* RTL810x has no jumbo frame support. */
3306                         CSR_WRITE_2(sc, RL_MAXRXPKTLEN, RE_RX_DESC_BUFLEN);
3307                 } else
3308                         CSR_WRITE_2(sc, RL_MAXRXPKTLEN, 16383);
3309         }
3310
3311         if (sc->rl_testmode)
3312                 return;
3313
3314         CSR_WRITE_1(sc, sc->rl_cfg1, CSR_READ_1(sc, sc->rl_cfg1) |
3315             RL_CFG1_DRVLOAD);
3316
3317         ifp->if_drv_flags |= IFF_DRV_RUNNING;
3318         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
3319
3320         sc->rl_flags &= ~RL_FLAG_LINK;
3321         mii_mediachg(mii);
3322
3323         sc->rl_watchdog_timer = 0;
3324         callout_reset(&sc->rl_stat_callout, hz, re_tick, sc);
3325 }
3326
3327 /*
3328  * Set media options.
3329  */
3330 static int
3331 re_ifmedia_upd(struct ifnet *ifp)
3332 {
3333         struct rl_softc         *sc;
3334         struct mii_data         *mii;
3335         int                     error;
3336
3337         sc = ifp->if_softc;
3338         mii = device_get_softc(sc->rl_miibus);
3339         RL_LOCK(sc);
3340         error = mii_mediachg(mii);
3341         RL_UNLOCK(sc);
3342
3343         return (error);
3344 }
3345
3346 /*
3347  * Report current media status.
3348  */
3349 static void
3350 re_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
3351 {
3352         struct rl_softc         *sc;
3353         struct mii_data         *mii;
3354
3355         sc = ifp->if_softc;
3356         mii = device_get_softc(sc->rl_miibus);
3357
3358         RL_LOCK(sc);
3359         mii_pollstat(mii);
3360         ifmr->ifm_active = mii->mii_media_active;
3361         ifmr->ifm_status = mii->mii_media_status;
3362         RL_UNLOCK(sc);
3363 }
3364
3365 static int
3366 re_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
3367 {
3368         struct rl_softc         *sc = ifp->if_softc;
3369         struct ifreq            *ifr = (struct ifreq *) data;
3370         struct mii_data         *mii;
3371         int                     error = 0;
3372
3373         switch (command) {
3374         case SIOCSIFMTU:
3375                 if (ifr->ifr_mtu < ETHERMIN ||
3376                     ifr->ifr_mtu > sc->rl_hwrev->rl_max_mtu ||
3377                     ((sc->rl_flags & RL_FLAG_FASTETHER) != 0 &&
3378                     ifr->ifr_mtu > RL_MTU)) {
3379                         error = EINVAL;
3380                         break;
3381                 }
3382                 RL_LOCK(sc);
3383                 if (ifp->if_mtu != ifr->ifr_mtu) {
3384                         ifp->if_mtu = ifr->ifr_mtu;
3385                         if ((sc->rl_flags & RL_FLAG_JUMBOV2) != 0 &&
3386                             (ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) {
3387                                 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3388                                 re_init_locked(sc);
3389                         }
3390                         if (ifp->if_mtu > RL_TSO_MTU &&
3391                             (ifp->if_capenable & IFCAP_TSO4) != 0) {
3392                                 ifp->if_capenable &= ~(IFCAP_TSO4 |
3393                                     IFCAP_VLAN_HWTSO);
3394                                 ifp->if_hwassist &= ~CSUM_TSO;
3395                         }
3396                         VLAN_CAPABILITIES(ifp);
3397                 }
3398                 RL_UNLOCK(sc);
3399                 break;
3400         case SIOCSIFFLAGS:
3401                 RL_LOCK(sc);
3402                 if ((ifp->if_flags & IFF_UP) != 0) {
3403                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) {
3404                                 if (((ifp->if_flags ^ sc->rl_if_flags)
3405                                     & (IFF_PROMISC | IFF_ALLMULTI)) != 0)
3406                                         re_set_rxmode(sc);
3407                         } else
3408                                 re_init_locked(sc);
3409                 } else {
3410                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0)
3411                                 re_stop(sc);
3412                 }
3413                 sc->rl_if_flags = ifp->if_flags;
3414                 RL_UNLOCK(sc);
3415                 break;
3416         case SIOCADDMULTI:
3417         case SIOCDELMULTI:
3418                 RL_LOCK(sc);
3419                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0)
3420                         re_set_rxmode(sc);
3421                 RL_UNLOCK(sc);
3422                 break;
3423         case SIOCGIFMEDIA:
3424         case SIOCSIFMEDIA:
3425                 mii = device_get_softc(sc->rl_miibus);
3426                 error = ifmedia_ioctl(ifp, ifr, &mii->mii_media, command);
3427                 break;
3428         case SIOCSIFCAP:
3429             {
3430                 int mask, reinit;
3431
3432                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
3433                 reinit = 0;
3434 #ifdef DEVICE_POLLING
3435                 if (mask & IFCAP_POLLING) {
3436                         if (ifr->ifr_reqcap & IFCAP_POLLING) {
3437                                 error = ether_poll_register(re_poll, ifp);
3438                                 if (error)
3439                                         return (error);
3440                                 RL_LOCK(sc);
3441                                 /* Disable interrupts */
3442                                 CSR_WRITE_2(sc, RL_IMR, 0x0000);
3443                                 ifp->if_capenable |= IFCAP_POLLING;
3444                                 RL_UNLOCK(sc);
3445                         } else {
3446                                 error = ether_poll_deregister(ifp);
3447                                 /* Enable interrupts. */
3448                                 RL_LOCK(sc);
3449                                 CSR_WRITE_2(sc, RL_IMR, RL_INTRS_CPLUS);
3450                                 ifp->if_capenable &= ~IFCAP_POLLING;
3451                                 RL_UNLOCK(sc);
3452                         }
3453                 }
3454 #endif /* DEVICE_POLLING */
3455                 RL_LOCK(sc);
3456                 if ((mask & IFCAP_TXCSUM) != 0 &&
3457                     (ifp->if_capabilities & IFCAP_TXCSUM) != 0) {
3458                         ifp->if_capenable ^= IFCAP_TXCSUM;
3459                         if ((ifp->if_capenable & IFCAP_TXCSUM) != 0)
3460                                 ifp->if_hwassist |= RE_CSUM_FEATURES;
3461                         else
3462                                 ifp->if_hwassist &= ~RE_CSUM_FEATURES;
3463                         reinit = 1;
3464                 }
3465                 if ((mask & IFCAP_RXCSUM) != 0 &&
3466                     (ifp->if_capabilities & IFCAP_RXCSUM) != 0) {
3467                         ifp->if_capenable ^= IFCAP_RXCSUM;
3468                         reinit = 1;
3469                 }
3470                 if ((mask & IFCAP_TSO4) != 0 &&
3471                     (ifp->if_capabilities & IFCAP_TSO4) != 0) {
3472                         ifp->if_capenable ^= IFCAP_TSO4;
3473                         if ((IFCAP_TSO4 & ifp->if_capenable) != 0)
3474                                 ifp->if_hwassist |= CSUM_TSO;
3475                         else
3476                                 ifp->if_hwassist &= ~CSUM_TSO;
3477                         if (ifp->if_mtu > RL_TSO_MTU &&
3478                             (ifp->if_capenable & IFCAP_TSO4) != 0) {
3479                                 ifp->if_capenable &= ~IFCAP_TSO4;
3480                                 ifp->if_hwassist &= ~CSUM_TSO;
3481                         }
3482                 }
3483                 if ((mask & IFCAP_VLAN_HWTSO) != 0 &&
3484                     (ifp->if_capabilities & IFCAP_VLAN_HWTSO) != 0)
3485                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
3486                 if ((mask & IFCAP_VLAN_HWTAGGING) != 0 &&
3487                     (ifp->if_capabilities & IFCAP_VLAN_HWTAGGING) != 0) {
3488                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
3489                         /* TSO over VLAN requires VLAN hardware tagging. */
3490                         if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) == 0)
3491                                 ifp->if_capenable &= ~IFCAP_VLAN_HWTSO;
3492                         reinit = 1;
3493                 }
3494                 if ((sc->rl_flags & RL_FLAG_JUMBOV2) != 0 &&
3495                     (mask & (IFCAP_HWCSUM | IFCAP_TSO4 |
3496                     IFCAP_VLAN_HWTSO)) != 0)
3497                                 reinit = 1;
3498                 if ((mask & IFCAP_WOL) != 0 &&
3499                     (ifp->if_capabilities & IFCAP_WOL) != 0) {
3500                         if ((mask & IFCAP_WOL_UCAST) != 0)
3501                                 ifp->if_capenable ^= IFCAP_WOL_UCAST;
3502                         if ((mask & IFCAP_WOL_MCAST) != 0)
3503                                 ifp->if_capenable ^= IFCAP_WOL_MCAST;
3504                         if ((mask & IFCAP_WOL_MAGIC) != 0)
3505                                 ifp->if_capenable ^= IFCAP_WOL_MAGIC;
3506                 }
3507                 if (reinit && ifp->if_drv_flags & IFF_DRV_RUNNING) {
3508                         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3509                         re_init_locked(sc);
3510                 }
3511                 RL_UNLOCK(sc);
3512                 VLAN_CAPABILITIES(ifp);
3513             }
3514                 break;
3515         default:
3516                 error = ether_ioctl(ifp, command, data);
3517                 break;
3518         }
3519
3520         return (error);
3521 }
3522
3523 static void
3524 re_watchdog(struct rl_softc *sc)
3525 {
3526         struct ifnet            *ifp;
3527
3528         RL_LOCK_ASSERT(sc);
3529
3530         if (sc->rl_watchdog_timer == 0 || --sc->rl_watchdog_timer != 0)
3531                 return;
3532
3533         ifp = sc->rl_ifp;
3534         re_txeof(sc);
3535         if (sc->rl_ldata.rl_tx_free == sc->rl_ldata.rl_tx_desc_cnt) {
3536                 if_printf(ifp, "watchdog timeout (missed Tx interrupts) "
3537                     "-- recovering\n");
3538                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
3539                         re_start_locked(ifp);
3540                 return;
3541         }
3542
3543         if_printf(ifp, "watchdog timeout\n");
3544         if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
3545
3546         re_rxeof(sc, NULL);
3547         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3548         re_init_locked(sc);
3549         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
3550                 re_start_locked(ifp);
3551 }
3552
3553 /*
3554  * Stop the adapter and free any mbufs allocated to the
3555  * RX and TX lists.
3556  */
3557 static void
3558 re_stop(struct rl_softc *sc)
3559 {
3560         int                     i;
3561         struct ifnet            *ifp;
3562         struct rl_txdesc        *txd;
3563         struct rl_rxdesc        *rxd;
3564
3565         RL_LOCK_ASSERT(sc);
3566
3567         ifp = sc->rl_ifp;
3568
3569         sc->rl_watchdog_timer = 0;
3570         callout_stop(&sc->rl_stat_callout);
3571         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
3572
3573         /*
3574          * Disable accepting frames to put RX MAC into idle state.
3575          * Otherwise it's possible to get frames while stop command
3576          * execution is in progress and controller can DMA the frame
3577          * to already freed RX buffer during that period.
3578          */
3579         CSR_WRITE_4(sc, RL_RXCFG, CSR_READ_4(sc, RL_RXCFG) &
3580             ~(RL_RXCFG_RX_ALLPHYS | RL_RXCFG_RX_INDIV | RL_RXCFG_RX_MULTI |
3581             RL_RXCFG_RX_BROAD));
3582
3583         if ((sc->rl_flags & RL_FLAG_WAIT_TXPOLL) != 0) {
3584                 for (i = RL_TIMEOUT; i > 0; i--) {
3585                         if ((CSR_READ_1(sc, sc->rl_txstart) &
3586                             RL_TXSTART_START) == 0)
3587                                 break;
3588                         DELAY(20);
3589                 }
3590                 if (i == 0)
3591                         device_printf(sc->rl_dev,
3592                             "stopping TX poll timed out!\n");
3593                 CSR_WRITE_1(sc, RL_COMMAND, 0x00);
3594         } else if ((sc->rl_flags & RL_FLAG_CMDSTOP) != 0) {
3595                 CSR_WRITE_1(sc, RL_COMMAND, RL_CMD_STOPREQ | RL_CMD_TX_ENB |
3596                     RL_CMD_RX_ENB);
3597                 if ((sc->rl_flags & RL_FLAG_CMDSTOP_WAIT_TXQ) != 0) {
3598                         for (i = RL_TIMEOUT; i > 0; i--) {
3599                                 if ((CSR_READ_4(sc, RL_TXCFG) &
3600                                     RL_TXCFG_QUEUE_EMPTY) != 0)
3601                                         break;
3602                                 DELAY(100);
3603                         }
3604                         if (i == 0)
3605                                 device_printf(sc->rl_dev,
3606                                    "stopping TXQ timed out!\n");
3607                 }
3608         } else
3609                 CSR_WRITE_1(sc, RL_COMMAND, 0x00);
3610         DELAY(1000);
3611         CSR_WRITE_2(sc, RL_IMR, 0x0000);
3612         CSR_WRITE_2(sc, RL_ISR, 0xFFFF);
3613
3614         if (sc->rl_head != NULL) {
3615                 m_freem(sc->rl_head);
3616                 sc->rl_head = sc->rl_tail = NULL;
3617         }
3618
3619         /* Free the TX list buffers. */
3620         for (i = 0; i < sc->rl_ldata.rl_tx_desc_cnt; i++) {
3621                 txd = &sc->rl_ldata.rl_tx_desc[i];
3622                 if (txd->tx_m != NULL) {
3623                         bus_dmamap_sync(sc->rl_ldata.rl_tx_mtag,
3624                             txd->tx_dmamap, BUS_DMASYNC_POSTWRITE);
3625                         bus_dmamap_unload(sc->rl_ldata.rl_tx_mtag,
3626                             txd->tx_dmamap);
3627                         m_freem(txd->tx_m);
3628                         txd->tx_m = NULL;
3629                 }
3630         }
3631
3632         /* Free the RX list buffers. */
3633         for (i = 0; i < sc->rl_ldata.rl_rx_desc_cnt; i++) {
3634                 rxd = &sc->rl_ldata.rl_rx_desc[i];
3635                 if (rxd->rx_m != NULL) {
3636                         bus_dmamap_sync(sc->rl_ldata.rl_rx_mtag,
3637                             rxd->rx_dmamap, BUS_DMASYNC_POSTREAD);
3638                         bus_dmamap_unload(sc->rl_ldata.rl_rx_mtag,
3639                             rxd->rx_dmamap);
3640                         m_freem(rxd->rx_m);
3641                         rxd->rx_m = NULL;
3642                 }
3643         }
3644
3645         if ((sc->rl_flags & RL_FLAG_JUMBOV2) != 0) {
3646                 for (i = 0; i < sc->rl_ldata.rl_rx_desc_cnt; i++) {
3647                         rxd = &sc->rl_ldata.rl_jrx_desc[i];
3648                         if (rxd->rx_m != NULL) {
3649                                 bus_dmamap_sync(sc->rl_ldata.rl_jrx_mtag,
3650                                     rxd->rx_dmamap, BUS_DMASYNC_POSTREAD);
3651                                 bus_dmamap_unload(sc->rl_ldata.rl_jrx_mtag,
3652                                     rxd->rx_dmamap);
3653                                 m_freem(rxd->rx_m);
3654                                 rxd->rx_m = NULL;
3655                         }
3656                 }
3657         }
3658 }
3659
3660 /*
3661  * Device suspend routine.  Stop the interface and save some PCI
3662  * settings in case the BIOS doesn't restore them properly on
3663  * resume.
3664  */
3665 static int
3666 re_suspend(device_t dev)
3667 {
3668         struct rl_softc         *sc;
3669
3670         sc = device_get_softc(dev);
3671
3672         RL_LOCK(sc);
3673         re_stop(sc);
3674         re_setwol(sc);
3675         sc->suspended = 1;
3676         RL_UNLOCK(sc);
3677
3678         return (0);
3679 }
3680
3681 /*
3682  * Device resume routine.  Restore some PCI settings in case the BIOS
3683  * doesn't, re-enable busmastering, and restart the interface if
3684  * appropriate.
3685  */
3686 static int
3687 re_resume(device_t dev)
3688 {
3689         struct rl_softc         *sc;
3690         struct ifnet            *ifp;
3691
3692         sc = device_get_softc(dev);
3693
3694         RL_LOCK(sc);
3695
3696         ifp = sc->rl_ifp;
3697         /* Take controller out of sleep mode. */
3698         if ((sc->rl_flags & RL_FLAG_MACSLEEP) != 0) {
3699                 if ((CSR_READ_1(sc, RL_MACDBG) & 0x80) == 0x80)
3700                         CSR_WRITE_1(sc, RL_GPIO,
3701                             CSR_READ_1(sc, RL_GPIO) | 0x01);
3702         }
3703
3704         /*
3705          * Clear WOL matching such that normal Rx filtering
3706          * wouldn't interfere with WOL patterns.
3707          */
3708         re_clrwol(sc);
3709
3710         /* reinitialize interface if necessary */
3711         if (ifp->if_flags & IFF_UP)
3712                 re_init_locked(sc);
3713
3714         sc->suspended = 0;
3715         RL_UNLOCK(sc);
3716
3717         return (0);
3718 }
3719
3720 /*
3721  * Stop all chip I/O so that the kernel's probe routines don't
3722  * get confused by errant DMAs when rebooting.
3723  */
3724 static int
3725 re_shutdown(device_t dev)
3726 {
3727         struct rl_softc         *sc;
3728
3729         sc = device_get_softc(dev);
3730
3731         RL_LOCK(sc);
3732         re_stop(sc);
3733         /*
3734          * Mark interface as down since otherwise we will panic if
3735          * interrupt comes in later on, which can happen in some
3736          * cases.
3737          */
3738         sc->rl_ifp->if_flags &= ~IFF_UP;
3739         re_setwol(sc);
3740         RL_UNLOCK(sc);
3741
3742         return (0);
3743 }
3744
3745 static void
3746 re_set_linkspeed(struct rl_softc *sc)
3747 {
3748         struct mii_softc *miisc;
3749         struct mii_data *mii;
3750         int aneg, i, phyno;
3751
3752         RL_LOCK_ASSERT(sc);
3753
3754         mii = device_get_softc(sc->rl_miibus);
3755         mii_pollstat(mii);
3756         aneg = 0;
3757         if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) ==
3758             (IFM_ACTIVE | IFM_AVALID)) {
3759                 switch IFM_SUBTYPE(mii->mii_media_active) {
3760                 case IFM_10_T:
3761                 case IFM_100_TX:
3762                         return;
3763                 case IFM_1000_T:
3764                         aneg++;
3765                         break;
3766                 default:
3767                         break;
3768                 }
3769         }
3770         miisc = LIST_FIRST(&mii->mii_phys);
3771         phyno = miisc->mii_phy;
3772         LIST_FOREACH(miisc, &mii->mii_phys, mii_list)
3773                 PHY_RESET(miisc);
3774         re_miibus_writereg(sc->rl_dev, phyno, MII_100T2CR, 0);
3775         re_miibus_writereg(sc->rl_dev, phyno,
3776             MII_ANAR, ANAR_TX_FD | ANAR_TX | ANAR_10_FD | ANAR_10 | ANAR_CSMA);
3777         re_miibus_writereg(sc->rl_dev, phyno,
3778             MII_BMCR, BMCR_AUTOEN | BMCR_STARTNEG);
3779         DELAY(1000);
3780         if (aneg != 0) {
3781                 /*
3782                  * Poll link state until re(4) get a 10/100Mbps link.
3783                  */
3784                 for (i = 0; i < MII_ANEGTICKS_GIGE; i++) {
3785                         mii_pollstat(mii);
3786                         if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID))
3787                             == (IFM_ACTIVE | IFM_AVALID)) {
3788                                 switch (IFM_SUBTYPE(mii->mii_media_active)) {
3789                                 case IFM_10_T:
3790                                 case IFM_100_TX:
3791                                         return;
3792                                 default:
3793                                         break;
3794                                 }
3795                         }
3796                         RL_UNLOCK(sc);
3797                         pause("relnk", hz);
3798                         RL_LOCK(sc);
3799                 }
3800                 if (i == MII_ANEGTICKS_GIGE)
3801                         device_printf(sc->rl_dev,
3802                             "establishing a link failed, WOL may not work!");
3803         }
3804         /*
3805          * No link, force MAC to have 100Mbps, full-duplex link.
3806          * MAC does not require reprogramming on resolved speed/duplex,
3807          * so this is just for completeness.
3808          */
3809         mii->mii_media_status = IFM_AVALID | IFM_ACTIVE;
3810         mii->mii_media_active = IFM_ETHER | IFM_100_TX | IFM_FDX;
3811 }
3812
3813 static void
3814 re_setwol(struct rl_softc *sc)
3815 {
3816         struct ifnet            *ifp;
3817         int                     pmc;
3818         uint16_t                pmstat;
3819         uint8_t                 v;
3820
3821         RL_LOCK_ASSERT(sc);
3822
3823         if (pci_find_cap(sc->rl_dev, PCIY_PMG, &pmc) != 0)
3824                 return;
3825
3826         ifp = sc->rl_ifp;
3827         /* Put controller into sleep mode. */
3828         if ((sc->rl_flags & RL_FLAG_MACSLEEP) != 0) {
3829                 if ((CSR_READ_1(sc, RL_MACDBG) & 0x80) == 0x80)
3830                         CSR_WRITE_1(sc, RL_GPIO,
3831                             CSR_READ_1(sc, RL_GPIO) & ~0x01);
3832         }
3833         if ((ifp->if_capenable & IFCAP_WOL) != 0) {
3834                 re_set_rxmode(sc);
3835                 if ((sc->rl_flags & RL_FLAG_WOL_MANLINK) != 0)
3836                         re_set_linkspeed(sc);
3837                 if ((sc->rl_flags & RL_FLAG_WOLRXENB) != 0)
3838                         CSR_WRITE_1(sc, RL_COMMAND, RL_CMD_RX_ENB);
3839         }
3840         /* Enable config register write. */
3841         CSR_WRITE_1(sc, RL_EECMD, RL_EE_MODE);
3842
3843         /* Enable PME. */
3844         v = CSR_READ_1(sc, sc->rl_cfg1);
3845         v &= ~RL_CFG1_PME;
3846         if ((ifp->if_capenable & IFCAP_WOL) != 0)
3847                 v |= RL_CFG1_PME;
3848         CSR_WRITE_1(sc, sc->rl_cfg1, v);
3849
3850         v = CSR_READ_1(sc, sc->rl_cfg3);
3851         v &= ~(RL_CFG3_WOL_LINK | RL_CFG3_WOL_MAGIC);
3852         if ((ifp->if_capenable & IFCAP_WOL_MAGIC) != 0)
3853                 v |= RL_CFG3_WOL_MAGIC;
3854         CSR_WRITE_1(sc, sc->rl_cfg3, v);
3855
3856         v = CSR_READ_1(sc, sc->rl_cfg5);
3857         v &= ~(RL_CFG5_WOL_BCAST | RL_CFG5_WOL_MCAST | RL_CFG5_WOL_UCAST |
3858             RL_CFG5_WOL_LANWAKE);
3859         if ((ifp->if_capenable & IFCAP_WOL_UCAST) != 0)
3860                 v |= RL_CFG5_WOL_UCAST;
3861         if ((ifp->if_capenable & IFCAP_WOL_MCAST) != 0)
3862                 v |= RL_CFG5_WOL_MCAST | RL_CFG5_WOL_BCAST;
3863         if ((ifp->if_capenable & IFCAP_WOL) != 0)
3864                 v |= RL_CFG5_WOL_LANWAKE;
3865         CSR_WRITE_1(sc, sc->rl_cfg5, v);
3866
3867         /* Config register write done. */
3868         CSR_WRITE_1(sc, RL_EECMD, RL_EEMODE_OFF);
3869
3870         if ((ifp->if_capenable & IFCAP_WOL) == 0 &&
3871             (sc->rl_flags & RL_FLAG_PHYWAKE_PM) != 0)
3872                 CSR_WRITE_1(sc, RL_PMCH, CSR_READ_1(sc, RL_PMCH) & ~0x80);
3873         /*
3874          * It seems that hardware resets its link speed to 100Mbps in
3875          * power down mode so switching to 100Mbps in driver is not
3876          * needed.
3877          */
3878
3879         /* Request PME if WOL is requested. */
3880         pmstat = pci_read_config(sc->rl_dev, pmc + PCIR_POWER_STATUS, 2);
3881         pmstat &= ~(PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE);
3882         if ((ifp->if_capenable & IFCAP_WOL) != 0)
3883                 pmstat |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
3884         pci_write_config(sc->rl_dev, pmc + PCIR_POWER_STATUS, pmstat, 2);
3885 }
3886
3887 static void
3888 re_clrwol(struct rl_softc *sc)
3889 {
3890         int                     pmc;
3891         uint8_t                 v;
3892
3893         RL_LOCK_ASSERT(sc);
3894
3895         if (pci_find_cap(sc->rl_dev, PCIY_PMG, &pmc) != 0)
3896                 return;
3897
3898         /* Enable config register write. */
3899         CSR_WRITE_1(sc, RL_EECMD, RL_EE_MODE);
3900
3901         v = CSR_READ_1(sc, sc->rl_cfg3);
3902         v &= ~(RL_CFG3_WOL_LINK | RL_CFG3_WOL_MAGIC);
3903         CSR_WRITE_1(sc, sc->rl_cfg3, v);
3904
3905         /* Config register write done. */
3906         CSR_WRITE_1(sc, RL_EECMD, RL_EEMODE_OFF);
3907
3908         v = CSR_READ_1(sc, sc->rl_cfg5);
3909         v &= ~(RL_CFG5_WOL_BCAST | RL_CFG5_WOL_MCAST | RL_CFG5_WOL_UCAST);
3910         v &= ~RL_CFG5_WOL_LANWAKE;
3911         CSR_WRITE_1(sc, sc->rl_cfg5, v);
3912 }
3913
3914 static void
3915 re_add_sysctls(struct rl_softc *sc)
3916 {
3917         struct sysctl_ctx_list  *ctx;
3918         struct sysctl_oid_list  *children;
3919         int                     error;
3920
3921         ctx = device_get_sysctl_ctx(sc->rl_dev);
3922         children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->rl_dev));
3923
3924         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "stats",
3925             CTLTYPE_INT | CTLFLAG_RW, sc, 0, re_sysctl_stats, "I",
3926             "Statistics Information");
3927         if ((sc->rl_flags & (RL_FLAG_MSI | RL_FLAG_MSIX)) == 0)
3928                 return;
3929
3930         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "int_rx_mod",
3931             CTLTYPE_INT | CTLFLAG_RW, &sc->rl_int_rx_mod, 0,
3932             sysctl_hw_re_int_mod, "I", "re RX interrupt moderation");
3933         /* Pull in device tunables. */
3934         sc->rl_int_rx_mod = RL_TIMER_DEFAULT;
3935         error = resource_int_value(device_get_name(sc->rl_dev),
3936             device_get_unit(sc->rl_dev), "int_rx_mod", &sc->rl_int_rx_mod);
3937         if (error == 0) {
3938                 if (sc->rl_int_rx_mod < RL_TIMER_MIN ||
3939                     sc->rl_int_rx_mod > RL_TIMER_MAX) {
3940                         device_printf(sc->rl_dev, "int_rx_mod value out of "
3941                             "range; using default: %d\n",
3942                             RL_TIMER_DEFAULT);
3943                         sc->rl_int_rx_mod = RL_TIMER_DEFAULT;
3944                 }
3945         }
3946
3947 }
3948
3949 static int
3950 re_sysctl_stats(SYSCTL_HANDLER_ARGS)
3951 {
3952         struct rl_softc         *sc;
3953         struct rl_stats         *stats;
3954         int                     error, i, result;
3955
3956         result = -1;
3957         error = sysctl_handle_int(oidp, &result, 0, req);
3958         if (error || req->newptr == NULL)
3959                 return (error);
3960
3961         if (result == 1) {
3962                 sc = (struct rl_softc *)arg1;
3963                 RL_LOCK(sc);
3964                 if ((sc->rl_ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
3965                         RL_UNLOCK(sc);
3966                         goto done;
3967                 }
3968                 bus_dmamap_sync(sc->rl_ldata.rl_stag,
3969                     sc->rl_ldata.rl_smap, BUS_DMASYNC_PREREAD);
3970                 CSR_WRITE_4(sc, RL_DUMPSTATS_HI,
3971                     RL_ADDR_HI(sc->rl_ldata.rl_stats_addr));
3972                 CSR_WRITE_4(sc, RL_DUMPSTATS_LO,
3973                     RL_ADDR_LO(sc->rl_ldata.rl_stats_addr));
3974                 CSR_WRITE_4(sc, RL_DUMPSTATS_LO,
3975                     RL_ADDR_LO(sc->rl_ldata.rl_stats_addr |
3976                     RL_DUMPSTATS_START));
3977                 for (i = RL_TIMEOUT; i > 0; i--) {
3978                         if ((CSR_READ_4(sc, RL_DUMPSTATS_LO) &
3979                             RL_DUMPSTATS_START) == 0)
3980                                 break;
3981                         DELAY(1000);
3982                 }
3983                 bus_dmamap_sync(sc->rl_ldata.rl_stag,
3984                     sc->rl_ldata.rl_smap, BUS_DMASYNC_POSTREAD);
3985                 RL_UNLOCK(sc);
3986                 if (i == 0) {
3987                         device_printf(sc->rl_dev,
3988                             "DUMP statistics request timed out\n");
3989                         return (ETIMEDOUT);
3990                 }
3991 done:
3992                 stats = sc->rl_ldata.rl_stats;
3993                 printf("%s statistics:\n", device_get_nameunit(sc->rl_dev));
3994                 printf("Tx frames : %ju\n",
3995                     (uintmax_t)le64toh(stats->rl_tx_pkts));
3996                 printf("Rx frames : %ju\n",
3997                     (uintmax_t)le64toh(stats->rl_rx_pkts));
3998                 printf("Tx errors : %ju\n",
3999                     (uintmax_t)le64toh(stats->rl_tx_errs));
4000                 printf("Rx errors : %u\n",
4001                     le32toh(stats->rl_rx_errs));
4002                 printf("Rx missed frames : %u\n",
4003                     (uint32_t)le16toh(stats->rl_missed_pkts));
4004                 printf("Rx frame alignment errs : %u\n",
4005                     (uint32_t)le16toh(stats->rl_rx_framealign_errs));
4006                 printf("Tx single collisions : %u\n",
4007                     le32toh(stats->rl_tx_onecoll));
4008                 printf("Tx multiple collisions : %u\n",
4009                     le32toh(stats->rl_tx_multicolls));
4010                 printf("Rx unicast frames : %ju\n",
4011                     (uintmax_t)le64toh(stats->rl_rx_ucasts));
4012                 printf("Rx broadcast frames : %ju\n",
4013                     (uintmax_t)le64toh(stats->rl_rx_bcasts));
4014                 printf("Rx multicast frames : %u\n",
4015                     le32toh(stats->rl_rx_mcasts));
4016                 printf("Tx aborts : %u\n",
4017                     (uint32_t)le16toh(stats->rl_tx_aborts));
4018                 printf("Tx underruns : %u\n",
4019                     (uint32_t)le16toh(stats->rl_rx_underruns));
4020         }
4021
4022         return (error);
4023 }
4024
4025 static int
4026 sysctl_int_range(SYSCTL_HANDLER_ARGS, int low, int high)
4027 {
4028         int error, value;
4029
4030         if (arg1 == NULL)
4031                 return (EINVAL);
4032         value = *(int *)arg1;
4033         error = sysctl_handle_int(oidp, &value, 0, req);
4034         if (error || req->newptr == NULL)
4035                 return (error);
4036         if (value < low || value > high)
4037                 return (EINVAL);
4038         *(int *)arg1 = value;
4039
4040         return (0);
4041 }
4042
4043 static int
4044 sysctl_hw_re_int_mod(SYSCTL_HANDLER_ARGS)
4045 {
4046
4047         return (sysctl_int_range(oidp, arg1, arg2, req, RL_TIMER_MIN,
4048             RL_TIMER_MAX));
4049 }