sys/dev/re/if_re.c

   1 /*-
   2  * Copyright (c) 1997, 1998-2003
   3  *      Bill Paul <wpaul@windriver.com>.  All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  * 3. All advertising materials mentioning features or use of this software
  14  *    must display the following acknowledgement:
  15  *      This product includes software developed by Bill Paul.
  16  * 4. Neither the name of the author nor the names of any co-contributors
  17  *    may be used to endorse or promote products derived from this software
  18  *    without specific prior written permission.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND
  21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23  * ARE DISCLAIMED.  IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD
  24  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  30  * THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <sys/cdefs.h>
  34 __FBSDID("$FreeBSD$");
  35
  36 /*
  37  * RealTek 8139C+/8169/8169S/8110S/8168/8111/8101E PCI NIC driver
  38  *
  39  * Written by Bill Paul <wpaul@windriver.com>
  40  * Senior Networking Software Engineer
  41  * Wind River Systems
  42  */
  43
  44 /*
  45  * This driver is designed to support RealTek's next generation of
  46  * 10/100 and 10/100/1000 PCI ethernet controllers. There are currently
  47  * seven devices in this family: the RTL8139C+, the RTL8169, the RTL8169S,
  48  * RTL8110S, the RTL8168, the RTL8111 and the RTL8101E.
  49  *
  50  * The 8139C+ is a 10/100 ethernet chip. It is backwards compatible
  51  * with the older 8139 family, however it also supports a special
  52  * C+ mode of operation that provides several new performance enhancing
  53  * features. These include:
  54  *
  55  *      o Descriptor based DMA mechanism. Each descriptor represents
  56  *        a single packet fragment. Data buffers may be aligned on
  57  *        any byte boundary.
  58  *
  59  *      o 64-bit DMA
  60  *
  61  *      o TCP/IP checksum offload for both RX and TX
  62  *
  63  *      o High and normal priority transmit DMA rings
  64  *
  65  *      o VLAN tag insertion and extraction
  66  *
  67  *      o TCP large send (segmentation offload)
  68  *
  69  * Like the 8139, the 8139C+ also has a built-in 10/100 PHY. The C+
  70  * programming API is fairly straightforward. The RX filtering, EEPROM
  71  * access and PHY access is the same as it is on the older 8139 series
  72  * chips.
  73  *
  74  * The 8169 is a 64-bit 10/100/1000 gigabit ethernet MAC. It has almost the
  75  * same programming API and feature set as the 8139C+ with the following
  76  * differences and additions:
  77  *
  78  *      o 1000Mbps mode
  79  *
  80  *      o Jumbo frames
  81  *
  82  *      o GMII and TBI ports/registers for interfacing with copper
  83  *        or fiber PHYs
  84  *
  85  *      o RX and TX DMA rings can have up to 1024 descriptors
  86  *        (the 8139C+ allows a maximum of 64)
  87  *
  88  *      o Slight differences in register layout from the 8139C+
  89  *
  90  * The TX start and timer interrupt registers are at different locations
  91  * on the 8169 than they are on the 8139C+. Also, the status word in the
  92  * RX descriptor has a slightly different bit layout. The 8169 does not
  93  * have a built-in PHY. Most reference boards use a Marvell 88E1000 'Alaska'
  94  * copper gigE PHY.
  95  *
  96  * The 8169S/8110S 10/100/1000 devices have built-in copper gigE PHYs
  97  * (the 'S' stands for 'single-chip'). These devices have the same
  98  * programming API as the older 8169, but also have some vendor-specific
  99  * registers for the on-board PHY. The 8110S is a LAN-on-motherboard
 100  * part designed to be pin-compatible with the RealTek 8100 10/100 chip.
 101  *
 102  * This driver takes advantage of the RX and TX checksum offload and
 103  * VLAN tag insertion/extraction features. It also implements TX
 104  * interrupt moderation using the timer interrupt registers, which
 105  * significantly reduces TX interrupt load. There is also support
 106  * for jumbo frames, however the 8169/8169S/8110S can not transmit
 107  * jumbo frames larger than 7440, so the max MTU possible with this
 108  * driver is 7422 bytes.
 109  */
 110
 111 #ifdef HAVE_KERNEL_OPTION_HEADERS
 112 #include "opt_device_polling.h"
 113 #endif
 114
 115 #include <sys/param.h>
 116 #include <sys/endian.h>
 117 #include <sys/systm.h>
 118 #include <sys/sockio.h>
 119 #include <sys/mbuf.h>
 120 #include <sys/malloc.h>
 121 #include <sys/module.h>
 122 #include <sys/kernel.h>
 123 #include <sys/socket.h>
 124 #include <sys/lock.h>
 125 #include <sys/mutex.h>
 126 #include <sys/sysctl.h>
 127 #include <sys/taskqueue.h>
 128
 129 #include <net/if.h>
 130 #include <net/if_arp.h>
 131 #include <net/ethernet.h>
 132 #include <net/if_dl.h>
 133 #include <net/if_media.h>
 134 #include <net/if_types.h>
 135 #include <net/if_vlan_var.h>
 136
 137 #include <net/bpf.h>
 138
 139 #include <machine/bus.h>
 140 #include <machine/resource.h>
 141 #include <sys/bus.h>
 142 #include <sys/rman.h>
 143
 144 #include <dev/mii/mii.h>
 145 #include <dev/mii/miivar.h>
 146
 147 #include <dev/pci/pcireg.h>
 148 #include <dev/pci/pcivar.h>
 149
 150 #include <pci/if_rlreg.h>
 151
 152 MODULE_DEPEND(re, pci, 1, 1, 1);
 153 MODULE_DEPEND(re, ether, 1, 1, 1);
 154 MODULE_DEPEND(re, miibus, 1, 1, 1);
 155
 156 /* "device miibus" required.  See GENERIC if you get errors here. */
 157 #include "miibus_if.h"
 158
 159 /* Tunables. */
 160 static int intr_filter = 0;
 161 TUNABLE_INT("hw.re.intr_filter", &intr_filter);
 162 static int msi_disable = 0;
 163 TUNABLE_INT("hw.re.msi_disable", &msi_disable);
 164 static int msix_disable = 0;
 165 TUNABLE_INT("hw.re.msix_disable", &msix_disable);
 166 static int prefer_iomap = 0;
 167 TUNABLE_INT("hw.re.prefer_iomap", &prefer_iomap);
 168
 169 #define RE_CSUM_FEATURES    (CSUM_IP | CSUM_TCP | CSUM_UDP)
 170
 171 /*
 172  * Various supported device vendors/types and their names.
 173  */
 174 static const struct rl_type re_devs[] = {
 175         { DLINK_VENDORID, DLINK_DEVICEID_528T, 0,
 176             "D-Link DGE-528(T) Gigabit Ethernet Adapter" },
 177         { DLINK_VENDORID, DLINK_DEVICEID_530T_REVC, 0,
 178             "D-Link DGE-530(T) Gigabit Ethernet Adapter" },
 179         { RT_VENDORID, RT_DEVICEID_8139, 0,
 180             "RealTek 8139C+ 10/100BaseTX" },
 181         { RT_VENDORID, RT_DEVICEID_8101E, 0,
 182             "RealTek 810xE PCIe 10/100baseTX" },
 183         { RT_VENDORID, RT_DEVICEID_8168, 0,
 184             "RealTek 8168/8111 B/C/CP/D/DP/E/F/G PCIe Gigabit Ethernet" },
 185         { RT_VENDORID, RT_DEVICEID_8169, 0,
 186             "RealTek 8169/8169S/8169SB(L)/8110S/8110SB(L) Gigabit Ethernet" },
 187         { RT_VENDORID, RT_DEVICEID_8169SC, 0,
 188             "RealTek 8169SC/8110SC Single-chip Gigabit Ethernet" },
 189         { COREGA_VENDORID, COREGA_DEVICEID_CGLAPCIGT, 0,
 190             "Corega CG-LAPCIGT (RTL8169S) Gigabit Ethernet" },
 191         { LINKSYS_VENDORID, LINKSYS_DEVICEID_EG1032, 0,
 192             "Linksys EG1032 (RTL8169S) Gigabit Ethernet" },
 193         { USR_VENDORID, USR_DEVICEID_997902, 0,
 194             "US Robotics 997902 (RTL8169S) Gigabit Ethernet" }
 195 };
 196
 197 static const struct rl_hwrev re_hwrevs[] = {
 198         { RL_HWREV_8139, RL_8139, "", RL_MTU },
 199         { RL_HWREV_8139A, RL_8139, "A", RL_MTU },
 200         { RL_HWREV_8139AG, RL_8139, "A-G", RL_MTU },
 201         { RL_HWREV_8139B, RL_8139, "B", RL_MTU },
 202         { RL_HWREV_8130, RL_8139, "8130", RL_MTU },
 203         { RL_HWREV_8139C, RL_8139, "C", RL_MTU },
 204         { RL_HWREV_8139D, RL_8139, "8139D/8100B/8100C", RL_MTU },
 205         { RL_HWREV_8139CPLUS, RL_8139CPLUS, "C+", RL_MTU },
 206         { RL_HWREV_8168B_SPIN1, RL_8169, "8168", RL_JUMBO_MTU },
 207         { RL_HWREV_8169, RL_8169, "8169", RL_JUMBO_MTU },
 208         { RL_HWREV_8169S, RL_8169, "8169S", RL_JUMBO_MTU },
 209         { RL_HWREV_8110S, RL_8169, "8110S", RL_JUMBO_MTU },
 210         { RL_HWREV_8169_8110SB, RL_8169, "8169SB/8110SB", RL_JUMBO_MTU },
 211         { RL_HWREV_8169_8110SC, RL_8169, "8169SC/8110SC", RL_JUMBO_MTU },
 212         { RL_HWREV_8169_8110SBL, RL_8169, "8169SBL/8110SBL", RL_JUMBO_MTU },
 213         { RL_HWREV_8169_8110SCE, RL_8169, "8169SC/8110SC", RL_JUMBO_MTU },
 214         { RL_HWREV_8100, RL_8139, "8100", RL_MTU },
 215         { RL_HWREV_8101, RL_8139, "8101", RL_MTU },
 216         { RL_HWREV_8100E, RL_8169, "8100E", RL_MTU },
 217         { RL_HWREV_8101E, RL_8169, "8101E", RL_MTU },
 218         { RL_HWREV_8102E, RL_8169, "8102E", RL_MTU },
 219         { RL_HWREV_8102EL, RL_8169, "8102EL", RL_MTU },
 220         { RL_HWREV_8102EL_SPIN1, RL_8169, "8102EL", RL_MTU },
 221         { RL_HWREV_8103E, RL_8169, "8103E", RL_MTU },
 222         { RL_HWREV_8401E, RL_8169, "8401E", RL_MTU },
 223         { RL_HWREV_8402, RL_8169, "8402", RL_MTU },
 224         { RL_HWREV_8105E, RL_8169, "8105E", RL_MTU },
 225         { RL_HWREV_8105E_SPIN1, RL_8169, "8105E", RL_MTU },
 226         { RL_HWREV_8106E, RL_8169, "8106E", RL_MTU },
 227         { RL_HWREV_8168B_SPIN2, RL_8169, "8168", RL_JUMBO_MTU },
 228         { RL_HWREV_8168B_SPIN3, RL_8169, "8168", RL_JUMBO_MTU },
 229         { RL_HWREV_8168C, RL_8169, "8168C/8111C", RL_JUMBO_MTU_6K },
 230         { RL_HWREV_8168C_SPIN2, RL_8169, "8168C/8111C", RL_JUMBO_MTU_6K },
 231         { RL_HWREV_8168CP, RL_8169, "8168CP/8111CP", RL_JUMBO_MTU_6K },
 232         { RL_HWREV_8168D, RL_8169, "8168D/8111D", RL_JUMBO_MTU_9K },
 233         { RL_HWREV_8168DP, RL_8169, "8168DP/8111DP", RL_JUMBO_MTU_9K },
 234         { RL_HWREV_8168E, RL_8169, "8168E/8111E", RL_JUMBO_MTU_9K},
 235         { RL_HWREV_8168E_VL, RL_8169, "8168E/8111E-VL", RL_JUMBO_MTU_6K},
 236         { RL_HWREV_8168EP, RL_8169, "8168EP/8111EP", RL_JUMBO_MTU_9K},
 237         { RL_HWREV_8168F, RL_8169, "8168F/8111F", RL_JUMBO_MTU_9K},
 238         { RL_HWREV_8168G, RL_8169, "8168G/8111G", RL_JUMBO_MTU_9K},
 239         { RL_HWREV_8168GU, RL_8169, "8168GU/8111GU", RL_JUMBO_MTU_9K},
 240         { RL_HWREV_8411, RL_8169, "8411", RL_JUMBO_MTU_9K},
 241         { RL_HWREV_8411B, RL_8169, "8411B", RL_JUMBO_MTU_9K},
 242         { 0, 0, NULL, 0 }
 243 };
 244
 245 static int re_probe             (device_t);
 246 static int re_attach            (device_t);
 247 static int re_detach            (device_t);
 248
 249 static int re_encap             (struct rl_softc *, struct mbuf **);
 250
 251 static void re_dma_map_addr     (void *, bus_dma_segment_t *, int, int);
 252 static int re_allocmem          (device_t, struct rl_softc *);
 253 static __inline void re_discard_rxbuf
 254                                 (struct rl_softc *, int);
 255 static int re_newbuf            (struct rl_softc *, int);
 256 static int re_jumbo_newbuf      (struct rl_softc *, int);
 257 static int re_rx_list_init      (struct rl_softc *);
 258 static int re_jrx_list_init     (struct rl_softc *);
 259 static int re_tx_list_init      (struct rl_softc *);
 260 #ifdef RE_FIXUP_RX
 261 static __inline void re_fixup_rx
 262                                 (struct mbuf *);
 263 #endif
 264 static int re_rxeof             (struct rl_softc *, int *);
 265 static void re_txeof            (struct rl_softc *);
 266 #ifdef DEVICE_POLLING
 267 static int re_poll              (struct ifnet *, enum poll_cmd, int);
 268 static int re_poll_locked       (struct ifnet *, enum poll_cmd, int);
 269 #endif
 270 static int re_intr              (void *);
 271 static void re_intr_msi         (void *);
 272 static void re_tick             (void *);
 273 static void re_int_task         (void *, int);
 274 static void re_start            (struct ifnet *);
 275 static void re_start_locked     (struct ifnet *);
 276 static int re_ioctl             (struct ifnet *, u_long, caddr_t);
 277 static void re_init             (void *);
 278 static void re_init_locked      (struct rl_softc *);
 279 static void re_stop             (struct rl_softc *);
 280 static void re_watchdog         (struct rl_softc *);
 281 static int re_suspend           (device_t);
 282 static int re_resume            (device_t);
 283 static int re_shutdown          (device_t);
 284 static int re_ifmedia_upd       (struct ifnet *);
 285 static void re_ifmedia_sts      (struct ifnet *, struct ifmediareq *);
 286
 287 static void re_eeprom_putbyte   (struct rl_softc *, int);
 288 static void re_eeprom_getword   (struct rl_softc *, int, u_int16_t *);
 289 static void re_read_eeprom      (struct rl_softc *, caddr_t, int, int);
 290 static int re_gmii_readreg      (device_t, int, int);
 291 static int re_gmii_writereg     (device_t, int, int, int);
 292
 293 static int re_miibus_readreg    (device_t, int, int);
 294 static int re_miibus_writereg   (device_t, int, int, int);
 295 static void re_miibus_statchg   (device_t);
 296
 297 static void re_set_jumbo        (struct rl_softc *, int);
 298 static void re_set_rxmode               (struct rl_softc *);
 299 static void re_reset            (struct rl_softc *);
 300 static void re_setwol           (struct rl_softc *);
 301 static void re_clrwol           (struct rl_softc *);
 302 static void re_set_linkspeed    (struct rl_softc *);
 303
 304 #ifdef DEV_NETMAP       /* see ixgbe.c for details */
 305 #include <dev/netmap/if_re_netmap.h>
 306 #endif /* !DEV_NETMAP */
 307
 308 #ifdef RE_DIAG
 309 static int re_diag              (struct rl_softc *);
 310 #endif
 311
 312 static void re_add_sysctls      (struct rl_softc *);
 313 static int re_sysctl_stats      (SYSCTL_HANDLER_ARGS);
 314 static int sysctl_int_range     (SYSCTL_HANDLER_ARGS, int, int);
 315 static int sysctl_hw_re_int_mod (SYSCTL_HANDLER_ARGS);
 316
 317 static device_method_t re_methods[] = {
 318         /* Device interface */
 319         DEVMETHOD(device_probe,         re_probe),
 320         DEVMETHOD(device_attach,        re_attach),
 321         DEVMETHOD(device_detach,        re_detach),
 322         DEVMETHOD(device_suspend,       re_suspend),
 323         DEVMETHOD(device_resume,        re_resume),
 324         DEVMETHOD(device_shutdown,      re_shutdown),
 325
 326         /* MII interface */
 327         DEVMETHOD(miibus_readreg,       re_miibus_readreg),
 328         DEVMETHOD(miibus_writereg,      re_miibus_writereg),
 329         DEVMETHOD(miibus_statchg,       re_miibus_statchg),
 330
 331         DEVMETHOD_END
 332 };
 333
 334 static driver_t re_driver = {
 335         "re",
 336         re_methods,
 337         sizeof(struct rl_softc)
 338 };
 339
 340 static devclass_t re_devclass;
 341
 342 DRIVER_MODULE(re, pci, re_driver, re_devclass, 0, 0);
 343 DRIVER_MODULE(miibus, re, miibus_driver, miibus_devclass, 0, 0);
 344
 345 #define EE_SET(x)                                       \
 346         CSR_WRITE_1(sc, RL_EECMD,                       \
 347                 CSR_READ_1(sc, RL_EECMD) | x)
 348
 349 #define EE_CLR(x)                                       \
 350         CSR_WRITE_1(sc, RL_EECMD,                       \
 351                 CSR_READ_1(sc, RL_EECMD) & ~x)
 352
 353 /*
 354  * Send a read command and address to the EEPROM, check for ACK.
 355  */
 356 static void
 357 re_eeprom_putbyte(struct rl_softc *sc, int addr)
 358 {
 359         int                     d, i;
 360
 361         d = addr | (RL_9346_READ << sc->rl_eewidth);
 362
 363         /*
 364          * Feed in each bit and strobe the clock.
 365          */
 366
 367         for (i = 1 << (sc->rl_eewidth + 3); i; i >>= 1) {
 368                 if (d & i) {
 369                         EE_SET(RL_EE_DATAIN);
 370                 } else {
 371                         EE_CLR(RL_EE_DATAIN);
 372                 }
 373                 DELAY(100);
 374                 EE_SET(RL_EE_CLK);
 375                 DELAY(150);
 376                 EE_CLR(RL_EE_CLK);
 377                 DELAY(100);
 378         }
 379 }
 380
 381 /*
 382  * Read a word of data stored in the EEPROM at address 'addr.'
 383  */
 384 static void
 385 re_eeprom_getword(struct rl_softc *sc, int addr, u_int16_t *dest)
 386 {
 387         int                     i;
 388         u_int16_t               word = 0;
 389
 390         /*
 391          * Send address of word we want to read.
 392          */
 393         re_eeprom_putbyte(sc, addr);
 394
 395         /*
 396          * Start reading bits from EEPROM.
 397          */
 398         for (i = 0x8000; i; i >>= 1) {
 399                 EE_SET(RL_EE_CLK);
 400                 DELAY(100);
 401                 if (CSR_READ_1(sc, RL_EECMD) & RL_EE_DATAOUT)
 402                         word |= i;
 403                 EE_CLR(RL_EE_CLK);
 404                 DELAY(100);
 405         }
 406
 407         *dest = word;
 408 }
 409
 410 /*
 411  * Read a sequence of words from the EEPROM.
 412  */
 413 static void
 414 re_read_eeprom(struct rl_softc *sc, caddr_t dest, int off, int cnt)
 415 {
 416         int                     i;
 417         u_int16_t               word = 0, *ptr;
 418
 419         CSR_SETBIT_1(sc, RL_EECMD, RL_EEMODE_PROGRAM);
 420
 421         DELAY(100);
 422
 423         for (i = 0; i < cnt; i++) {
 424                 CSR_SETBIT_1(sc, RL_EECMD, RL_EE_SEL);
 425                 re_eeprom_getword(sc, off + i, &word);
 426                 CSR_CLRBIT_1(sc, RL_EECMD, RL_EE_SEL);
 427                 ptr = (u_int16_t *)(dest + (i * 2));
 428                 *ptr = word;
 429         }
 430
 431         CSR_CLRBIT_1(sc, RL_EECMD, RL_EEMODE_PROGRAM);
 432 }
 433
 434 static int
 435 re_gmii_readreg(device_t dev, int phy, int reg)
 436 {
 437         struct rl_softc         *sc;
 438         u_int32_t               rval;
 439         int                     i;
 440
 441         sc = device_get_softc(dev);
 442
 443         /* Let the rgephy driver read the GMEDIASTAT register */
 444
 445         if (reg == RL_GMEDIASTAT) {
 446                 rval = CSR_READ_1(sc, RL_GMEDIASTAT);
 447                 return (rval);
 448         }
 449
 450         CSR_WRITE_4(sc, RL_PHYAR, reg << 16);
 451
 452         for (i = 0; i < RL_PHY_TIMEOUT; i++) {
 453                 rval = CSR_READ_4(sc, RL_PHYAR);
 454                 if (rval & RL_PHYAR_BUSY)
 455                         break;
 456                 DELAY(25);
 457         }
 458
 459         if (i == RL_PHY_TIMEOUT) {
 460                 device_printf(sc->rl_dev, "PHY read failed\n");
 461                 return (0);
 462         }
 463
 464         /*
 465          * Controller requires a 20us delay to process next MDIO request.
 466          */
 467         DELAY(20);
 468
 469         return (rval & RL_PHYAR_PHYDATA);
 470 }
 471
 472 static int
 473 re_gmii_writereg(device_t dev, int phy, int reg, int data)
 474 {
 475         struct rl_softc         *sc;
 476         u_int32_t               rval;
 477         int                     i;
 478
 479         sc = device_get_softc(dev);
 480
 481         CSR_WRITE_4(sc, RL_PHYAR, (reg << 16) |
 482             (data & RL_PHYAR_PHYDATA) | RL_PHYAR_BUSY);
 483
 484         for (i = 0; i < RL_PHY_TIMEOUT; i++) {
 485                 rval = CSR_READ_4(sc, RL_PHYAR);
 486                 if (!(rval & RL_PHYAR_BUSY))
 487                         break;
 488                 DELAY(25);
 489         }
 490
 491         if (i == RL_PHY_TIMEOUT) {
 492                 device_printf(sc->rl_dev, "PHY write failed\n");
 493                 return (0);
 494         }
 495
 496         /*
 497          * Controller requires a 20us delay to process next MDIO request.
 498          */
 499         DELAY(20);
 500
 501         return (0);
 502 }
 503
 504 static int
 505 re_miibus_readreg(device_t dev, int phy, int reg)
 506 {
 507         struct rl_softc         *sc;
 508         u_int16_t               rval = 0;
 509         u_int16_t               re8139_reg = 0;
 510
 511         sc = device_get_softc(dev);
 512
 513         if (sc->rl_type == RL_8169) {
 514                 rval = re_gmii_readreg(dev, phy, reg);
 515                 return (rval);
 516         }
 517
 518         switch (reg) {
 519         case MII_BMCR:
 520                 re8139_reg = RL_BMCR;
 521                 break;
 522         case MII_BMSR:
 523                 re8139_reg = RL_BMSR;
 524                 break;
 525         case MII_ANAR:
 526                 re8139_reg = RL_ANAR;
 527                 break;
 528         case MII_ANER:
 529                 re8139_reg = RL_ANER;
 530                 break;
 531         case MII_ANLPAR:
 532                 re8139_reg = RL_LPAR;
 533                 break;
 534         case MII_PHYIDR1:
 535         case MII_PHYIDR2:
 536                 return (0);
 537         /*
 538          * Allow the rlphy driver to read the media status
 539          * register. If we have a link partner which does not
 540          * support NWAY, this is the register which will tell
 541          * us the results of parallel detection.
 542          */
 543         case RL_MEDIASTAT:
 544                 rval = CSR_READ_1(sc, RL_MEDIASTAT);
 545                 return (rval);
 546         default:
 547                 device_printf(sc->rl_dev, "bad phy register\n");
 548                 return (0);
 549         }
 550         rval = CSR_READ_2(sc, re8139_reg);
 551         if (sc->rl_type == RL_8139CPLUS && re8139_reg == RL_BMCR) {
 552                 /* 8139C+ has different bit layout. */
 553                 rval &= ~(BMCR_LOOP | BMCR_ISO);
 554         }
 555         return (rval);
 556 }
 557
 558 static int
 559 re_miibus_writereg(device_t dev, int phy, int reg, int data)
 560 {
 561         struct rl_softc         *sc;
 562         u_int16_t               re8139_reg = 0;
 563         int                     rval = 0;
 564
 565         sc = device_get_softc(dev);
 566
 567         if (sc->rl_type == RL_8169) {
 568                 rval = re_gmii_writereg(dev, phy, reg, data);
 569                 return (rval);
 570         }
 571
 572         switch (reg) {
 573         case MII_BMCR:
 574                 re8139_reg = RL_BMCR;
 575                 if (sc->rl_type == RL_8139CPLUS) {
 576                         /* 8139C+ has different bit layout. */
 577                         data &= ~(BMCR_LOOP | BMCR_ISO);
 578                 }
 579                 break;
 580         case MII_BMSR:
 581                 re8139_reg = RL_BMSR;
 582                 break;
 583         case MII_ANAR:
 584                 re8139_reg = RL_ANAR;
 585                 break;
 586         case MII_ANER:
 587                 re8139_reg = RL_ANER;
 588                 break;
 589         case MII_ANLPAR:
 590                 re8139_reg = RL_LPAR;
 591                 break;
 592         case MII_PHYIDR1:
 593         case MII_PHYIDR2:
 594                 return (0);
 595                 break;
 596         default:
 597                 device_printf(sc->rl_dev, "bad phy register\n");
 598                 return (0);
 599         }
 600         CSR_WRITE_2(sc, re8139_reg, data);
 601         return (0);
 602 }
 603
 604 static void
 605 re_miibus_statchg(device_t dev)
 606 {
 607         struct rl_softc         *sc;
 608         struct ifnet            *ifp;
 609         struct mii_data         *mii;
 610
 611         sc = device_get_softc(dev);
 612         mii = device_get_softc(sc->rl_miibus);
 613         ifp = sc->rl_ifp;
 614         if (mii == NULL || ifp == NULL ||
 615             (ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 616                 return;
 617
 618         sc->rl_flags &= ~RL_FLAG_LINK;
 619         if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) ==
 620             (IFM_ACTIVE | IFM_AVALID)) {
 621                 switch (IFM_SUBTYPE(mii->mii_media_active)) {
 622                 case IFM_10_T:
 623                 case IFM_100_TX:
 624                         sc->rl_flags |= RL_FLAG_LINK;
 625                         break;
 626                 case IFM_1000_T:
 627                         if ((sc->rl_flags & RL_FLAG_FASTETHER) != 0)
 628                                 break;
 629                         sc->rl_flags |= RL_FLAG_LINK;
 630                         break;
 631                 default:
 632                         break;
 633                 }
 634         }
 635         /*
 636          * RealTek controllers does not provide any interface to
 637          * Tx/Rx MACs for resolved speed, duplex and flow-control
 638          * parameters.
 639          */
 640 }
 641
 642 /*
 643  * Set the RX configuration and 64-bit multicast hash filter.
 644  */
 645 static void
 646 re_set_rxmode(struct rl_softc *sc)
 647 {
 648         struct ifnet            *ifp;
 649         struct ifmultiaddr      *ifma;
 650         uint32_t                hashes[2] = { 0, 0 };
 651         uint32_t                h, rxfilt;
 652
 653         RL_LOCK_ASSERT(sc);
 654
 655         ifp = sc->rl_ifp;
 656
 657         rxfilt = RL_RXCFG_CONFIG | RL_RXCFG_RX_INDIV | RL_RXCFG_RX_BROAD;
 658         if ((sc->rl_flags & RL_FLAG_EARLYOFF) != 0)
 659                 rxfilt |= RL_RXCFG_EARLYOFF;
 660         else if ((sc->rl_flags & RL_FLAG_EARLYOFFV2) != 0)
 661                 rxfilt |= RL_RXCFG_EARLYOFFV2;
 662
 663         if (ifp->if_flags & (IFF_ALLMULTI | IFF_PROMISC)) {
 664                 if (ifp->if_flags & IFF_PROMISC)
 665                         rxfilt |= RL_RXCFG_RX_ALLPHYS;
 666                 /*
 667                  * Unlike other hardwares, we have to explicitly set
 668                  * RL_RXCFG_RX_MULTI to receive multicast frames in
 669                  * promiscuous mode.
 670                  */
 671                 rxfilt |= RL_RXCFG_RX_MULTI;
 672                 hashes[0] = hashes[1] = 0xffffffff;
 673                 goto done;
 674         }
 675
 676         if_maddr_rlock(ifp);
 677         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
 678                 if (ifma->ifma_addr->sa_family != AF_LINK)
 679                         continue;
 680                 h = ether_crc32_be(LLADDR((struct sockaddr_dl *)
 681                     ifma->ifma_addr), ETHER_ADDR_LEN) >> 26;
 682                 if (h < 32)
 683                         hashes[0] |= (1 << h);
 684                 else
 685                         hashes[1] |= (1 << (h - 32));
 686         }
 687         if_maddr_runlock(ifp);
 688
 689         if (hashes[0] != 0 || hashes[1] != 0) {
 690                 /*
 691                  * For some unfathomable reason, RealTek decided to
 692                  * reverse the order of the multicast hash registers
 693                  * in the PCI Express parts.  This means we have to
 694                  * write the hash pattern in reverse order for those
 695                  * devices.
 696                  */
 697                 if ((sc->rl_flags & RL_FLAG_PCIE) != 0) {
 698                         h = bswap32(hashes[0]);
 699                         hashes[0] = bswap32(hashes[1]);
 700                         hashes[1] = h;
 701                 }
 702                 rxfilt |= RL_RXCFG_RX_MULTI;
 703         }
 704
 705 done:
 706         CSR_WRITE_4(sc, RL_MAR0, hashes[0]);
 707         CSR_WRITE_4(sc, RL_MAR4, hashes[1]);
 708         CSR_WRITE_4(sc, RL_RXCFG, rxfilt);
 709 }
 710
 711 static void
 712 re_reset(struct rl_softc *sc)
 713 {
 714         int                     i;
 715
 716         RL_LOCK_ASSERT(sc);
 717
 718         CSR_WRITE_1(sc, RL_COMMAND, RL_CMD_RESET);
 719
 720         for (i = 0; i < RL_TIMEOUT; i++) {
 721                 DELAY(10);
 722                 if (!(CSR_READ_1(sc, RL_COMMAND) & RL_CMD_RESET))
 723                         break;
 724         }
 725         if (i == RL_TIMEOUT)
 726                 device_printf(sc->rl_dev, "reset never completed!\n");
 727
 728         if ((sc->rl_flags & RL_FLAG_MACRESET) != 0)
 729                 CSR_WRITE_1(sc, 0x82, 1);
 730         if (sc->rl_hwrev->rl_rev == RL_HWREV_8169S)
 731                 re_gmii_writereg(sc->rl_dev, 1, 0x0b, 0);
 732 }
 733
 734 #ifdef RE_DIAG
 735
 736 /*
 737  * The following routine is designed to test for a defect on some
 738  * 32-bit 8169 cards. Some of these NICs have the REQ64# and ACK64#
 739  * lines connected to the bus, however for a 32-bit only card, they
 740  * should be pulled high. The result of this defect is that the
 741  * NIC will not work right if you plug it into a 64-bit slot: DMA
 742  * operations will be done with 64-bit transfers, which will fail
 743  * because the 64-bit data lines aren't connected.
 744  *
 745  * There's no way to work around this (short of talking a soldering
 746  * iron to the board), however we can detect it. The method we use
 747  * here is to put the NIC into digital loopback mode, set the receiver
 748  * to promiscuous mode, and then try to send a frame. We then compare
 749  * the frame data we sent to what was received. If the data matches,
 750  * then the NIC is working correctly, otherwise we know the user has
 751  * a defective NIC which has been mistakenly plugged into a 64-bit PCI
 752  * slot. In the latter case, there's no way the NIC can work correctly,
 753  * so we print out a message on the console and abort the device attach.
 754  */
 755
 756 static int
 757 re_diag(struct rl_softc *sc)
 758 {
 759         struct ifnet            *ifp = sc->rl_ifp;
 760         struct mbuf             *m0;
 761         struct ether_header     *eh;
 762         struct rl_desc          *cur_rx;
 763         u_int16_t               status;
 764         u_int32_t               rxstat;
 765         int                     total_len, i, error = 0, phyaddr;
 766         u_int8_t                dst[] = { 0x00, 'h', 'e', 'l', 'l', 'o' };
 767         u_int8_t                src[] = { 0x00, 'w', 'o', 'r', 'l', 'd' };
 768
 769         /* Allocate a single mbuf */
 770         MGETHDR(m0, M_NOWAIT, MT_DATA);
 771         if (m0 == NULL)
 772                 return (ENOBUFS);
 773
 774         RL_LOCK(sc);
 775
 776         /*
 777          * Initialize the NIC in test mode. This sets the chip up
 778          * so that it can send and receive frames, but performs the
 779          * following special functions:
 780          * - Puts receiver in promiscuous mode
 781          * - Enables digital loopback mode
 782          * - Leaves interrupts turned off
 783          */
 784
 785         ifp->if_flags |= IFF_PROMISC;
 786         sc->rl_testmode = 1;
 787         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 788         re_init_locked(sc);
 789         sc->rl_flags |= RL_FLAG_LINK;
 790         if (sc->rl_type == RL_8169)
 791                 phyaddr = 1;
 792         else
 793                 phyaddr = 0;
 794
 795         re_miibus_writereg(sc->rl_dev, phyaddr, MII_BMCR, BMCR_RESET);
 796         for (i = 0; i < RL_TIMEOUT; i++) {
 797                 status = re_miibus_readreg(sc->rl_dev, phyaddr, MII_BMCR);
 798                 if (!(status & BMCR_RESET))
 799                         break;
 800         }
 801
 802         re_miibus_writereg(sc->rl_dev, phyaddr, MII_BMCR, BMCR_LOOP);
 803         CSR_WRITE_2(sc, RL_ISR, RL_INTRS);
 804
 805         DELAY(100000);
 806
 807         /* Put some data in the mbuf */
 808
 809         eh = mtod(m0, struct ether_header *);
 810         bcopy ((char *)&dst, eh->ether_dhost, ETHER_ADDR_LEN);
 811         bcopy ((char *)&src, eh->ether_shost, ETHER_ADDR_LEN);
 812         eh->ether_type = htons(ETHERTYPE_IP);
 813         m0->m_pkthdr.len = m0->m_len = ETHER_MIN_LEN - ETHER_CRC_LEN;
 814
 815         /*
 816          * Queue the packet, start transmission.
 817          * Note: IF_HANDOFF() ultimately calls re_start() for us.
 818          */
 819
 820         CSR_WRITE_2(sc, RL_ISR, 0xFFFF);
 821         RL_UNLOCK(sc);
 822         /* XXX: re_diag must not be called when in ALTQ mode */
 823         IF_HANDOFF(&ifp->if_snd, m0, ifp);
 824         RL_LOCK(sc);
 825         m0 = NULL;
 826
 827         /* Wait for it to propagate through the chip */
 828
 829         DELAY(100000);
 830         for (i = 0; i < RL_TIMEOUT; i++) {
 831                 status = CSR_READ_2(sc, RL_ISR);
 832                 CSR_WRITE_2(sc, RL_ISR, status);
 833                 if ((status & (RL_ISR_TIMEOUT_EXPIRED|RL_ISR_RX_OK)) ==
 834                     (RL_ISR_TIMEOUT_EXPIRED|RL_ISR_RX_OK))
 835                         break;
 836                 DELAY(10);
 837         }
 838
 839         if (i == RL_TIMEOUT) {
 840                 device_printf(sc->rl_dev,
 841                     "diagnostic failed, failed to receive packet in"
 842                     " loopback mode\n");
 843                 error = EIO;
 844                 goto done;
 845         }
 846
 847         /*
 848          * The packet should have been dumped into the first
 849          * entry in the RX DMA ring. Grab it from there.
 850          */
 851
 852         bus_dmamap_sync(sc->rl_ldata.rl_rx_list_tag,
 853             sc->rl_ldata.rl_rx_list_map,
 854             BUS_DMASYNC_POSTREAD);
 855         bus_dmamap_sync(sc->rl_ldata.rl_rx_mtag,
 856             sc->rl_ldata.rl_rx_desc[0].rx_dmamap,
 857             BUS_DMASYNC_POSTREAD);
 858         bus_dmamap_unload(sc->rl_ldata.rl_rx_mtag,
 859             sc->rl_ldata.rl_rx_desc[0].rx_dmamap);
 860
 861         m0 = sc->rl_ldata.rl_rx_desc[0].rx_m;
 862         sc->rl_ldata.rl_rx_desc[0].rx_m = NULL;
 863         eh = mtod(m0, struct ether_header *);
 864
 865         cur_rx = &sc->rl_ldata.rl_rx_list[0];
 866         total_len = RL_RXBYTES(cur_rx);
 867         rxstat = le32toh(cur_rx->rl_cmdstat);
 868
 869         if (total_len != ETHER_MIN_LEN) {
 870                 device_printf(sc->rl_dev,
 871                     "diagnostic failed, received short packet\n");
 872                 error = EIO;
 873                 goto done;
 874         }
 875
 876         /* Test that the received packet data matches what we sent. */
 877
 878         if (bcmp((char *)&eh->ether_dhost, (char *)&dst, ETHER_ADDR_LEN) ||
 879             bcmp((char *)&eh->ether_shost, (char *)&src, ETHER_ADDR_LEN) ||
 880             ntohs(eh->ether_type) != ETHERTYPE_IP) {
 881                 device_printf(sc->rl_dev, "WARNING, DMA FAILURE!\n");
 882                 device_printf(sc->rl_dev, "expected TX data: %6D/%6D/0x%x\n",
 883                     dst, ":", src, ":", ETHERTYPE_IP);
 884                 device_printf(sc->rl_dev, "received RX data: %6D/%6D/0x%x\n",
 885                     eh->ether_dhost, ":", eh->ether_shost, ":",
 886                     ntohs(eh->ether_type));
 887                 device_printf(sc->rl_dev, "You may have a defective 32-bit "
 888                     "NIC plugged into a 64-bit PCI slot.\n");
 889                 device_printf(sc->rl_dev, "Please re-install the NIC in a "
 890                     "32-bit slot for proper operation.\n");
 891                 device_printf(sc->rl_dev, "Read the re(4) man page for more "
 892                     "details.\n");
 893                 error = EIO;
 894         }
 895
 896 done:
 897         /* Turn interface off, release resources */
 898
 899         sc->rl_testmode = 0;
 900         sc->rl_flags &= ~RL_FLAG_LINK;
 901         ifp->if_flags &= ~IFF_PROMISC;
 902         re_stop(sc);
 903         if (m0 != NULL)
 904                 m_freem(m0);
 905
 906         RL_UNLOCK(sc);
 907
 908         return (error);
 909 }
 910
 911 #endif
 912
 913 /*
 914  * Probe for a RealTek 8139C+/8169/8110 chip. Check the PCI vendor and device
 915  * IDs against our list and return a device name if we find a match.
 916  */
 917 static int
 918 re_probe(device_t dev)
 919 {
 920         const struct rl_type    *t;
 921         uint16_t                devid, vendor;
 922         uint16_t                revid, sdevid;
 923         int                     i;
 924
 925         vendor = pci_get_vendor(dev);
 926         devid = pci_get_device(dev);
 927         revid = pci_get_revid(dev);
 928         sdevid = pci_get_subdevice(dev);
 929
 930         if (vendor == LINKSYS_VENDORID && devid == LINKSYS_DEVICEID_EG1032) {
 931                 if (sdevid != LINKSYS_SUBDEVICE_EG1032_REV3) {
 932                         /*
 933                          * Only attach to rev. 3 of the Linksys EG1032 adapter.
 934                          * Rev. 2 is supported by sk(4).
 935                          */
 936                         return (ENXIO);
 937                 }
 938         }
 939
 940         if (vendor == RT_VENDORID && devid == RT_DEVICEID_8139) {
 941                 if (revid != 0x20) {
 942                         /* 8139, let rl(4) take care of this device. */
 943                         return (ENXIO);
 944                 }
 945         }
 946
 947         t = re_devs;
 948         for (i = 0; i < sizeof(re_devs) / sizeof(re_devs[0]); i++, t++) {
 949                 if (vendor == t->rl_vid && devid == t->rl_did) {
 950                         device_set_desc(dev, t->rl_name);
 951                         return (BUS_PROBE_DEFAULT);
 952                 }
 953         }
 954
 955         return (ENXIO);
 956 }
 957
 958 /*
 959  * Map a single buffer address.
 960  */
 961
 962 static void
 963 re_dma_map_addr(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 964 {
 965         bus_addr_t              *addr;
 966
 967         if (error)
 968                 return;
 969
 970         KASSERT(nseg == 1, ("too many DMA segments, %d should be 1", nseg));
 971         addr = arg;
 972         *addr = segs->ds_addr;
 973 }
 974
 975 static int
 976 re_allocmem(device_t dev, struct rl_softc *sc)
 977 {
 978         bus_addr_t              lowaddr;
 979         bus_size_t              rx_list_size, tx_list_size;
 980         int                     error;
 981         int                     i;
 982
 983         rx_list_size = sc->rl_ldata.rl_rx_desc_cnt * sizeof(struct rl_desc);
 984         tx_list_size = sc->rl_ldata.rl_tx_desc_cnt * sizeof(struct rl_desc);
 985
 986         /*
 987          * Allocate the parent bus DMA tag appropriate for PCI.
 988          * In order to use DAC, RL_CPLUSCMD_PCI_DAC bit of RL_CPLUS_CMD
 989          * register should be set. However some RealTek chips are known
 990          * to be buggy on DAC handling, therefore disable DAC by limiting
 991          * DMA address space to 32bit. PCIe variants of RealTek chips
 992          * may not have the limitation.
 993          */
 994         lowaddr = BUS_SPACE_MAXADDR;
 995         if ((sc->rl_flags & RL_FLAG_PCIE) == 0)
 996                 lowaddr = BUS_SPACE_MAXADDR_32BIT;
 997         error = bus_dma_tag_create(bus_get_dma_tag(dev), 1, 0,
 998             lowaddr, BUS_SPACE_MAXADDR, NULL, NULL,
 999             BUS_SPACE_MAXSIZE_32BIT, 0, BUS_SPACE_MAXSIZE_32BIT, 0,
1000             NULL, NULL, &sc->rl_parent_tag);
1001         if (error) {
1002                 device_printf(dev, "could not allocate parent DMA tag\n");
1003                 return (error);
1004         }
1005
1006         /*
1007          * Allocate map for TX mbufs.
1008          */
1009         error = bus_dma_tag_create(sc->rl_parent_tag, 1, 0,
1010             BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL,
1011             NULL, MCLBYTES * RL_NTXSEGS, RL_NTXSEGS, 4096, 0,
1012             NULL, NULL, &sc->rl_ldata.rl_tx_mtag);
1013         if (error) {
1014                 device_printf(dev, "could not allocate TX DMA tag\n");
1015                 return (error);
1016         }
1017
1018         /*
1019          * Allocate map for RX mbufs.
1020          */
1021
1022         if ((sc->rl_flags & RL_FLAG_JUMBOV2) != 0) {
1023                 error = bus_dma_tag_create(sc->rl_parent_tag, sizeof(uint64_t),
1024                     0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
1025                     MJUM9BYTES, 1, MJUM9BYTES, 0, NULL, NULL,
1026                     &sc->rl_ldata.rl_jrx_mtag);
1027                 if (error) {
1028                         device_printf(dev,
1029                             "could not allocate jumbo RX DMA tag\n");
1030                         return (error);
1031                 }
1032         }
1033         error = bus_dma_tag_create(sc->rl_parent_tag, sizeof(uint64_t), 0,
1034             BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
1035             MCLBYTES, 1, MCLBYTES, 0, NULL, NULL, &sc->rl_ldata.rl_rx_mtag);
1036         if (error) {
1037                 device_printf(dev, "could not allocate RX DMA tag\n");
1038                 return (error);
1039         }
1040
1041         /*
1042          * Allocate map for TX descriptor list.
1043          */
1044         error = bus_dma_tag_create(sc->rl_parent_tag, RL_RING_ALIGN,
1045             0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL,
1046             NULL, tx_list_size, 1, tx_list_size, 0,
1047             NULL, NULL, &sc->rl_ldata.rl_tx_list_tag);
1048         if (error) {
1049                 device_printf(dev, "could not allocate TX DMA ring tag\n");
1050                 return (error);
1051         }
1052
1053         /* Allocate DMA'able memory for the TX ring */
1054
1055         error = bus_dmamem_alloc(sc->rl_ldata.rl_tx_list_tag,
1056             (void **)&sc->rl_ldata.rl_tx_list,
1057             BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO,
1058             &sc->rl_ldata.rl_tx_list_map);
1059         if (error) {
1060                 device_printf(dev, "could not allocate TX DMA ring\n");
1061                 return (error);
1062         }
1063
1064         /* Load the map for the TX ring. */
1065
1066         sc->rl_ldata.rl_tx_list_addr = 0;
1067         error = bus_dmamap_load(sc->rl_ldata.rl_tx_list_tag,
1068              sc->rl_ldata.rl_tx_list_map, sc->rl_ldata.rl_tx_list,
1069              tx_list_size, re_dma_map_addr,
1070              &sc->rl_ldata.rl_tx_list_addr, BUS_DMA_NOWAIT);
1071         if (error != 0 || sc->rl_ldata.rl_tx_list_addr == 0) {
1072                 device_printf(dev, "could not load TX DMA ring\n");
1073                 return (ENOMEM);
1074         }
1075
1076         /* Create DMA maps for TX buffers */
1077
1078         for (i = 0; i < sc->rl_ldata.rl_tx_desc_cnt; i++) {
1079                 error = bus_dmamap_create(sc->rl_ldata.rl_tx_mtag, 0,
1080                     &sc->rl_ldata.rl_tx_desc[i].tx_dmamap);
1081                 if (error) {
1082                         device_printf(dev, "could not create DMA map for TX\n");
1083                         return (error);
1084                 }
1085         }
1086
1087         /*
1088          * Allocate map for RX descriptor list.
1089          */
1090         error = bus_dma_tag_create(sc->rl_parent_tag, RL_RING_ALIGN,
1091             0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL,
1092             NULL, rx_list_size, 1, rx_list_size, 0,
1093             NULL, NULL, &sc->rl_ldata.rl_rx_list_tag);
1094         if (error) {
1095                 device_printf(dev, "could not create RX DMA ring tag\n");
1096                 return (error);
1097         }
1098
1099         /* Allocate DMA'able memory for the RX ring */
1100
1101         error = bus_dmamem_alloc(sc->rl_ldata.rl_rx_list_tag,
1102             (void **)&sc->rl_ldata.rl_rx_list,
1103             BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO,
1104             &sc->rl_ldata.rl_rx_list_map);
1105         if (error) {
1106                 device_printf(dev, "could not allocate RX DMA ring\n");
1107                 return (error);
1108         }
1109
1110         /* Load the map for the RX ring. */
1111
1112         sc->rl_ldata.rl_rx_list_addr = 0;
1113         error = bus_dmamap_load(sc->rl_ldata.rl_rx_list_tag,
1114              sc->rl_ldata.rl_rx_list_map, sc->rl_ldata.rl_rx_list,
1115              rx_list_size, re_dma_map_addr,
1116              &sc->rl_ldata.rl_rx_list_addr, BUS_DMA_NOWAIT);
1117         if (error != 0 || sc->rl_ldata.rl_rx_list_addr == 0) {
1118                 device_printf(dev, "could not load RX DMA ring\n");
1119                 return (ENOMEM);
1120         }
1121
1122         /* Create DMA maps for RX buffers */
1123
1124         if ((sc->rl_flags & RL_FLAG_JUMBOV2) != 0) {
1125                 error = bus_dmamap_create(sc->rl_ldata.rl_jrx_mtag, 0,
1126                     &sc->rl_ldata.rl_jrx_sparemap);
1127                 if (error) {
1128                         device_printf(dev,
1129                             "could not create spare DMA map for jumbo RX\n");
1130                         return (error);
1131                 }
1132                 for (i = 0; i < sc->rl_ldata.rl_rx_desc_cnt; i++) {
1133                         error = bus_dmamap_create(sc->rl_ldata.rl_jrx_mtag, 0,
1134                             &sc->rl_ldata.rl_jrx_desc[i].rx_dmamap);
1135                         if (error) {
1136                                 device_printf(dev,
1137                                     "could not create DMA map for jumbo RX\n");
1138                                 return (error);
1139                         }
1140                 }
1141         }
1142         error = bus_dmamap_create(sc->rl_ldata.rl_rx_mtag, 0,
1143             &sc->rl_ldata.rl_rx_sparemap);
1144         if (error) {
1145                 device_printf(dev, "could not create spare DMA map for RX\n");
1146                 return (error);
1147         }
1148         for (i = 0; i < sc->rl_ldata.rl_rx_desc_cnt; i++) {
1149                 error = bus_dmamap_create(sc->rl_ldata.rl_rx_mtag, 0,
1150                     &sc->rl_ldata.rl_rx_desc[i].rx_dmamap);
1151                 if (error) {
1152                         device_printf(dev, "could not create DMA map for RX\n");
1153                         return (error);
1154                 }
1155         }
1156
1157         /* Create DMA map for statistics. */
1158         error = bus_dma_tag_create(sc->rl_parent_tag, RL_DUMP_ALIGN, 0,
1159             BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
1160             sizeof(struct rl_stats), 1, sizeof(struct rl_stats), 0, NULL, NULL,
1161             &sc->rl_ldata.rl_stag);
1162         if (error) {
1163                 device_printf(dev, "could not create statistics DMA tag\n");
1164                 return (error);
1165         }
1166         /* Allocate DMA'able memory for statistics. */
1167         error = bus_dmamem_alloc(sc->rl_ldata.rl_stag,
1168             (void **)&sc->rl_ldata.rl_stats,
1169             BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO,
1170             &sc->rl_ldata.rl_smap);
1171         if (error) {
1172                 device_printf(dev,
1173                     "could not allocate statistics DMA memory\n");
1174                 return (error);
1175         }
1176         /* Load the map for statistics. */
1177         sc->rl_ldata.rl_stats_addr = 0;
1178         error = bus_dmamap_load(sc->rl_ldata.rl_stag, sc->rl_ldata.rl_smap,
1179             sc->rl_ldata.rl_stats, sizeof(struct rl_stats), re_dma_map_addr,
1180              &sc->rl_ldata.rl_stats_addr, BUS_DMA_NOWAIT);
1181         if (error != 0 || sc->rl_ldata.rl_stats_addr == 0) {
1182                 device_printf(dev, "could not load statistics DMA memory\n");
1183                 return (ENOMEM);
1184         }
1185
1186         return (0);
1187 }
1188
1189 /*
1190  * Attach the interface. Allocate softc structures, do ifmedia
1191  * setup and ethernet/BPF attach.
1192  */
1193 static int
1194 re_attach(device_t dev)
1195 {
1196         u_char                  eaddr[ETHER_ADDR_LEN];
1197         u_int16_t               as[ETHER_ADDR_LEN / 2];
1198         struct rl_softc         *sc;
1199         struct ifnet            *ifp;
1200         const struct rl_hwrev   *hw_rev;
1201         u_int32_t               cap, ctl;
1202         int                     hwrev;
1203         u_int16_t               devid, re_did = 0;
1204         int                     error = 0, i, phy, rid;
1205         int                     msic, msixc, reg;
1206         uint8_t                 cfg;
1207
1208         sc = device_get_softc(dev);
1209         sc->rl_dev = dev;
1210
1211         mtx_init(&sc->rl_mtx, device_get_nameunit(dev), MTX_NETWORK_LOCK,
1212             MTX_DEF);
1213         callout_init_mtx(&sc->rl_stat_callout, &sc->rl_mtx, 0);
1214
1215         /*
1216          * Map control/status registers.
1217          */
1218         pci_enable_busmaster(dev);
1219
1220         devid = pci_get_device(dev);
1221         /*
1222          * Prefer memory space register mapping over IO space.
1223          * Because RTL8169SC does not seem to work when memory mapping
1224          * is used always activate io mapping.
1225          */
1226         if (devid == RT_DEVICEID_8169SC)
1227                 prefer_iomap = 1;
1228         if (prefer_iomap == 0) {
1229                 sc->rl_res_id = PCIR_BAR(1);
1230                 sc->rl_res_type = SYS_RES_MEMORY;
1231                 /* RTL8168/8101E seems to use different BARs. */
1232                 if (devid == RT_DEVICEID_8168 || devid == RT_DEVICEID_8101E)
1233                         sc->rl_res_id = PCIR_BAR(2);
1234         } else {
1235                 sc->rl_res_id = PCIR_BAR(0);
1236                 sc->rl_res_type = SYS_RES_IOPORT;
1237         }
1238         sc->rl_res = bus_alloc_resource_any(dev, sc->rl_res_type,
1239             &sc->rl_res_id, RF_ACTIVE);
1240         if (sc->rl_res == NULL && prefer_iomap == 0) {
1241                 sc->rl_res_id = PCIR_BAR(0);
1242                 sc->rl_res_type = SYS_RES_IOPORT;
1243                 sc->rl_res = bus_alloc_resource_any(dev, sc->rl_res_type,
1244                     &sc->rl_res_id, RF_ACTIVE);
1245         }
1246         if (sc->rl_res == NULL) {
1247                 device_printf(dev, "couldn't map ports/memory\n");
1248                 error = ENXIO;
1249                 goto fail;
1250         }
1251
1252         sc->rl_btag = rman_get_bustag(sc->rl_res);
1253         sc->rl_bhandle = rman_get_bushandle(sc->rl_res);
1254
1255         msic = pci_msi_count(dev);
1256         msixc = pci_msix_count(dev);
1257         if (pci_find_cap(dev, PCIY_EXPRESS, &reg) == 0) {
1258                 sc->rl_flags |= RL_FLAG_PCIE;
1259                 sc->rl_expcap = reg;
1260         }
1261         if (bootverbose) {
1262                 device_printf(dev, "MSI count : %d\n", msic);
1263                 device_printf(dev, "MSI-X count : %d\n", msixc);
1264         }
1265         if (msix_disable > 0)
1266                 msixc = 0;
1267         if (msi_disable > 0)
1268                 msic = 0;
1269         /* Prefer MSI-X to MSI. */
1270         if (msixc > 0) {
1271                 msixc = RL_MSI_MESSAGES;
1272                 rid = PCIR_BAR(4);
1273                 sc->rl_res_pba = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
1274                     &rid, RF_ACTIVE);
1275                 if (sc->rl_res_pba == NULL) {
1276                         device_printf(sc->rl_dev,
1277                             "could not allocate MSI-X PBA resource\n");
1278                 }
1279                 if (sc->rl_res_pba != NULL &&
1280                     pci_alloc_msix(dev, &msixc) == 0) {
1281                         if (msixc == RL_MSI_MESSAGES) {
1282                                 device_printf(dev, "Using %d MSI-X message\n",
1283                                     msixc);
1284                                 sc->rl_flags |= RL_FLAG_MSIX;
1285                         } else
1286                                 pci_release_msi(dev);
1287                 }
1288                 if ((sc->rl_flags & RL_FLAG_MSIX) == 0) {
1289                         if (sc->rl_res_pba != NULL)
1290                                 bus_release_resource(dev, SYS_RES_MEMORY, rid,
1291                                     sc->rl_res_pba);
1292                         sc->rl_res_pba = NULL;
1293                         msixc = 0;
1294                 }
1295         }
1296         /* Prefer MSI to INTx. */
1297         if (msixc == 0 && msic > 0) {
1298                 msic = RL_MSI_MESSAGES;
1299                 if (pci_alloc_msi(dev, &msic) == 0) {
1300                         if (msic == RL_MSI_MESSAGES) {
1301                                 device_printf(dev, "Using %d MSI message\n",
1302                                     msic);
1303                                 sc->rl_flags |= RL_FLAG_MSI;
1304                                 /* Explicitly set MSI enable bit. */
1305                                 CSR_WRITE_1(sc, RL_EECMD, RL_EE_MODE);
1306                                 cfg = CSR_READ_1(sc, RL_CFG2);
1307                                 cfg |= RL_CFG2_MSI;
1308                                 CSR_WRITE_1(sc, RL_CFG2, cfg);
1309                                 CSR_WRITE_1(sc, RL_EECMD, RL_EEMODE_OFF);
1310                         } else
1311                                 pci_release_msi(dev);
1312                 }
1313                 if ((sc->rl_flags & RL_FLAG_MSI) == 0)
1314                         msic = 0;
1315         }
1316
1317         /* Allocate interrupt */
1318         if ((sc->rl_flags & (RL_FLAG_MSI | RL_FLAG_MSIX)) == 0) {
1319                 rid = 0;
1320                 sc->rl_irq[0] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
1321                     RF_SHAREABLE | RF_ACTIVE);
1322                 if (sc->rl_irq[0] == NULL) {
1323                         device_printf(dev, "couldn't allocate IRQ resources\n");
1324                         error = ENXIO;
1325                         goto fail;
1326                 }
1327         } else {
1328                 for (i = 0, rid = 1; i < RL_MSI_MESSAGES; i++, rid++) {
1329                         sc->rl_irq[i] = bus_alloc_resource_any(dev,
1330                             SYS_RES_IRQ, &rid, RF_ACTIVE);
1331                         if (sc->rl_irq[i] == NULL) {
1332                                 device_printf(dev,
1333                                     "couldn't allocate IRQ resources for "
1334                                     "message %d\n", rid);
1335                                 error = ENXIO;
1336                                 goto fail;
1337                         }
1338                 }
1339         }
1340
1341         if ((sc->rl_flags & RL_FLAG_MSI) == 0) {
1342                 CSR_WRITE_1(sc, RL_EECMD, RL_EE_MODE);
1343                 cfg = CSR_READ_1(sc, RL_CFG2);
1344                 if ((cfg & RL_CFG2_MSI) != 0) {
1345                         device_printf(dev, "turning off MSI enable bit.\n");
1346                         cfg &= ~RL_CFG2_MSI;
1347                         CSR_WRITE_1(sc, RL_CFG2, cfg);
1348                 }
1349                 CSR_WRITE_1(sc, RL_EECMD, RL_EEMODE_OFF);
1350         }
1351
1352         /* Disable ASPM L0S/L1. */
1353         if (sc->rl_expcap != 0) {
1354                 cap = pci_read_config(dev, sc->rl_expcap +
1355                     PCIER_LINK_CAP, 2);
1356                 if ((cap & PCIEM_LINK_CAP_ASPM) != 0) {
1357                         ctl = pci_read_config(dev, sc->rl_expcap +
1358                             PCIER_LINK_CTL, 2);
1359                         if ((ctl & PCIEM_LINK_CTL_ASPMC) != 0) {
1360                                 ctl &= ~PCIEM_LINK_CTL_ASPMC;
1361                                 pci_write_config(dev, sc->rl_expcap +
1362                                     PCIER_LINK_CTL, ctl, 2);
1363                                 device_printf(dev, "ASPM disabled\n");
1364                         }
1365                 } else
1366                         device_printf(dev, "no ASPM capability\n");
1367         }
1368
1369         hw_rev = re_hwrevs;
1370         hwrev = CSR_READ_4(sc, RL_TXCFG);
1371         switch (hwrev & 0x70000000) {
1372         case 0x00000000:
1373         case 0x10000000:
1374                 device_printf(dev, "Chip rev. 0x%08x\n", hwrev & 0xfc800000);
1375                 hwrev &= (RL_TXCFG_HWREV | 0x80000000);
1376                 break;
1377         default:
1378                 device_printf(dev, "Chip rev. 0x%08x\n", hwrev & 0x7c800000);
1379                 sc->rl_macrev = hwrev & 0x00700000;
1380                 hwrev &= RL_TXCFG_HWREV;
1381                 break;
1382         }
1383         device_printf(dev, "MAC rev. 0x%08x\n", sc->rl_macrev);
1384         while (hw_rev->rl_desc != NULL) {
1385                 if (hw_rev->rl_rev == hwrev) {
1386                         sc->rl_type = hw_rev->rl_type;
1387                         sc->rl_hwrev = hw_rev;
1388                         break;
1389                 }
1390                 hw_rev++;
1391         }
1392         if (hw_rev->rl_desc == NULL) {
1393                 device_printf(dev, "Unknown H/W revision: 0x%08x\n", hwrev);
1394                 error = ENXIO;
1395                 goto fail;
1396         }
1397
1398         switch (hw_rev->rl_rev) {
1399         case RL_HWREV_8139CPLUS:
1400                 sc->rl_flags |= RL_FLAG_FASTETHER | RL_FLAG_AUTOPAD;
1401                 break;
1402         case RL_HWREV_8100E:
1403         case RL_HWREV_8101E:
1404                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_FASTETHER;
1405                 break;
1406         case RL_HWREV_8102E:
1407         case RL_HWREV_8102EL:
1408         case RL_HWREV_8102EL_SPIN1:
1409                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PAR | RL_FLAG_DESCV2 |
1410                     RL_FLAG_MACSTAT | RL_FLAG_FASTETHER | RL_FLAG_CMDSTOP |
1411                     RL_FLAG_AUTOPAD;
1412                 break;
1413         case RL_HWREV_8103E:
1414                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PAR | RL_FLAG_DESCV2 |
1415                     RL_FLAG_MACSTAT | RL_FLAG_FASTETHER | RL_FLAG_CMDSTOP |
1416                     RL_FLAG_AUTOPAD | RL_FLAG_MACSLEEP;
1417                 break;
1418         case RL_HWREV_8401E:
1419         case RL_HWREV_8105E:
1420         case RL_HWREV_8105E_SPIN1:
1421         case RL_HWREV_8106E:
1422                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PHYWAKE_PM |
1423                     RL_FLAG_PAR | RL_FLAG_DESCV2 | RL_FLAG_MACSTAT |
1424                     RL_FLAG_FASTETHER | RL_FLAG_CMDSTOP | RL_FLAG_AUTOPAD;
1425                 break;
1426         case RL_HWREV_8402:
1427                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PHYWAKE_PM |
1428                     RL_FLAG_PAR | RL_FLAG_DESCV2 | RL_FLAG_MACSTAT |
1429                     RL_FLAG_FASTETHER | RL_FLAG_CMDSTOP | RL_FLAG_AUTOPAD |
1430                     RL_FLAG_CMDSTOP_WAIT_TXQ;
1431                 break;
1432         case RL_HWREV_8168B_SPIN1:
1433         case RL_HWREV_8168B_SPIN2:
1434                 sc->rl_flags |= RL_FLAG_WOLRXENB;
1435                 /* FALLTHROUGH */
1436         case RL_HWREV_8168B_SPIN3:
1437                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_MACSTAT;
1438                 break;
1439         case RL_HWREV_8168C_SPIN2:
1440                 sc->rl_flags |= RL_FLAG_MACSLEEP;
1441                 /* FALLTHROUGH */
1442         case RL_HWREV_8168C:
1443                 if (sc->rl_macrev == 0x00200000)
1444                         sc->rl_flags |= RL_FLAG_MACSLEEP;
1445                 /* FALLTHROUGH */
1446         case RL_HWREV_8168CP:
1447                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PAR |
1448                     RL_FLAG_DESCV2 | RL_FLAG_MACSTAT | RL_FLAG_CMDSTOP |
1449                     RL_FLAG_AUTOPAD | RL_FLAG_JUMBOV2 | RL_FLAG_WOL_MANLINK;
1450                 break;
1451         case RL_HWREV_8168D:
1452                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PHYWAKE_PM |
1453                     RL_FLAG_PAR | RL_FLAG_DESCV2 | RL_FLAG_MACSTAT |
1454                     RL_FLAG_CMDSTOP | RL_FLAG_AUTOPAD | RL_FLAG_JUMBOV2 |
1455                     RL_FLAG_WOL_MANLINK;
1456                 break;
1457         case RL_HWREV_8168DP:
1458                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PAR |
1459                     RL_FLAG_DESCV2 | RL_FLAG_MACSTAT | RL_FLAG_AUTOPAD |
1460                     RL_FLAG_JUMBOV2 | RL_FLAG_WAIT_TXPOLL | RL_FLAG_WOL_MANLINK;
1461                 break;
1462         case RL_HWREV_8168E:
1463                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PHYWAKE_PM |
1464                     RL_FLAG_PAR | RL_FLAG_DESCV2 | RL_FLAG_MACSTAT |
1465                     RL_FLAG_CMDSTOP | RL_FLAG_AUTOPAD | RL_FLAG_JUMBOV2 |
1466                     RL_FLAG_WOL_MANLINK;
1467                 break;
1468         case RL_HWREV_8168E_VL:
1469         case RL_HWREV_8168F:
1470                 sc->rl_flags |= RL_FLAG_EARLYOFF;
1471                 /* FALLTHROUGH */
1472         case RL_HWREV_8411:
1473                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PAR |
1474                     RL_FLAG_DESCV2 | RL_FLAG_MACSTAT | RL_FLAG_CMDSTOP |
1475                     RL_FLAG_AUTOPAD | RL_FLAG_JUMBOV2 |
1476                     RL_FLAG_CMDSTOP_WAIT_TXQ | RL_FLAG_WOL_MANLINK;
1477                 break;
1478         case RL_HWREV_8168EP:
1479         case RL_HWREV_8168G:
1480         case RL_HWREV_8411B:
1481                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PAR |
1482                     RL_FLAG_DESCV2 | RL_FLAG_MACSTAT | RL_FLAG_CMDSTOP |
1483                     RL_FLAG_AUTOPAD | RL_FLAG_JUMBOV2 |
1484                     RL_FLAG_CMDSTOP_WAIT_TXQ | RL_FLAG_WOL_MANLINK |
1485                     RL_FLAG_EARLYOFFV2 | RL_FLAG_RXDV_GATED;
1486                 break;
1487         case RL_HWREV_8168GU:
1488                 if (pci_get_device(dev) == RT_DEVICEID_8101E) {
1489                         /* RTL8106EUS */
1490                         sc->rl_flags |= RL_FLAG_FASTETHER;
1491                 } else
1492                         sc->rl_flags |= RL_FLAG_JUMBOV2 | RL_FLAG_WOL_MANLINK;
1493
1494                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PAR |
1495                     RL_FLAG_DESCV2 | RL_FLAG_MACSTAT | RL_FLAG_CMDSTOP |
1496                     RL_FLAG_AUTOPAD | RL_FLAG_CMDSTOP_WAIT_TXQ |
1497                     RL_FLAG_EARLYOFFV2 | RL_FLAG_RXDV_GATED;
1498                 break;
1499         case RL_HWREV_8169_8110SB:
1500         case RL_HWREV_8169_8110SBL:
1501         case RL_HWREV_8169_8110SC:
1502         case RL_HWREV_8169_8110SCE:
1503                 sc->rl_flags |= RL_FLAG_PHYWAKE;
1504                 /* FALLTHROUGH */
1505         case RL_HWREV_8169:
1506         case RL_HWREV_8169S:
1507         case RL_HWREV_8110S:
1508                 sc->rl_flags |= RL_FLAG_MACRESET;
1509                 break;
1510         default:
1511                 break;
1512         }
1513
1514         if (sc->rl_hwrev->rl_rev == RL_HWREV_8139CPLUS) {
1515                 sc->rl_cfg0 = RL_8139_CFG0;
1516                 sc->rl_cfg1 = RL_8139_CFG1;
1517                 sc->rl_cfg2 = 0;
1518                 sc->rl_cfg3 = RL_8139_CFG3;
1519                 sc->rl_cfg4 = RL_8139_CFG4;
1520                 sc->rl_cfg5 = RL_8139_CFG5;
1521         } else {
1522                 sc->rl_cfg0 = RL_CFG0;
1523                 sc->rl_cfg1 = RL_CFG1;
1524                 sc->rl_cfg2 = RL_CFG2;
1525                 sc->rl_cfg3 = RL_CFG3;
1526                 sc->rl_cfg4 = RL_CFG4;
1527                 sc->rl_cfg5 = RL_CFG5;
1528         }
1529
1530         /* Reset the adapter. */
1531         RL_LOCK(sc);
1532         re_reset(sc);
1533         RL_UNLOCK(sc);
1534
1535         /* Enable PME. */
1536         CSR_WRITE_1(sc, RL_EECMD, RL_EE_MODE);
1537         cfg = CSR_READ_1(sc, sc->rl_cfg1);
1538         cfg |= RL_CFG1_PME;
1539         CSR_WRITE_1(sc, sc->rl_cfg1, cfg);
1540         cfg = CSR_READ_1(sc, sc->rl_cfg5);
1541         cfg &= RL_CFG5_PME_STS;
1542         CSR_WRITE_1(sc, sc->rl_cfg5, cfg);
1543         CSR_WRITE_1(sc, RL_EECMD, RL_EEMODE_OFF);
1544
1545         if ((sc->rl_flags & RL_FLAG_PAR) != 0) {
1546                 /*
1547                  * XXX Should have a better way to extract station
1548                  * address from EEPROM.
1549                  */
1550                 for (i = 0; i < ETHER_ADDR_LEN; i++)
1551                         eaddr[i] = CSR_READ_1(sc, RL_IDR0 + i);
1552         } else {
1553                 sc->rl_eewidth = RL_9356_ADDR_LEN;
1554                 re_read_eeprom(sc, (caddr_t)&re_did, 0, 1);
1555                 if (re_did != 0x8129)
1556                         sc->rl_eewidth = RL_9346_ADDR_LEN;
1557
1558                 /*
1559                  * Get station address from the EEPROM.
1560                  */
1561                 re_read_eeprom(sc, (caddr_t)as, RL_EE_EADDR, 3);
1562                 for (i = 0; i < ETHER_ADDR_LEN / 2; i++)
1563                         as[i] = le16toh(as[i]);
1564                 bcopy(as, eaddr, ETHER_ADDR_LEN);
1565         }
1566
1567         if (sc->rl_type == RL_8169) {
1568                 /* Set RX length mask and number of descriptors. */
1569                 sc->rl_rxlenmask = RL_RDESC_STAT_GFRAGLEN;
1570                 sc->rl_txstart = RL_GTXSTART;
1571                 sc->rl_ldata.rl_tx_desc_cnt = RL_8169_TX_DESC_CNT;
1572                 sc->rl_ldata.rl_rx_desc_cnt = RL_8169_RX_DESC_CNT;
1573         } else {
1574                 /* Set RX length mask and number of descriptors. */
1575                 sc->rl_rxlenmask = RL_RDESC_STAT_FRAGLEN;
1576                 sc->rl_txstart = RL_TXSTART;
1577                 sc->rl_ldata.rl_tx_desc_cnt = RL_8139_TX_DESC_CNT;
1578                 sc->rl_ldata.rl_rx_desc_cnt = RL_8139_RX_DESC_CNT;
1579         }
1580
1581         error = re_allocmem(dev, sc);
1582         if (error)
1583                 goto fail;
1584         re_add_sysctls(sc);
1585
1586         ifp = sc->rl_ifp = if_alloc(IFT_ETHER);
1587         if (ifp == NULL) {
1588                 device_printf(dev, "can not if_alloc()\n");
1589                 error = ENOSPC;
1590                 goto fail;
1591         }
1592
1593         /* Take controller out of deep sleep mode. */
1594         if ((sc->rl_flags & RL_FLAG_MACSLEEP) != 0) {
1595                 if ((CSR_READ_1(sc, RL_MACDBG) & 0x80) == 0x80)
1596                         CSR_WRITE_1(sc, RL_GPIO,
1597                             CSR_READ_1(sc, RL_GPIO) | 0x01);
1598                 else
1599                         CSR_WRITE_1(sc, RL_GPIO,
1600                             CSR_READ_1(sc, RL_GPIO) & ~0x01);
1601         }
1602
1603         /* Take PHY out of power down mode. */
1604         if ((sc->rl_flags & RL_FLAG_PHYWAKE_PM) != 0) {
1605                 CSR_WRITE_1(sc, RL_PMCH, CSR_READ_1(sc, RL_PMCH) | 0x80);
1606                 if (hw_rev->rl_rev == RL_HWREV_8401E)
1607                         CSR_WRITE_1(sc, 0xD1, CSR_READ_1(sc, 0xD1) & ~0x08);
1608         }
1609         if ((sc->rl_flags & RL_FLAG_PHYWAKE) != 0) {
1610                 re_gmii_writereg(dev, 1, 0x1f, 0);
1611                 re_gmii_writereg(dev, 1, 0x0e, 0);
1612         }
1613
1614         ifp->if_softc = sc;
1615         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1616         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1617         ifp->if_ioctl = re_ioctl;
1618         ifp->if_start = re_start;
1619         /*
1620          * RTL8168/8111C generates wrong IP checksummed frame if the
1621          * packet has IP options so disable TX checksum offloading.
1622          */
1623         if (sc->rl_hwrev->rl_rev == RL_HWREV_8168C ||
1624             sc->rl_hwrev->rl_rev == RL_HWREV_8168C_SPIN2 ||
1625             sc->rl_hwrev->rl_rev == RL_HWREV_8168CP) {
1626                 ifp->if_hwassist = 0;
1627                 ifp->if_capabilities = IFCAP_RXCSUM | IFCAP_TSO4;
1628         } else {
1629                 ifp->if_hwassist = CSUM_IP | CSUM_TCP | CSUM_UDP;
1630                 ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_TSO4;
1631         }
1632         ifp->if_hwassist |= CSUM_TSO;
1633         ifp->if_capenable = ifp->if_capabilities;
1634         ifp->if_init = re_init;
1635         IFQ_SET_MAXLEN(&ifp->if_snd, RL_IFQ_MAXLEN);
1636         ifp->if_snd.ifq_drv_maxlen = RL_IFQ_MAXLEN;
1637         IFQ_SET_READY(&ifp->if_snd);
1638
1639         TASK_INIT(&sc->rl_inttask, 0, re_int_task, sc);
1640
1641 #define RE_PHYAD_INTERNAL        0
1642
1643         /* Do MII setup. */
1644         phy = RE_PHYAD_INTERNAL;
1645         if (sc->rl_type == RL_8169)
1646                 phy = 1;
1647         error = mii_attach(dev, &sc->rl_miibus, ifp, re_ifmedia_upd,
1648             re_ifmedia_sts, BMSR_DEFCAPMASK, phy, MII_OFFSET_ANY, MIIF_DOPAUSE);
1649         if (error != 0) {
1650                 device_printf(dev, "attaching PHYs failed\n");
1651                 goto fail;
1652         }
1653
1654         /*
1655          * Call MI attach routine.
1656          */
1657         ether_ifattach(ifp, eaddr);
1658
1659         /* VLAN capability setup */
1660         ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING;
1661         if (ifp->if_capabilities & IFCAP_HWCSUM)
1662                 ifp->if_capabilities |= IFCAP_VLAN_HWCSUM;
1663         /* Enable WOL if PM is supported. */
1664         if (pci_find_cap(sc->rl_dev, PCIY_PMG, &reg) == 0)
1665                 ifp->if_capabilities |= IFCAP_WOL;
1666         ifp->if_capenable = ifp->if_capabilities;
1667         ifp->if_capenable &= ~(IFCAP_WOL_UCAST | IFCAP_WOL_MCAST);
1668         /*
1669          * Don't enable TSO by default.  It is known to generate
1670          * corrupted TCP segments(bad TCP options) under certain
1671          * circumstances.
1672          */
1673         ifp->if_hwassist &= ~CSUM_TSO;
1674         ifp->if_capenable &= ~(IFCAP_TSO4 | IFCAP_VLAN_HWTSO);
1675 #ifdef DEVICE_POLLING
1676         ifp->if_capabilities |= IFCAP_POLLING;
1677 #endif
1678         /*
1679          * Tell the upper layer(s) we support long frames.
1680          * Must appear after the call to ether_ifattach() because
1681          * ether_ifattach() sets ifi_hdrlen to the default value.
1682          */
1683         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
1684
1685 #ifdef DEV_NETMAP
1686         re_netmap_attach(sc);
1687 #endif /* DEV_NETMAP */
1688 #ifdef RE_DIAG
1689         /*
1690          * Perform hardware diagnostic on the original RTL8169.
1691          * Some 32-bit cards were incorrectly wired and would
1692          * malfunction if plugged into a 64-bit slot.
1693          */
1694
1695         if (hwrev == RL_HWREV_8169) {
1696                 error = re_diag(sc);
1697                 if (error) {
1698                         device_printf(dev,
1699                         "attach aborted due to hardware diag failure\n");
1700                         ether_ifdetach(ifp);
1701                         goto fail;
1702                 }
1703         }
1704 #endif
1705
1706 #ifdef RE_TX_MODERATION
1707         intr_filter = 1;
1708 #endif
1709         /* Hook interrupt last to avoid having to lock softc */
1710         if ((sc->rl_flags & (RL_FLAG_MSI | RL_FLAG_MSIX)) != 0 &&
1711             intr_filter == 0) {
1712                 error = bus_setup_intr(dev, sc->rl_irq[0],
1713                     INTR_TYPE_NET | INTR_MPSAFE, NULL, re_intr_msi, sc,
1714                     &sc->rl_intrhand[0]);
1715         } else {
1716                 error = bus_setup_intr(dev, sc->rl_irq[0],
1717                     INTR_TYPE_NET | INTR_MPSAFE, re_intr, NULL, sc,
1718                     &sc->rl_intrhand[0]);
1719         }
1720         if (error) {
1721                 device_printf(dev, "couldn't set up irq\n");
1722                 ether_ifdetach(ifp);
1723         }
1724
1725 fail:
1726
1727         if (error)
1728                 re_detach(dev);
1729
1730         return (error);
1731 }
1732
1733 /*
1734  * Shutdown hardware and free up resources. This can be called any
1735  * time after the mutex has been initialized. It is called in both
1736  * the error case in attach and the normal detach case so it needs
1737  * to be careful about only freeing resources that have actually been
1738  * allocated.
1739  */
1740 static int
1741 re_detach(device_t dev)
1742 {
1743         struct rl_softc         *sc;
1744         struct ifnet            *ifp;
1745         int                     i, rid;
1746
1747         sc = device_get_softc(dev);
1748         ifp = sc->rl_ifp;
1749         KASSERT(mtx_initialized(&sc->rl_mtx), ("re mutex not initialized"));
1750
1751         /* These should only be active if attach succeeded */
1752         if (device_is_attached(dev)) {
1753 #ifdef DEVICE_POLLING
1754                 if (ifp->if_capenable & IFCAP_POLLING)
1755                         ether_poll_deregister(ifp);
1756 #endif
1757                 RL_LOCK(sc);
1758 #if 0
1759                 sc->suspended = 1;
1760 #endif
1761                 re_stop(sc);
1762                 RL_UNLOCK(sc);
1763                 callout_drain(&sc->rl_stat_callout);
1764                 taskqueue_drain(taskqueue_fast, &sc->rl_inttask);
1765                 /*
1766                  * Force off the IFF_UP flag here, in case someone
1767                  * still had a BPF descriptor attached to this
1768                  * interface. If they do, ether_ifdetach() will cause
1769                  * the BPF code to try and clear the promisc mode
1770                  * flag, which will bubble down to re_ioctl(),
1771                  * which will try to call re_init() again. This will
1772                  * turn the NIC back on and restart the MII ticker,
1773                  * which will panic the system when the kernel tries
1774                  * to invoke the re_tick() function that isn't there
1775                  * anymore.
1776                  */
1777                 ifp->if_flags &= ~IFF_UP;
1778                 ether_ifdetach(ifp);
1779         }
1780         if (sc->rl_miibus)
1781                 device_delete_child(dev, sc->rl_miibus);
1782         bus_generic_detach(dev);
1783
1784         /*
1785          * The rest is resource deallocation, so we should already be
1786          * stopped here.
1787          */
1788
1789         if (sc->rl_intrhand[0] != NULL) {
1790                 bus_teardown_intr(dev, sc->rl_irq[0], sc->rl_intrhand[0]);
1791                 sc->rl_intrhand[0] = NULL;
1792         }
1793         if (ifp != NULL) {
1794 #ifdef DEV_NETMAP
1795                 netmap_detach(ifp);
1796 #endif /* DEV_NETMAP */
1797                 if_free(ifp);
1798         }
1799         if ((sc->rl_flags & (RL_FLAG_MSI | RL_FLAG_MSIX)) == 0)
1800                 rid = 0;
1801         else
1802                 rid = 1;
1803         if (sc->rl_irq[0] != NULL) {
1804                 bus_release_resource(dev, SYS_RES_IRQ, rid, sc->rl_irq[0]);
1805                 sc->rl_irq[0] = NULL;
1806         }
1807         if ((sc->rl_flags & (RL_FLAG_MSI | RL_FLAG_MSIX)) != 0)
1808                 pci_release_msi(dev);
1809         if (sc->rl_res_pba) {
1810                 rid = PCIR_BAR(4);
1811                 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->rl_res_pba);
1812         }
1813         if (sc->rl_res)
1814                 bus_release_resource(dev, sc->rl_res_type, sc->rl_res_id,
1815                     sc->rl_res);
1816
1817         /* Unload and free the RX DMA ring memory and map */
1818
1819         if (sc->rl_ldata.rl_rx_list_tag) {
1820                 if (sc->rl_ldata.rl_rx_list_map)
1821                         bus_dmamap_unload(sc->rl_ldata.rl_rx_list_tag,
1822                             sc->rl_ldata.rl_rx_list_map);
1823                 if (sc->rl_ldata.rl_rx_list_map && sc->rl_ldata.rl_rx_list)
1824                         bus_dmamem_free(sc->rl_ldata.rl_rx_list_tag,
1825                             sc->rl_ldata.rl_rx_list,
1826                             sc->rl_ldata.rl_rx_list_map);
1827                 bus_dma_tag_destroy(sc->rl_ldata.rl_rx_list_tag);
1828         }
1829
1830         /* Unload and free the TX DMA ring memory and map */
1831
1832         if (sc->rl_ldata.rl_tx_list_tag) {
1833                 if (sc->rl_ldata.rl_tx_list_map)
1834                         bus_dmamap_unload(sc->rl_ldata.rl_tx_list_tag,
1835                             sc->rl_ldata.rl_tx_list_map);
1836                 if (sc->rl_ldata.rl_tx_list_map && sc->rl_ldata.rl_tx_list)
1837                         bus_dmamem_free(sc->rl_ldata.rl_tx_list_tag,
1838                             sc->rl_ldata.rl_tx_list,
1839                             sc->rl_ldata.rl_tx_list_map);
1840                 bus_dma_tag_destroy(sc->rl_ldata.rl_tx_list_tag);
1841         }
1842
1843         /* Destroy all the RX and TX buffer maps */
1844
1845         if (sc->rl_ldata.rl_tx_mtag) {
1846                 for (i = 0; i < sc->rl_ldata.rl_tx_desc_cnt; i++) {
1847                         if (sc->rl_ldata.rl_tx_desc[i].tx_dmamap)
1848                                 bus_dmamap_destroy(sc->rl_ldata.rl_tx_mtag,
1849                                     sc->rl_ldata.rl_tx_desc[i].tx_dmamap);
1850                 }
1851                 bus_dma_tag_destroy(sc->rl_ldata.rl_tx_mtag);
1852         }
1853         if (sc->rl_ldata.rl_rx_mtag) {
1854                 for (i = 0; i < sc->rl_ldata.rl_rx_desc_cnt; i++) {
1855                         if (sc->rl_ldata.rl_rx_desc[i].rx_dmamap)
1856                                 bus_dmamap_destroy(sc->rl_ldata.rl_rx_mtag,
1857                                     sc->rl_ldata.rl_rx_desc[i].rx_dmamap);
1858                 }
1859                 if (sc->rl_ldata.rl_rx_sparemap)
1860                         bus_dmamap_destroy(sc->rl_ldata.rl_rx_mtag,
1861                             sc->rl_ldata.rl_rx_sparemap);
1862                 bus_dma_tag_destroy(sc->rl_ldata.rl_rx_mtag);
1863         }
1864         if (sc->rl_ldata.rl_jrx_mtag) {
1865                 for (i = 0; i < sc->rl_ldata.rl_rx_desc_cnt; i++) {
1866                         if (sc->rl_ldata.rl_jrx_desc[i].rx_dmamap)
1867                                 bus_dmamap_destroy(sc->rl_ldata.rl_jrx_mtag,
1868                                     sc->rl_ldata.rl_jrx_desc[i].rx_dmamap);
1869                 }
1870                 if (sc->rl_ldata.rl_jrx_sparemap)
1871                         bus_dmamap_destroy(sc->rl_ldata.rl_jrx_mtag,
1872                             sc->rl_ldata.rl_jrx_sparemap);
1873                 bus_dma_tag_destroy(sc->rl_ldata.rl_jrx_mtag);
1874         }
1875         /* Unload and free the stats buffer and map */
1876
1877         if (sc->rl_ldata.rl_stag) {
1878                 if (sc->rl_ldata.rl_smap)
1879                         bus_dmamap_unload(sc->rl_ldata.rl_stag,
1880                             sc->rl_ldata.rl_smap);
1881                 if (sc->rl_ldata.rl_smap && sc->rl_ldata.rl_stats)
1882                         bus_dmamem_free(sc->rl_ldata.rl_stag,
1883                             sc->rl_ldata.rl_stats, sc->rl_ldata.rl_smap);
1884                 bus_dma_tag_destroy(sc->rl_ldata.rl_stag);
1885         }
1886
1887         if (sc->rl_parent_tag)
1888                 bus_dma_tag_destroy(sc->rl_parent_tag);
1889
1890         mtx_destroy(&sc->rl_mtx);
1891
1892         return (0);
1893 }
1894
1895 static __inline void
1896 re_discard_rxbuf(struct rl_softc *sc, int idx)
1897 {
1898         struct rl_desc          *desc;
1899         struct rl_rxdesc        *rxd;
1900         uint32_t                cmdstat;
1901
1902         if (sc->rl_ifp->if_mtu > RL_MTU &&
1903             (sc->rl_flags & RL_FLAG_JUMBOV2) != 0)
1904                 rxd = &sc->rl_ldata.rl_jrx_desc[idx];
1905         else
1906                 rxd = &sc->rl_ldata.rl_rx_desc[idx];
1907         desc = &sc->rl_ldata.rl_rx_list[idx];
1908         desc->rl_vlanctl = 0;
1909         cmdstat = rxd->rx_size;
1910         if (idx == sc->rl_ldata.rl_rx_desc_cnt - 1)
1911                 cmdstat |= RL_RDESC_CMD_EOR;
1912         desc->rl_cmdstat = htole32(cmdstat | RL_RDESC_CMD_OWN);
1913 }
1914
1915 static int
1916 re_newbuf(struct rl_softc *sc, int idx)
1917 {
1918         struct mbuf             *m;
1919         struct rl_rxdesc        *rxd;
1920         bus_dma_segment_t       segs[1];
1921         bus_dmamap_t            map;
1922         struct rl_desc          *desc;
1923         uint32_t                cmdstat;
1924         int                     error, nsegs;
1925
1926         m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
1927         if (m == NULL)
1928                 return (ENOBUFS);
1929
1930         m->m_len = m->m_pkthdr.len = MCLBYTES;
1931 #ifdef RE_FIXUP_RX
1932         /*
1933          * This is part of an evil trick to deal with non-x86 platforms.
1934          * The RealTek chip requires RX buffers to be aligned on 64-bit
1935          * boundaries, but that will hose non-x86 machines. To get around
1936          * this, we leave some empty space at the start of each buffer
1937          * and for non-x86 hosts, we copy the buffer back six bytes
1938          * to achieve word alignment. This is slightly more efficient
1939          * than allocating a new buffer, copying the contents, and
1940          * discarding the old buffer.
1941          */
1942         m_adj(m, RE_ETHER_ALIGN);
1943 #endif
1944         error = bus_dmamap_load_mbuf_sg(sc->rl_ldata.rl_rx_mtag,
1945             sc->rl_ldata.rl_rx_sparemap, m, segs, &nsegs, BUS_DMA_NOWAIT);
1946         if (error != 0) {
1947                 m_freem(m);
1948                 return (ENOBUFS);
1949         }
1950         KASSERT(nsegs == 1, ("%s: %d segment returned!", __func__, nsegs));
1951
1952         rxd = &sc->rl_ldata.rl_rx_desc[idx];
1953         if (rxd->rx_m != NULL) {
1954                 bus_dmamap_sync(sc->rl_ldata.rl_rx_mtag, rxd->rx_dmamap,
1955                     BUS_DMASYNC_POSTREAD);
1956                 bus_dmamap_unload(sc->rl_ldata.rl_rx_mtag, rxd->rx_dmamap);
1957         }
1958
1959         rxd->rx_m = m;
1960         map = rxd->rx_dmamap;
1961         rxd->rx_dmamap = sc->rl_ldata.rl_rx_sparemap;
1962         rxd->rx_size = segs[0].ds_len;
1963         sc->rl_ldata.rl_rx_sparemap = map;
1964         bus_dmamap_sync(sc->rl_ldata.rl_rx_mtag, rxd->rx_dmamap,
1965             BUS_DMASYNC_PREREAD);
1966
1967         desc = &sc->rl_ldata.rl_rx_list[idx];
1968         desc->rl_vlanctl = 0;
1969         desc->rl_bufaddr_lo = htole32(RL_ADDR_LO(segs[0].ds_addr));
1970         desc->rl_bufaddr_hi = htole32(RL_ADDR_HI(segs[0].ds_addr));
1971         cmdstat = segs[0].ds_len;
1972         if (idx == sc->rl_ldata.rl_rx_desc_cnt - 1)
1973                 cmdstat |= RL_RDESC_CMD_EOR;
1974         desc->rl_cmdstat = htole32(cmdstat | RL_RDESC_CMD_OWN);
1975
1976         return (0);
1977 }
1978
1979 static int
1980 re_jumbo_newbuf(struct rl_softc *sc, int idx)
1981 {
1982         struct mbuf             *m;
1983         struct rl_rxdesc        *rxd;
1984         bus_dma_segment_t       segs[1];
1985         bus_dmamap_t            map;
1986         struct rl_desc          *desc;
1987         uint32_t                cmdstat;
1988         int                     error, nsegs;
1989
1990         m = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, MJUM9BYTES);
1991         if (m == NULL)
1992                 return (ENOBUFS);
1993         m->m_len = m->m_pkthdr.len = MJUM9BYTES;
1994 #ifdef RE_FIXUP_RX
1995         m_adj(m, RE_ETHER_ALIGN);
1996 #endif
1997         error = bus_dmamap_load_mbuf_sg(sc->rl_ldata.rl_jrx_mtag,
1998             sc->rl_ldata.rl_jrx_sparemap, m, segs, &nsegs, BUS_DMA_NOWAIT);
1999         if (error != 0) {
2000                 m_freem(m);
2001                 return (ENOBUFS);
2002         }
2003         KASSERT(nsegs == 1, ("%s: %d segment returned!", __func__, nsegs));
2004
2005         rxd = &sc->rl_ldata.rl_jrx_desc[idx];
2006         if (rxd->rx_m != NULL) {
2007                 bus_dmamap_sync(sc->rl_ldata.rl_jrx_mtag, rxd->rx_dmamap,
2008                     BUS_DMASYNC_POSTREAD);
2009                 bus_dmamap_unload(sc->rl_ldata.rl_jrx_mtag, rxd->rx_dmamap);
2010         }
2011
2012         rxd->rx_m = m;
2013         map = rxd->rx_dmamap;
2014         rxd->rx_dmamap = sc->rl_ldata.rl_jrx_sparemap;
2015         rxd->rx_size = segs[0].ds_len;
2016         sc->rl_ldata.rl_jrx_sparemap = map;
2017         bus_dmamap_sync(sc->rl_ldata.rl_jrx_mtag, rxd->rx_dmamap,
2018             BUS_DMASYNC_PREREAD);
2019
2020         desc = &sc->rl_ldata.rl_rx_list[idx];
2021         desc->rl_vlanctl = 0;
2022         desc->rl_bufaddr_lo = htole32(RL_ADDR_LO(segs[0].ds_addr));
2023         desc->rl_bufaddr_hi = htole32(RL_ADDR_HI(segs[0].ds_addr));
2024         cmdstat = segs[0].ds_len;
2025         if (idx == sc->rl_ldata.rl_rx_desc_cnt - 1)
2026                 cmdstat |= RL_RDESC_CMD_EOR;
2027         desc->rl_cmdstat = htole32(cmdstat | RL_RDESC_CMD_OWN);
2028
2029         return (0);
2030 }
2031
2032 #ifdef RE_FIXUP_RX
2033 static __inline void
2034 re_fixup_rx(struct mbuf *m)
2035 {
2036         int                     i;
2037         uint16_t                *src, *dst;
2038
2039         src = mtod(m, uint16_t *);
2040         dst = src - (RE_ETHER_ALIGN - ETHER_ALIGN) / sizeof *src;
2041
2042         for (i = 0; i < (m->m_len / sizeof(uint16_t) + 1); i++)
2043                 *dst++ = *src++;
2044
2045         m->m_data -= RE_ETHER_ALIGN - ETHER_ALIGN;
2046 }
2047 #endif
2048
2049 static int
2050 re_tx_list_init(struct rl_softc *sc)
2051 {
2052         struct rl_desc          *desc;
2053         int                     i;
2054
2055         RL_LOCK_ASSERT(sc);
2056
2057         bzero(sc->rl_ldata.rl_tx_list,
2058             sc->rl_ldata.rl_tx_desc_cnt * sizeof(struct rl_desc));
2059         for (i = 0; i < sc->rl_ldata.rl_tx_desc_cnt; i++)
2060                 sc->rl_ldata.rl_tx_desc[i].tx_m = NULL;
2061 #ifdef DEV_NETMAP
2062         re_netmap_tx_init(sc);
2063 #endif /* DEV_NETMAP */
2064         /* Set EOR. */
2065         desc = &sc->rl_ldata.rl_tx_list[sc->rl_ldata.rl_tx_desc_cnt - 1];
2066         desc->rl_cmdstat |= htole32(RL_TDESC_CMD_EOR);
2067
2068         bus_dmamap_sync(sc->rl_ldata.rl_tx_list_tag,
2069             sc->rl_ldata.rl_tx_list_map,
2070             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2071
2072         sc->rl_ldata.rl_tx_prodidx = 0;
2073         sc->rl_ldata.rl_tx_considx = 0;
2074         sc->rl_ldata.rl_tx_free = sc->rl_ldata.rl_tx_desc_cnt;
2075
2076         return (0);
2077 }
2078
2079 static int
2080 re_rx_list_init(struct rl_softc *sc)
2081 {
2082         int                     error, i;
2083
2084         bzero(sc->rl_ldata.rl_rx_list,
2085             sc->rl_ldata.rl_rx_desc_cnt * sizeof(struct rl_desc));
2086         for (i = 0; i < sc->rl_ldata.rl_rx_desc_cnt; i++) {
2087                 sc->rl_ldata.rl_rx_desc[i].rx_m = NULL;
2088                 if ((error = re_newbuf(sc, i)) != 0)
2089                         return (error);
2090         }
2091 #ifdef DEV_NETMAP
2092         re_netmap_rx_init(sc);
2093 #endif /* DEV_NETMAP */
2094
2095         /* Flush the RX descriptors */
2096
2097         bus_dmamap_sync(sc->rl_ldata.rl_rx_list_tag,
2098             sc->rl_ldata.rl_rx_list_map,
2099             BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD);
2100
2101         sc->rl_ldata.rl_rx_prodidx = 0;
2102         sc->rl_head = sc->rl_tail = NULL;
2103         sc->rl_int_rx_act = 0;
2104
2105         return (0);
2106 }
2107
2108 static int
2109 re_jrx_list_init(struct rl_softc *sc)
2110 {
2111         int                     error, i;
2112
2113         bzero(sc->rl_ldata.rl_rx_list,
2114             sc->rl_ldata.rl_rx_desc_cnt * sizeof(struct rl_desc));
2115         for (i = 0; i < sc->rl_ldata.rl_rx_desc_cnt; i++) {
2116                 sc->rl_ldata.rl_jrx_desc[i].rx_m = NULL;
2117                 if ((error = re_jumbo_newbuf(sc, i)) != 0)
2118                         return (error);
2119         }
2120
2121         bus_dmamap_sync(sc->rl_ldata.rl_rx_list_tag,
2122             sc->rl_ldata.rl_rx_list_map,
2123             BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2124
2125         sc->rl_ldata.rl_rx_prodidx = 0;
2126         sc->rl_head = sc->rl_tail = NULL;
2127         sc->rl_int_rx_act = 0;
2128
2129         return (0);
2130 }
2131
2132 /*
2133  * RX handler for C+ and 8169. For the gigE chips, we support
2134  * the reception of jumbo frames that have been fragmented
2135  * across multiple 2K mbuf cluster buffers.
2136  */
2137 static int
2138 re_rxeof(struct rl_softc *sc, int *rx_npktsp)
2139 {
2140         struct mbuf             *m;
2141         struct ifnet            *ifp;
2142         int                     i, rxerr, total_len;
2143         struct rl_desc          *cur_rx;
2144         u_int32_t               rxstat, rxvlan;
2145         int                     jumbo, maxpkt = 16, rx_npkts = 0;
2146
2147         RL_LOCK_ASSERT(sc);
2148
2149         ifp = sc->rl_ifp;
2150 #ifdef DEV_NETMAP
2151         if (netmap_rx_irq(ifp, 0, &rx_npkts))
2152                 return 0;
2153 #endif /* DEV_NETMAP */
2154         if (ifp->if_mtu > RL_MTU && (sc->rl_flags & RL_FLAG_JUMBOV2) != 0)
2155                 jumbo = 1;
2156         else
2157                 jumbo = 0;
2158
2159         /* Invalidate the descriptor memory */
2160
2161         bus_dmamap_sync(sc->rl_ldata.rl_rx_list_tag,
2162             sc->rl_ldata.rl_rx_list_map,
2163             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2164
2165         for (i = sc->rl_ldata.rl_rx_prodidx; maxpkt > 0;
2166             i = RL_RX_DESC_NXT(sc, i)) {
2167                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
2168                         break;
2169                 cur_rx = &sc->rl_ldata.rl_rx_list[i];
2170                 rxstat = le32toh(cur_rx->rl_cmdstat);
2171                 if ((rxstat & RL_RDESC_STAT_OWN) != 0)
2172                         break;
2173                 total_len = rxstat & sc->rl_rxlenmask;
2174                 rxvlan = le32toh(cur_rx->rl_vlanctl);
2175                 if (jumbo != 0)
2176                         m = sc->rl_ldata.rl_jrx_desc[i].rx_m;
2177                 else
2178                         m = sc->rl_ldata.rl_rx_desc[i].rx_m;
2179
2180                 if ((sc->rl_flags & RL_FLAG_JUMBOV2) != 0 &&
2181                     (rxstat & (RL_RDESC_STAT_SOF | RL_RDESC_STAT_EOF)) !=
2182                     (RL_RDESC_STAT_SOF | RL_RDESC_STAT_EOF)) {
2183                         /*
2184                          * RTL8168C or later controllers do not
2185                          * support multi-fragment packet.
2186                          */
2187                         re_discard_rxbuf(sc, i);
2188                         continue;
2189                 } else if ((rxstat & RL_RDESC_STAT_EOF) == 0) {
2190                         if (re_newbuf(sc, i) != 0) {
2191                                 /*
2192                                  * If this is part of a multi-fragment packet,
2193                                  * discard all the pieces.
2194                                  */
2195                                 if (sc->rl_head != NULL) {
2196                                         m_freem(sc->rl_head);
2197                                         sc->rl_head = sc->rl_tail = NULL;
2198                                 }
2199                                 re_discard_rxbuf(sc, i);
2200                                 continue;
2201                         }
2202                         m->m_len = RE_RX_DESC_BUFLEN;
2203                         if (sc->rl_head == NULL)
2204                                 sc->rl_head = sc->rl_tail = m;
2205                         else {
2206                                 m->m_flags &= ~M_PKTHDR;
2207                                 sc->rl_tail->m_next = m;
2208                                 sc->rl_tail = m;
2209                         }
2210                         continue;
2211                 }
2212
2213                 /*
2214                  * NOTE: for the 8139C+, the frame length field
2215                  * is always 12 bits in size, but for the gigE chips,
2216                  * it is 13 bits (since the max RX frame length is 16K).
2217                  * Unfortunately, all 32 bits in the status word
2218                  * were already used, so to make room for the extra
2219                  * length bit, RealTek took out the 'frame alignment
2220                  * error' bit and shifted the other status bits
2221                  * over one slot. The OWN, EOR, FS and LS bits are
2222                  * still in the same places. We have already extracted
2223                  * the frame length and checked the OWN bit, so rather
2224                  * than using an alternate bit mapping, we shift the
2225                  * status bits one space to the right so we can evaluate
2226                  * them using the 8169 status as though it was in the
2227                  * same format as that of the 8139C+.
2228                  */
2229                 if (sc->rl_type == RL_8169)
2230                         rxstat >>= 1;
2231
2232                 /*
2233                  * if total_len > 2^13-1, both _RXERRSUM and _GIANT will be
2234                  * set, but if CRC is clear, it will still be a valid frame.
2235                  */
2236                 if ((rxstat & RL_RDESC_STAT_RXERRSUM) != 0) {
2237                         rxerr = 1;
2238                         if ((sc->rl_flags & RL_FLAG_JUMBOV2) == 0 &&
2239                             total_len > 8191 &&
2240                             (rxstat & RL_RDESC_STAT_ERRS) == RL_RDESC_STAT_GIANT)
2241                                 rxerr = 0;
2242                         if (rxerr != 0) {
2243                                 ifp->if_ierrors++;
2244                                 /*
2245                                  * If this is part of a multi-fragment packet,
2246                                  * discard all the pieces.
2247                                  */
2248                                 if (sc->rl_head != NULL) {
2249                                         m_freem(sc->rl_head);
2250                                         sc->rl_head = sc->rl_tail = NULL;
2251                                 }
2252                                 re_discard_rxbuf(sc, i);
2253                                 continue;
2254                         }
2255                 }
2256
2257                 /*
2258                  * If allocating a replacement mbuf fails,
2259                  * reload the current one.
2260                  */
2261                 if (jumbo != 0)
2262                         rxerr = re_jumbo_newbuf(sc, i);
2263                 else
2264                         rxerr = re_newbuf(sc, i);
2265                 if (rxerr != 0) {
2266                         ifp->if_iqdrops++;
2267                         if (sc->rl_head != NULL) {
2268                                 m_freem(sc->rl_head);
2269                                 sc->rl_head = sc->rl_tail = NULL;
2270                         }
2271                         re_discard_rxbuf(sc, i);
2272                         continue;
2273                 }
2274
2275                 if (sc->rl_head != NULL) {
2276                         if (jumbo != 0)
2277                                 m->m_len = total_len;
2278                         else {
2279                                 m->m_len = total_len % RE_RX_DESC_BUFLEN;
2280                                 if (m->m_len == 0)
2281                                         m->m_len = RE_RX_DESC_BUFLEN;
2282                         }
2283                         /*
2284                          * Special case: if there's 4 bytes or less
2285                          * in this buffer, the mbuf can be discarded:
2286                          * the last 4 bytes is the CRC, which we don't
2287                          * care about anyway.
2288                          */
2289                         if (m->m_len <= ETHER_CRC_LEN) {
2290                                 sc->rl_tail->m_len -=
2291                                     (ETHER_CRC_LEN - m->m_len);
2292                                 m_freem(m);
2293                         } else {
2294                                 m->m_len -= ETHER_CRC_LEN;
2295                                 m->m_flags &= ~M_PKTHDR;
2296                                 sc->rl_tail->m_next = m;
2297                         }
2298                         m = sc->rl_head;
2299                         sc->rl_head = sc->rl_tail = NULL;
2300                         m->m_pkthdr.len = total_len - ETHER_CRC_LEN;
2301                 } else
2302                         m->m_pkthdr.len = m->m_len =
2303                             (total_len - ETHER_CRC_LEN);
2304
2305 #ifdef RE_FIXUP_RX
2306                 re_fixup_rx(m);
2307 #endif
2308                 ifp->if_ipackets++;
2309                 m->m_pkthdr.rcvif = ifp;
2310
2311                 /* Do RX checksumming if enabled */
2312
2313                 if (ifp->if_capenable & IFCAP_RXCSUM) {
2314                         if ((sc->rl_flags & RL_FLAG_DESCV2) == 0) {
2315                                 /* Check IP header checksum */
2316                                 if (rxstat & RL_RDESC_STAT_PROTOID)
2317                                         m->m_pkthdr.csum_flags |=
2318                                             CSUM_IP_CHECKED;
2319                                 if (!(rxstat & RL_RDESC_STAT_IPSUMBAD))
2320                                         m->m_pkthdr.csum_flags |=
2321                                             CSUM_IP_VALID;
2322
2323                                 /* Check TCP/UDP checksum */
2324                                 if ((RL_TCPPKT(rxstat) &&
2325                                     !(rxstat & RL_RDESC_STAT_TCPSUMBAD)) ||
2326                                     (RL_UDPPKT(rxstat) &&
2327                                      !(rxstat & RL_RDESC_STAT_UDPSUMBAD))) {
2328                                         m->m_pkthdr.csum_flags |=
2329                                                 CSUM_DATA_VALID|CSUM_PSEUDO_HDR;
2330                                         m->m_pkthdr.csum_data = 0xffff;
2331                                 }
2332                         } else {
2333                                 /*
2334                                  * RTL8168C/RTL816CP/RTL8111C/RTL8111CP
2335                                  */
2336                                 if ((rxstat & RL_RDESC_STAT_PROTOID) &&
2337                                     (rxvlan & RL_RDESC_IPV4))
2338                                         m->m_pkthdr.csum_flags |=
2339                                             CSUM_IP_CHECKED;
2340                                 if (!(rxstat & RL_RDESC_STAT_IPSUMBAD) &&
2341                                     (rxvlan & RL_RDESC_IPV4))
2342                                         m->m_pkthdr.csum_flags |=
2343                                             CSUM_IP_VALID;
2344                                 if (((rxstat & RL_RDESC_STAT_TCP) &&
2345                                     !(rxstat & RL_RDESC_STAT_TCPSUMBAD)) ||
2346                                     ((rxstat & RL_RDESC_STAT_UDP) &&
2347                                     !(rxstat & RL_RDESC_STAT_UDPSUMBAD))) {
2348                                         m->m_pkthdr.csum_flags |=
2349                                                 CSUM_DATA_VALID|CSUM_PSEUDO_HDR;
2350                                         m->m_pkthdr.csum_data = 0xffff;
2351                                 }
2352                         }
2353                 }
2354                 maxpkt--;
2355                 if (rxvlan & RL_RDESC_VLANCTL_TAG) {
2356                         m->m_pkthdr.ether_vtag =
2357                             bswap16((rxvlan & RL_RDESC_VLANCTL_DATA));
2358                         m->m_flags |= M_VLANTAG;
2359                 }
2360                 RL_UNLOCK(sc);
2361                 (*ifp->if_input)(ifp, m);
2362                 RL_LOCK(sc);
2363                 rx_npkts++;
2364         }
2365
2366         /* Flush the RX DMA ring */
2367
2368         bus_dmamap_sync(sc->rl_ldata.rl_rx_list_tag,
2369             sc->rl_ldata.rl_rx_list_map,
2370             BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD);
2371
2372         sc->rl_ldata.rl_rx_prodidx = i;
2373
2374         if (rx_npktsp != NULL)
2375                 *rx_npktsp = rx_npkts;
2376         if (maxpkt)
2377                 return (EAGAIN);
2378
2379         return (0);
2380 }
2381
2382 static void
2383 re_txeof(struct rl_softc *sc)
2384 {
2385         struct ifnet            *ifp;
2386         struct rl_txdesc        *txd;
2387         u_int32_t               txstat;
2388         int                     cons;
2389
2390         cons = sc->rl_ldata.rl_tx_considx;
2391         if (cons == sc->rl_ldata.rl_tx_prodidx)
2392                 return;
2393
2394         ifp = sc->rl_ifp;
2395 #ifdef DEV_NETMAP
2396         if (netmap_tx_irq(ifp, 0))
2397                 return;
2398 #endif /* DEV_NETMAP */
2399         /* Invalidate the TX descriptor list */
2400         bus_dmamap_sync(sc->rl_ldata.rl_tx_list_tag,
2401             sc->rl_ldata.rl_tx_list_map,
2402             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2403
2404         for (; cons != sc->rl_ldata.rl_tx_prodidx;
2405             cons = RL_TX_DESC_NXT(sc, cons)) {
2406                 txstat = le32toh(sc->rl_ldata.rl_tx_list[cons].rl_cmdstat);
2407                 if (txstat & RL_TDESC_STAT_OWN)
2408                         break;
2409                 /*
2410                  * We only stash mbufs in the last descriptor
2411                  * in a fragment chain, which also happens to
2412                  * be the only place where the TX status bits
2413                  * are valid.
2414                  */
2415                 if (txstat & RL_TDESC_CMD_EOF) {
2416                         txd = &sc->rl_ldata.rl_tx_desc[cons];
2417                         bus_dmamap_sync(sc->rl_ldata.rl_tx_mtag,
2418                             txd->tx_dmamap, BUS_DMASYNC_POSTWRITE);
2419                         bus_dmamap_unload(sc->rl_ldata.rl_tx_mtag,
2420                             txd->tx_dmamap);
2421                         KASSERT(txd->tx_m != NULL,
2422                             ("%s: freeing NULL mbufs!", __func__));
2423                         m_freem(txd->tx_m);
2424                         txd->tx_m = NULL;
2425                         if (txstat & (RL_TDESC_STAT_EXCESSCOL|
2426                             RL_TDESC_STAT_COLCNT))
2427                                 ifp->if_collisions++;
2428                         if (txstat & RL_TDESC_STAT_TXERRSUM)
2429                                 ifp->if_oerrors++;
2430                         else
2431                                 ifp->if_opackets++;
2432                 }
2433                 sc->rl_ldata.rl_tx_free++;
2434                 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
2435         }
2436         sc->rl_ldata.rl_tx_considx = cons;
2437
2438         /* No changes made to the TX ring, so no flush needed */
2439
2440         if (sc->rl_ldata.rl_tx_free != sc->rl_ldata.rl_tx_desc_cnt) {
2441 #ifdef RE_TX_MODERATION
2442                 /*
2443                  * If not all descriptors have been reaped yet, reload
2444                  * the timer so that we will eventually get another
2445                  * interrupt that will cause us to re-enter this routine.
2446                  * This is done in case the transmitter has gone idle.
2447                  */
2448                 CSR_WRITE_4(sc, RL_TIMERCNT, 1);
2449 #endif
2450         } else
2451                 sc->rl_watchdog_timer = 0;
2452 }
2453
2454 static void
2455 re_tick(void *xsc)
2456 {
2457         struct rl_softc         *sc;
2458         struct mii_data         *mii;
2459
2460         sc = xsc;
2461
2462         RL_LOCK_ASSERT(sc);
2463
2464         mii = device_get_softc(sc->rl_miibus);
2465         mii_tick(mii);
2466         if ((sc->rl_flags & RL_FLAG_LINK) == 0)
2467                 re_miibus_statchg(sc->rl_dev);
2468         /*
2469          * Reclaim transmitted frames here. Technically it is not
2470          * necessary to do here but it ensures periodic reclamation
2471          * regardless of Tx completion interrupt which seems to be
2472          * lost on PCIe based controllers under certain situations.
2473          */
2474         re_txeof(sc);
2475         re_watchdog(sc);
2476         callout_reset(&sc->rl_stat_callout, hz, re_tick, sc);
2477 }
2478
2479 #ifdef DEVICE_POLLING
2480 static int
2481 re_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
2482 {
2483         struct rl_softc *sc = ifp->if_softc;
2484         int rx_npkts = 0;
2485
2486         RL_LOCK(sc);
2487         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
2488                 rx_npkts = re_poll_locked(ifp, cmd, count);
2489         RL_UNLOCK(sc);
2490         return (rx_npkts);
2491 }
2492
2493 static int
2494 re_poll_locked(struct ifnet *ifp, enum poll_cmd cmd, int count)
2495 {
2496         struct rl_softc *sc = ifp->if_softc;
2497         int rx_npkts;
2498
2499         RL_LOCK_ASSERT(sc);
2500
2501         sc->rxcycles = count;
2502         re_rxeof(sc, &rx_npkts);
2503         re_txeof(sc);
2504
2505         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
2506                 re_start_locked(ifp);
2507
2508         if (cmd == POLL_AND_CHECK_STATUS) { /* also check status register */
2509                 u_int16_t       status;
2510
2511                 status = CSR_READ_2(sc, RL_ISR);
2512                 if (status == 0xffff)
2513                         return (rx_npkts);
2514                 if (status)
2515                         CSR_WRITE_2(sc, RL_ISR, status);
2516                 if ((status & (RL_ISR_TX_OK | RL_ISR_TX_DESC_UNAVAIL)) &&
2517                     (sc->rl_flags & RL_FLAG_PCIE))
2518                         CSR_WRITE_1(sc, sc->rl_txstart, RL_TXSTART_START);
2519
2520                 /*
2521                  * XXX check behaviour on receiver stalls.
2522                  */
2523
2524                 if (status & RL_ISR_SYSTEM_ERR) {
2525                         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2526                         re_init_locked(sc);
2527                 }
2528         }
2529         return (rx_npkts);
2530 }
2531 #endif /* DEVICE_POLLING */
2532
2533 static int
2534 re_intr(void *arg)
2535 {
2536         struct rl_softc         *sc;
2537         uint16_t                status;
2538
2539         sc = arg;
2540
2541         status = CSR_READ_2(sc, RL_ISR);
2542         if (status == 0xFFFF || (status & RL_INTRS_CPLUS) == 0)
2543                 return (FILTER_STRAY);
2544         CSR_WRITE_2(sc, RL_IMR, 0);
2545
2546         taskqueue_enqueue_fast(taskqueue_fast, &sc->rl_inttask);
2547
2548         return (FILTER_HANDLED);
2549 }
2550
2551 static void
2552 re_int_task(void *arg, int npending)
2553 {
2554         struct rl_softc         *sc;
2555         struct ifnet            *ifp;
2556         u_int16_t               status;
2557         int                     rval = 0;
2558
2559         sc = arg;
2560         ifp = sc->rl_ifp;
2561
2562         RL_LOCK(sc);
2563
2564         status = CSR_READ_2(sc, RL_ISR);
2565         CSR_WRITE_2(sc, RL_ISR, status);
2566
2567         if (sc->suspended ||
2568             (ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
2569                 RL_UNLOCK(sc);
2570                 return;
2571         }
2572
2573 #ifdef DEVICE_POLLING
2574         if  (ifp->if_capenable & IFCAP_POLLING) {
2575                 RL_UNLOCK(sc);
2576                 return;
2577         }
2578 #endif
2579
2580         if (status & (RL_ISR_RX_OK|RL_ISR_RX_ERR|RL_ISR_FIFO_OFLOW))
2581                 rval = re_rxeof(sc, NULL);
2582
2583         /*
2584          * Some chips will ignore a second TX request issued
2585          * while an existing transmission is in progress. If
2586          * the transmitter goes idle but there are still
2587          * packets waiting to be sent, we need to restart the
2588          * channel here to flush them out. This only seems to
2589          * be required with the PCIe devices.
2590          */
2591         if ((status & (RL_ISR_TX_OK | RL_ISR_TX_DESC_UNAVAIL)) &&
2592             (sc->rl_flags & RL_FLAG_PCIE))
2593                 CSR_WRITE_1(sc, sc->rl_txstart, RL_TXSTART_START);
2594         if (status & (
2595 #ifdef RE_TX_MODERATION
2596             RL_ISR_TIMEOUT_EXPIRED|
2597 #else
2598             RL_ISR_TX_OK|
2599 #endif
2600             RL_ISR_TX_ERR|RL_ISR_TX_DESC_UNAVAIL))
2601                 re_txeof(sc);
2602
2603         if (status & RL_ISR_SYSTEM_ERR) {
2604                 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2605                 re_init_locked(sc);
2606         }
2607
2608         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
2609                 re_start_locked(ifp);
2610
2611         RL_UNLOCK(sc);
2612
2613         if ((CSR_READ_2(sc, RL_ISR) & RL_INTRS_CPLUS) || rval) {
2614                 taskqueue_enqueue_fast(taskqueue_fast, &sc->rl_inttask);
2615                 return;
2616         }
2617
2618         CSR_WRITE_2(sc, RL_IMR, RL_INTRS_CPLUS);
2619 }
2620
2621 static void
2622 re_intr_msi(void *xsc)
2623 {
2624         struct rl_softc         *sc;
2625         struct ifnet            *ifp;
2626         uint16_t                intrs, status;
2627
2628         sc = xsc;
2629         RL_LOCK(sc);
2630
2631         ifp = sc->rl_ifp;
2632 #ifdef DEVICE_POLLING
2633         if (ifp->if_capenable & IFCAP_POLLING) {
2634                 RL_UNLOCK(sc);
2635                 return;
2636         }
2637 #endif
2638         /* Disable interrupts. */
2639         CSR_WRITE_2(sc, RL_IMR, 0);
2640         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
2641                 RL_UNLOCK(sc);
2642                 return;
2643         }
2644
2645         intrs = RL_INTRS_CPLUS;
2646         status = CSR_READ_2(sc, RL_ISR);
2647         CSR_WRITE_2(sc, RL_ISR, status);
2648         if (sc->rl_int_rx_act > 0) {
2649                 intrs &= ~(RL_ISR_RX_OK | RL_ISR_RX_ERR | RL_ISR_FIFO_OFLOW |
2650                     RL_ISR_RX_OVERRUN);
2651                 status &= ~(RL_ISR_RX_OK | RL_ISR_RX_ERR | RL_ISR_FIFO_OFLOW |
2652                     RL_ISR_RX_OVERRUN);
2653         }
2654
2655         if (status & (RL_ISR_TIMEOUT_EXPIRED | RL_ISR_RX_OK | RL_ISR_RX_ERR |
2656             RL_ISR_FIFO_OFLOW | RL_ISR_RX_OVERRUN)) {
2657                 re_rxeof(sc, NULL);
2658                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) {
2659                         if (sc->rl_int_rx_mod != 0 &&
2660                             (status & (RL_ISR_RX_OK | RL_ISR_RX_ERR |
2661                             RL_ISR_FIFO_OFLOW | RL_ISR_RX_OVERRUN)) != 0) {
2662                                 /* Rearm one-shot timer. */
2663                                 CSR_WRITE_4(sc, RL_TIMERCNT, 1);
2664                                 intrs &= ~(RL_ISR_RX_OK | RL_ISR_RX_ERR |
2665                                     RL_ISR_FIFO_OFLOW | RL_ISR_RX_OVERRUN);
2666                                 sc->rl_int_rx_act = 1;
2667                         } else {
2668                                 intrs |= RL_ISR_RX_OK | RL_ISR_RX_ERR |
2669                                     RL_ISR_FIFO_OFLOW | RL_ISR_RX_OVERRUN;
2670                                 sc->rl_int_rx_act = 0;
2671                         }
2672                 }
2673         }
2674
2675         /*
2676          * Some chips will ignore a second TX request issued
2677          * while an existing transmission is in progress. If
2678          * the transmitter goes idle but there are still
2679          * packets waiting to be sent, we need to restart the
2680          * channel here to flush them out. This only seems to
2681          * be required with the PCIe devices.
2682          */
2683         if ((status & (RL_ISR_TX_OK | RL_ISR_TX_DESC_UNAVAIL)) &&
2684             (sc->rl_flags & RL_FLAG_PCIE))
2685                 CSR_WRITE_1(sc, sc->rl_txstart, RL_TXSTART_START);
2686         if (status & (RL_ISR_TX_OK | RL_ISR_TX_ERR | RL_ISR_TX_DESC_UNAVAIL))
2687                 re_txeof(sc);
2688
2689         if (status & RL_ISR_SYSTEM_ERR) {
2690                 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2691                 re_init_locked(sc);
2692         }
2693
2694         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) {
2695                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
2696                         re_start_locked(ifp);
2697                 CSR_WRITE_2(sc, RL_IMR, intrs);
2698         }
2699         RL_UNLOCK(sc);
2700 }
2701
2702 static int
2703 re_encap(struct rl_softc *sc, struct mbuf **m_head)
2704 {
2705         struct rl_txdesc        *txd, *txd_last;
2706         bus_dma_segment_t       segs[RL_NTXSEGS];
2707         bus_dmamap_t            map;
2708         struct mbuf             *m_new;
2709         struct rl_desc          *desc;
2710         int                     nsegs, prod;
2711         int                     i, error, ei, si;
2712         int                     padlen;
2713         uint32_t                cmdstat, csum_flags, vlanctl;
2714
2715         RL_LOCK_ASSERT(sc);
2716         M_ASSERTPKTHDR((*m_head));
2717
2718         /*
2719          * With some of the RealTek chips, using the checksum offload
2720          * support in conjunction with the autopadding feature results
2721          * in the transmission of corrupt frames. For example, if we
2722          * need to send a really small IP fragment that's less than 60
2723          * bytes in size, and IP header checksumming is enabled, the
2724          * resulting ethernet frame that appears on the wire will
2725          * have garbled payload. To work around this, if TX IP checksum
2726          * offload is enabled, we always manually pad short frames out
2727          * to the minimum ethernet frame size.
2728          */
2729         if ((sc->rl_flags & RL_FLAG_AUTOPAD) == 0 &&
2730             (*m_head)->m_pkthdr.len < RL_IP4CSUMTX_PADLEN &&
2731             ((*m_head)->m_pkthdr.csum_flags & CSUM_IP) != 0) {
2732                 padlen = RL_MIN_FRAMELEN - (*m_head)->m_pkthdr.len;
2733                 if (M_WRITABLE(*m_head) == 0) {
2734                         /* Get a writable copy. */
2735                         m_new = m_dup(*m_head, M_NOWAIT);
2736                         m_freem(*m_head);
2737                         if (m_new == NULL) {
2738                                 *m_head = NULL;
2739                                 return (ENOBUFS);
2740                         }
2741                         *m_head = m_new;
2742                 }
2743                 if ((*m_head)->m_next != NULL ||
2744                     M_TRAILINGSPACE(*m_head) < padlen) {
2745                         m_new = m_defrag(*m_head, M_NOWAIT);
2746                         if (m_new == NULL) {
2747                                 m_freem(*m_head);
2748                                 *m_head = NULL;
2749                                 return (ENOBUFS);
2750                         }
2751                 } else
2752                         m_new = *m_head;
2753
2754                 /*
2755                  * Manually pad short frames, and zero the pad space
2756                  * to avoid leaking data.
2757                  */
2758                 bzero(mtod(m_new, char *) + m_new->m_pkthdr.len, padlen);
2759                 m_new->m_pkthdr.len += padlen;
2760                 m_new->m_len = m_new->m_pkthdr.len;
2761                 *m_head = m_new;
2762         }
2763
2764         prod = sc->rl_ldata.rl_tx_prodidx;
2765         txd = &sc->rl_ldata.rl_tx_desc[prod];
2766         error = bus_dmamap_load_mbuf_sg(sc->rl_ldata.rl_tx_mtag, txd->tx_dmamap,
2767             *m_head, segs, &nsegs, BUS_DMA_NOWAIT);
2768         if (error == EFBIG) {
2769                 m_new = m_collapse(*m_head, M_NOWAIT, RL_NTXSEGS);
2770                 if (m_new == NULL) {
2771                         m_freem(*m_head);
2772                         *m_head = NULL;
2773                         return (ENOBUFS);
2774                 }
2775                 *m_head = m_new;
2776                 error = bus_dmamap_load_mbuf_sg(sc->rl_ldata.rl_tx_mtag,
2777                     txd->tx_dmamap, *m_head, segs, &nsegs, BUS_DMA_NOWAIT);
2778                 if (error != 0) {
2779                         m_freem(*m_head);
2780                         *m_head = NULL;
2781                         return (error);
2782                 }
2783         } else if (error != 0)
2784                 return (error);
2785         if (nsegs == 0) {
2786                 m_freem(*m_head);
2787                 *m_head = NULL;
2788                 return (EIO);
2789         }
2790
2791         /* Check for number of available descriptors. */
2792         if (sc->rl_ldata.rl_tx_free - nsegs <= 1) {
2793                 bus_dmamap_unload(sc->rl_ldata.rl_tx_mtag, txd->tx_dmamap);
2794                 return (ENOBUFS);
2795         }
2796
2797         bus_dmamap_sync(sc->rl_ldata.rl_tx_mtag, txd->tx_dmamap,
2798             BUS_DMASYNC_PREWRITE);
2799
2800         /*
2801          * Set up checksum offload. Note: checksum offload bits must
2802          * appear in all descriptors of a multi-descriptor transmit
2803          * attempt. This is according to testing done with an 8169
2804          * chip. This is a requirement.
2805          */
2806         vlanctl = 0;
2807         csum_flags = 0;
2808         if (((*m_head)->m_pkthdr.csum_flags & CSUM_TSO) != 0) {
2809                 if ((sc->rl_flags & RL_FLAG_DESCV2) != 0) {
2810                         csum_flags |= RL_TDESC_CMD_LGSEND;
2811                         vlanctl |= ((uint32_t)(*m_head)->m_pkthdr.tso_segsz <<
2812                             RL_TDESC_CMD_MSSVALV2_SHIFT);
2813                 } else {
2814                         csum_flags |= RL_TDESC_CMD_LGSEND |
2815                             ((uint32_t)(*m_head)->m_pkthdr.tso_segsz <<
2816                             RL_TDESC_CMD_MSSVAL_SHIFT);
2817                 }
2818         } else {
2819                 /*
2820                  * Unconditionally enable IP checksum if TCP or UDP
2821                  * checksum is required. Otherwise, TCP/UDP checksum
2822                  * doesn't make effects.
2823                  */
2824                 if (((*m_head)->m_pkthdr.csum_flags & RE_CSUM_FEATURES) != 0) {
2825                         if ((sc->rl_flags & RL_FLAG_DESCV2) == 0) {
2826                                 csum_flags |= RL_TDESC_CMD_IPCSUM;
2827                                 if (((*m_head)->m_pkthdr.csum_flags &
2828                                     CSUM_TCP) != 0)
2829                                         csum_flags |= RL_TDESC_CMD_TCPCSUM;
2830                                 if (((*m_head)->m_pkthdr.csum_flags &
2831                                     CSUM_UDP) != 0)
2832                                         csum_flags |= RL_TDESC_CMD_UDPCSUM;
2833                         } else {
2834                                 vlanctl |= RL_TDESC_CMD_IPCSUMV2;
2835                                 if (((*m_head)->m_pkthdr.csum_flags &
2836                                     CSUM_TCP) != 0)
2837                                         vlanctl |= RL_TDESC_CMD_TCPCSUMV2;
2838                                 if (((*m_head)->m_pkthdr.csum_flags &
2839                                     CSUM_UDP) != 0)
2840                                         vlanctl |= RL_TDESC_CMD_UDPCSUMV2;
2841                         }
2842                 }
2843         }
2844
2845         /*
2846          * Set up hardware VLAN tagging. Note: vlan tag info must
2847          * appear in all descriptors of a multi-descriptor
2848          * transmission attempt.
2849          */
2850         if ((*m_head)->m_flags & M_VLANTAG)
2851                 vlanctl |= bswap16((*m_head)->m_pkthdr.ether_vtag) |
2852                     RL_TDESC_VLANCTL_TAG;
2853
2854         si = prod;
2855         for (i = 0; i < nsegs; i++, prod = RL_TX_DESC_NXT(sc, prod)) {
2856                 desc = &sc->rl_ldata.rl_tx_list[prod];
2857                 desc->rl_vlanctl = htole32(vlanctl);
2858                 desc->rl_bufaddr_lo = htole32(RL_ADDR_LO(segs[i].ds_addr));
2859                 desc->rl_bufaddr_hi = htole32(RL_ADDR_HI(segs[i].ds_addr));
2860                 cmdstat = segs[i].ds_len;
2861                 if (i != 0)
2862                         cmdstat |= RL_TDESC_CMD_OWN;
2863                 if (prod == sc->rl_ldata.rl_tx_desc_cnt - 1)
2864                         cmdstat |= RL_TDESC_CMD_EOR;
2865                 desc->rl_cmdstat = htole32(cmdstat | csum_flags);
2866                 sc->rl_ldata.rl_tx_free--;
2867         }
2868         /* Update producer index. */
2869         sc->rl_ldata.rl_tx_prodidx = prod;
2870
2871         /* Set EOF on the last descriptor. */
2872         ei = RL_TX_DESC_PRV(sc, prod);
2873         desc = &sc->rl_ldata.rl_tx_list[ei];
2874         desc->rl_cmdstat |= htole32(RL_TDESC_CMD_EOF);
2875
2876         desc = &sc->rl_ldata.rl_tx_list[si];
2877         /* Set SOF and transfer ownership of packet to the chip. */
2878         desc->rl_cmdstat |= htole32(RL_TDESC_CMD_OWN | RL_TDESC_CMD_SOF);
2879
2880         /*
2881          * Insure that the map for this transmission
2882          * is placed at the array index of the last descriptor
2883          * in this chain.  (Swap last and first dmamaps.)
2884          */
2885         txd_last = &sc->rl_ldata.rl_tx_desc[ei];
2886         map = txd->tx_dmamap;
2887         txd->tx_dmamap = txd_last->tx_dmamap;
2888         txd_last->tx_dmamap = map;
2889         txd_last->tx_m = *m_head;
2890
2891         return (0);
2892 }
2893
2894 static void
2895 re_start(struct ifnet *ifp)
2896 {
2897         struct rl_softc         *sc;
2898
2899         sc = ifp->if_softc;
2900         RL_LOCK(sc);
2901         re_start_locked(ifp);
2902         RL_UNLOCK(sc);
2903 }
2904
2905 /*
2906  * Main transmit routine for C+ and gigE NICs.
2907  */
2908 static void
2909 re_start_locked(struct ifnet *ifp)
2910 {
2911         struct rl_softc         *sc;
2912         struct mbuf             *m_head;
2913         int                     queued;
2914
2915         sc = ifp->if_softc;
2916
2917 #ifdef DEV_NETMAP
2918         /* XXX is this necessary ? */
2919         if (ifp->if_capenable & IFCAP_NETMAP) {
2920                 struct netmap_kring *kring = &NA(ifp)->tx_rings[0];
2921                 if (sc->rl_ldata.rl_tx_prodidx != kring->nr_hwcur) {
2922                         /* kick the tx unit */
2923                         CSR_WRITE_1(sc, sc->rl_txstart, RL_TXSTART_START);
2924 #ifdef RE_TX_MODERATION
2925                         CSR_WRITE_4(sc, RL_TIMERCNT, 1);
2926 #endif
2927                         sc->rl_watchdog_timer = 5;
2928                 }
2929                 return;
2930         }
2931 #endif /* DEV_NETMAP */
2932         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
2933             IFF_DRV_RUNNING || (sc->rl_flags & RL_FLAG_LINK) == 0)
2934                 return;
2935
2936         for (queued = 0; !IFQ_DRV_IS_EMPTY(&ifp->if_snd) &&
2937             sc->rl_ldata.rl_tx_free > 1;) {
2938                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
2939                 if (m_head == NULL)
2940                         break;
2941
2942                 if (re_encap(sc, &m_head) != 0) {
2943                         if (m_head == NULL)
2944                                 break;
2945                         IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
2946                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
2947                         break;
2948                 }
2949
2950                 /*
2951                  * If there's a BPF listener, bounce a copy of this frame
2952                  * to him.
2953                  */
2954                 ETHER_BPF_MTAP(ifp, m_head);
2955
2956                 queued++;
2957         }
2958
2959         if (queued == 0) {
2960 #ifdef RE_TX_MODERATION
2961                 if (sc->rl_ldata.rl_tx_free != sc->rl_ldata.rl_tx_desc_cnt)
2962                         CSR_WRITE_4(sc, RL_TIMERCNT, 1);
2963 #endif
2964                 return;
2965         }
2966
2967         /* Flush the TX descriptors */
2968
2969         bus_dmamap_sync(sc->rl_ldata.rl_tx_list_tag,
2970             sc->rl_ldata.rl_tx_list_map,
2971             BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD);
2972
2973         CSR_WRITE_1(sc, sc->rl_txstart, RL_TXSTART_START);
2974
2975 #ifdef RE_TX_MODERATION
2976         /*
2977          * Use the countdown timer for interrupt moderation.
2978          * 'TX done' interrupts are disabled. Instead, we reset the
2979          * countdown timer, which will begin counting until it hits
2980          * the value in the TIMERINT register, and then trigger an
2981          * interrupt. Each time we write to the TIMERCNT register,
2982          * the timer count is reset to 0.
2983          */
2984         CSR_WRITE_4(sc, RL_TIMERCNT, 1);
2985 #endif
2986
2987         /*
2988          * Set a timeout in case the chip goes out to lunch.
2989          */
2990         sc->rl_watchdog_timer = 5;
2991 }
2992
2993 static void
2994 re_set_jumbo(struct rl_softc *sc, int jumbo)
2995 {
2996
2997         if (sc->rl_hwrev->rl_rev == RL_HWREV_8168E_VL) {
2998                 pci_set_max_read_req(sc->rl_dev, 4096);
2999                 return;
3000         }
3001
3002         CSR_WRITE_1(sc, RL_EECMD, RL_EEMODE_WRITECFG);
3003         if (jumbo != 0) {
3004                 CSR_WRITE_1(sc, sc->rl_cfg3, CSR_READ_1(sc, sc->rl_cfg3) |
3005                     RL_CFG3_JUMBO_EN0);
3006                 switch (sc->rl_hwrev->rl_rev) {
3007                 case RL_HWREV_8168DP:
3008                         break;
3009                 case RL_HWREV_8168E:
3010                         CSR_WRITE_1(sc, sc->rl_cfg4,
3011                             CSR_READ_1(sc, sc->rl_cfg4) | 0x01);
3012                         break;
3013                 default:
3014                         CSR_WRITE_1(sc, sc->rl_cfg4,
3015                             CSR_READ_1(sc, sc->rl_cfg4) | RL_CFG4_JUMBO_EN1);
3016                 }
3017         } else {
3018                 CSR_WRITE_1(sc, sc->rl_cfg3, CSR_READ_1(sc, sc->rl_cfg3) &
3019                     ~RL_CFG3_JUMBO_EN0);
3020                 switch (sc->rl_hwrev->rl_rev) {
3021                 case RL_HWREV_8168DP:
3022                         break;
3023                 case RL_HWREV_8168E:
3024                         CSR_WRITE_1(sc, sc->rl_cfg4,
3025                             CSR_READ_1(sc, sc->rl_cfg4) & ~0x01);
3026                         break;
3027                 default:
3028                         CSR_WRITE_1(sc, sc->rl_cfg4,
3029                             CSR_READ_1(sc, sc->rl_cfg4) & ~RL_CFG4_JUMBO_EN1);
3030                 }
3031         }
3032         CSR_WRITE_1(sc, RL_EECMD, RL_EEMODE_OFF);
3033
3034         switch (sc->rl_hwrev->rl_rev) {
3035         case RL_HWREV_8168DP:
3036                 pci_set_max_read_req(sc->rl_dev, 4096);
3037                 break;
3038         default:
3039                 if (jumbo != 0)
3040                         pci_set_max_read_req(sc->rl_dev, 512);
3041                 else
3042                         pci_set_max_read_req(sc->rl_dev, 4096);
3043         }
3044 }
3045
3046 static void
3047 re_init(void *xsc)
3048 {
3049         struct rl_softc         *sc = xsc;
3050
3051         RL_LOCK(sc);
3052         re_init_locked(sc);
3053         RL_UNLOCK(sc);
3054 }
3055
3056 static void
3057 re_init_locked(struct rl_softc *sc)
3058 {
3059         struct ifnet            *ifp = sc->rl_ifp;
3060         struct mii_data         *mii;
3061         uint32_t                reg;
3062         uint16_t                cfg;
3063         union {
3064                 uint32_t align_dummy;
3065                 u_char eaddr[ETHER_ADDR_LEN];
3066         } eaddr;
3067
3068         RL_LOCK_ASSERT(sc);
3069
3070         mii = device_get_softc(sc->rl_miibus);
3071
3072         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0)
3073                 return;
3074
3075         /*
3076          * Cancel pending I/O and free all RX/TX buffers.
3077          */
3078         re_stop(sc);
3079
3080         /* Put controller into known state. */
3081         re_reset(sc);
3082
3083         /*
3084          * For C+ mode, initialize the RX descriptors and mbufs.
3085          */
3086         if ((sc->rl_flags & RL_FLAG_JUMBOV2) != 0) {
3087                 if (ifp->if_mtu > RL_MTU) {
3088                         if (re_jrx_list_init(sc) != 0) {
3089                                 device_printf(sc->rl_dev,
3090                                     "no memory for jumbo RX buffers\n");
3091                                 re_stop(sc);
3092                                 return;
3093                         }
3094                         /* Disable checksum offloading for jumbo frames. */
3095                         ifp->if_capenable &= ~(IFCAP_HWCSUM | IFCAP_TSO4);
3096                         ifp->if_hwassist &= ~(RE_CSUM_FEATURES | CSUM_TSO);
3097                 } else {
3098                         if (re_rx_list_init(sc) != 0) {
3099                                 device_printf(sc->rl_dev,
3100                                     "no memory for RX buffers\n");
3101                                 re_stop(sc);
3102                                 return;
3103                         }
3104                 }
3105                 re_set_jumbo(sc, ifp->if_mtu > RL_MTU);
3106         } else {
3107                 if (re_rx_list_init(sc) != 0) {
3108                         device_printf(sc->rl_dev, "no memory for RX buffers\n");
3109                         re_stop(sc);
3110                         return;
3111                 }
3112                 if ((sc->rl_flags & RL_FLAG_PCIE) != 0 &&
3113                     pci_get_device(sc->rl_dev) != RT_DEVICEID_8101E) {
3114                         if (ifp->if_mtu > RL_MTU)
3115                                 pci_set_max_read_req(sc->rl_dev, 512);
3116                         else
3117                                 pci_set_max_read_req(sc->rl_dev, 4096);
3118                 }
3119         }
3120         re_tx_list_init(sc);
3121
3122         /*
3123          * Enable C+ RX and TX mode, as well as VLAN stripping and
3124          * RX checksum offload. We must configure the C+ register
3125          * before all others.
3126          */
3127         cfg = RL_CPLUSCMD_PCI_MRW;
3128         if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
3129                 cfg |= RL_CPLUSCMD_RXCSUM_ENB;
3130         if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0)
3131                 cfg |= RL_CPLUSCMD_VLANSTRIP;
3132         if ((sc->rl_flags & RL_FLAG_MACSTAT) != 0) {
3133                 cfg |= RL_CPLUSCMD_MACSTAT_DIS;
3134                 /* XXX magic. */
3135                 cfg |= 0x0001;
3136         } else
3137                 cfg |= RL_CPLUSCMD_RXENB | RL_CPLUSCMD_TXENB;
3138         CSR_WRITE_2(sc, RL_CPLUS_CMD, cfg);
3139         if (sc->rl_hwrev->rl_rev == RL_HWREV_8169_8110SC ||
3140             sc->rl_hwrev->rl_rev == RL_HWREV_8169_8110SCE) {
3141                 reg = 0x000fff00;
3142                 if ((CSR_READ_1(sc, sc->rl_cfg2) & RL_CFG2_PCI66MHZ) != 0)
3143                         reg |= 0x000000ff;
3144                 if (sc->rl_hwrev->rl_rev == RL_HWREV_8169_8110SCE)
3145                         reg |= 0x00f00000;
3146                 CSR_WRITE_4(sc, 0x7c, reg);
3147                 /* Disable interrupt mitigation. */
3148                 CSR_WRITE_2(sc, 0xe2, 0);
3149         }
3150         /*
3151          * Disable TSO if interface MTU size is greater than MSS
3152          * allowed in controller.
3153          */
3154         if (ifp->if_mtu > RL_TSO_MTU && (ifp->if_capenable & IFCAP_TSO4) != 0) {
3155                 ifp->if_capenable &= ~IFCAP_TSO4;
3156                 ifp->if_hwassist &= ~CSUM_TSO;
3157         }
3158
3159         /*
3160          * Init our MAC address.  Even though the chipset
3161          * documentation doesn't mention it, we need to enter "Config
3162          * register write enable" mode to modify the ID registers.
3163          */
3164         /* Copy MAC address on stack to align. */
3165         bcopy(IF_LLADDR(ifp), eaddr.eaddr, ETHER_ADDR_LEN);
3166         CSR_WRITE_1(sc, RL_EECMD, RL_EEMODE_WRITECFG);
3167         CSR_WRITE_4(sc, RL_IDR0,
3168             htole32(*(u_int32_t *)(&eaddr.eaddr[0])));
3169         CSR_WRITE_4(sc, RL_IDR4,
3170             htole32(*(u_int32_t *)(&eaddr.eaddr[4])));
3171         CSR_WRITE_1(sc, RL_EECMD, RL_EEMODE_OFF);
3172
3173         /*
3174          * Load the addresses of the RX and TX lists into the chip.
3175          */
3176
3177         CSR_WRITE_4(sc, RL_RXLIST_ADDR_HI,
3178             RL_ADDR_HI(sc->rl_ldata.rl_rx_list_addr));
3179         CSR_WRITE_4(sc, RL_RXLIST_ADDR_LO,
3180             RL_ADDR_LO(sc->rl_ldata.rl_rx_list_addr));
3181
3182         CSR_WRITE_4(sc, RL_TXLIST_ADDR_HI,
3183             RL_ADDR_HI(sc->rl_ldata.rl_tx_list_addr));
3184         CSR_WRITE_4(sc, RL_TXLIST_ADDR_LO,
3185             RL_ADDR_LO(sc->rl_ldata.rl_tx_list_addr));
3186
3187         if ((sc->rl_flags & RL_FLAG_RXDV_GATED) != 0)
3188                 CSR_WRITE_4(sc, RL_MISC, CSR_READ_4(sc, RL_MISC) &
3189                     ~0x00080000);
3190
3191         /*
3192          * Enable transmit and receive.
3193          */
3194         CSR_WRITE_1(sc, RL_COMMAND, RL_CMD_TX_ENB|RL_CMD_RX_ENB);
3195
3196         /*
3197          * Set the initial TX configuration.
3198          */
3199         if (sc->rl_testmode) {
3200                 if (sc->rl_type == RL_8169)
3201                         CSR_WRITE_4(sc, RL_TXCFG,
3202                             RL_TXCFG_CONFIG|RL_LOOPTEST_ON);
3203                 else
3204                         CSR_WRITE_4(sc, RL_TXCFG,
3205                             RL_TXCFG_CONFIG|RL_LOOPTEST_ON_CPLUS);
3206         } else
3207                 CSR_WRITE_4(sc, RL_TXCFG, RL_TXCFG_CONFIG);
3208
3209         CSR_WRITE_1(sc, RL_EARLY_TX_THRESH, 16);
3210
3211         /*
3212          * Set the initial RX configuration.
3213          */
3214         re_set_rxmode(sc);
3215
3216         /* Configure interrupt moderation. */
3217         if (sc->rl_type == RL_8169) {
3218                 /* Magic from vendor. */
3219                 CSR_WRITE_2(sc, RL_INTRMOD, 0x5100);
3220         }
3221
3222 #ifdef DEVICE_POLLING
3223         /*
3224          * Disable interrupts if we are polling.
3225          */
3226         if (ifp->if_capenable & IFCAP_POLLING)
3227                 CSR_WRITE_2(sc, RL_IMR, 0);
3228         else    /* otherwise ... */
3229 #endif
3230
3231         /*
3232          * Enable interrupts.
3233          */
3234         if (sc->rl_testmode)
3235                 CSR_WRITE_2(sc, RL_IMR, 0);
3236         else
3237                 CSR_WRITE_2(sc, RL_IMR, RL_INTRS_CPLUS);
3238         CSR_WRITE_2(sc, RL_ISR, RL_INTRS_CPLUS);
3239
3240         /* Set initial TX threshold */
3241         sc->rl_txthresh = RL_TX_THRESH_INIT;
3242
3243         /* Start RX/TX process. */
3244         CSR_WRITE_4(sc, RL_MISSEDPKT, 0);
3245 #ifdef notdef
3246         /* Enable receiver and transmitter. */
3247         CSR_WRITE_1(sc, RL_COMMAND, RL_CMD_TX_ENB|RL_CMD_RX_ENB);
3248 #endif
3249
3250         /*
3251          * Initialize the timer interrupt register so that
3252          * a timer interrupt will be generated once the timer
3253          * reaches a certain number of ticks. The timer is
3254          * reloaded on each transmit.
3255          */
3256 #ifdef RE_TX_MODERATION
3257         /*
3258          * Use timer interrupt register to moderate TX interrupt
3259          * moderation, which dramatically improves TX frame rate.
3260          */
3261         if (sc->rl_type == RL_8169)
3262                 CSR_WRITE_4(sc, RL_TIMERINT_8169, 0x800);
3263         else
3264                 CSR_WRITE_4(sc, RL_TIMERINT, 0x400);
3265 #else
3266         /*
3267          * Use timer interrupt register to moderate RX interrupt
3268          * moderation.
3269          */
3270         if ((sc->rl_flags & (RL_FLAG_MSI | RL_FLAG_MSIX)) != 0 &&
3271             intr_filter == 0) {
3272                 if (sc->rl_type == RL_8169)
3273                         CSR_WRITE_4(sc, RL_TIMERINT_8169,
3274                             RL_USECS(sc->rl_int_rx_mod));
3275         } else {
3276                 if (sc->rl_type == RL_8169)
3277                         CSR_WRITE_4(sc, RL_TIMERINT_8169, RL_USECS(0));
3278         }
3279 #endif
3280
3281         /*
3282          * For 8169 gigE NICs, set the max allowed RX packet
3283          * size so we can receive jumbo frames.
3284          */
3285         if (sc->rl_type == RL_8169) {
3286                 if ((sc->rl_flags & RL_FLAG_JUMBOV2) != 0) {
3287                         /*
3288                          * For controllers that use new jumbo frame scheme,
3289                          * set maximum size of jumbo frame depending on
3290                          * controller revisions.
3291                          */
3292                         if (ifp->if_mtu > RL_MTU)
3293                                 CSR_WRITE_2(sc, RL_MAXRXPKTLEN,
3294                                     sc->rl_hwrev->rl_max_mtu +
3295                                     ETHER_VLAN_ENCAP_LEN + ETHER_HDR_LEN +
3296                                     ETHER_CRC_LEN);
3297                         else
3298                                 CSR_WRITE_2(sc, RL_MAXRXPKTLEN,
3299                                     RE_RX_DESC_BUFLEN);
3300                 } else if ((sc->rl_flags & RL_FLAG_PCIE) != 0 &&
3301                     sc->rl_hwrev->rl_max_mtu == RL_MTU) {
3302                         /* RTL810x has no jumbo frame support. */
3303                         CSR_WRITE_2(sc, RL_MAXRXPKTLEN, RE_RX_DESC_BUFLEN);
3304                 } else
3305                         CSR_WRITE_2(sc, RL_MAXRXPKTLEN, 16383);
3306         }
3307
3308         if (sc->rl_testmode)
3309                 return;
3310
3311         CSR_WRITE_1(sc, sc->rl_cfg1, CSR_READ_1(sc, sc->rl_cfg1) |
3312             RL_CFG1_DRVLOAD);
3313
3314         ifp->if_drv_flags |= IFF_DRV_RUNNING;
3315         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
3316
3317         sc->rl_flags &= ~RL_FLAG_LINK;
3318         mii_mediachg(mii);
3319
3320         sc->rl_watchdog_timer = 0;
3321         callout_reset(&sc->rl_stat_callout, hz, re_tick, sc);
3322 }
3323
3324 /*
3325  * Set media options.
3326  */
3327 static int
3328 re_ifmedia_upd(struct ifnet *ifp)
3329 {
3330         struct rl_softc         *sc;
3331         struct mii_data         *mii;
3332         int                     error;
3333
3334         sc = ifp->if_softc;
3335         mii = device_get_softc(sc->rl_miibus);
3336         RL_LOCK(sc);
3337         error = mii_mediachg(mii);
3338         RL_UNLOCK(sc);
3339
3340         return (error);
3341 }
3342
3343 /*
3344  * Report current media status.
3345  */
3346 static void
3347 re_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
3348 {
3349         struct rl_softc         *sc;
3350         struct mii_data         *mii;
3351
3352         sc = ifp->if_softc;
3353         mii = device_get_softc(sc->rl_miibus);
3354
3355         RL_LOCK(sc);
3356         mii_pollstat(mii);
3357         ifmr->ifm_active = mii->mii_media_active;
3358         ifmr->ifm_status = mii->mii_media_status;
3359         RL_UNLOCK(sc);
3360 }
3361
3362 static int
3363 re_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
3364 {
3365         struct rl_softc         *sc = ifp->if_softc;
3366         struct ifreq            *ifr = (struct ifreq *) data;
3367         struct mii_data         *mii;
3368         int                     error = 0;
3369
3370         switch (command) {
3371         case SIOCSIFMTU:
3372                 if (ifr->ifr_mtu < ETHERMIN ||
3373                     ifr->ifr_mtu > sc->rl_hwrev->rl_max_mtu ||
3374                     ((sc->rl_flags & RL_FLAG_FASTETHER) != 0 &&
3375                     ifr->ifr_mtu > RL_MTU)) {
3376                         error = EINVAL;
3377                         break;
3378                 }
3379                 RL_LOCK(sc);
3380                 if (ifp->if_mtu != ifr->ifr_mtu) {
3381                         ifp->if_mtu = ifr->ifr_mtu;
3382                         if ((sc->rl_flags & RL_FLAG_JUMBOV2) != 0 &&
3383                             (ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) {
3384                                 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3385                                 re_init_locked(sc);
3386                         }
3387                         if (ifp->if_mtu > RL_TSO_MTU &&
3388                             (ifp->if_capenable & IFCAP_TSO4) != 0) {
3389                                 ifp->if_capenable &= ~(IFCAP_TSO4 |
3390                                     IFCAP_VLAN_HWTSO);
3391                                 ifp->if_hwassist &= ~CSUM_TSO;
3392                         }
3393                         VLAN_CAPABILITIES(ifp);
3394                 }
3395                 RL_UNLOCK(sc);
3396                 break;
3397         case SIOCSIFFLAGS:
3398                 RL_LOCK(sc);
3399                 if ((ifp->if_flags & IFF_UP) != 0) {
3400                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) {
3401                                 if (((ifp->if_flags ^ sc->rl_if_flags)
3402                                     & (IFF_PROMISC | IFF_ALLMULTI)) != 0)
3403                                         re_set_rxmode(sc);
3404                         } else
3405                                 re_init_locked(sc);
3406                 } else {
3407                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0)
3408                                 re_stop(sc);
3409                 }
3410                 sc->rl_if_flags = ifp->if_flags;
3411                 RL_UNLOCK(sc);
3412                 break;
3413         case SIOCADDMULTI:
3414         case SIOCDELMULTI:
3415                 RL_LOCK(sc);
3416                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0)
3417                         re_set_rxmode(sc);
3418                 RL_UNLOCK(sc);
3419                 break;
3420         case SIOCGIFMEDIA:
3421         case SIOCSIFMEDIA:
3422                 mii = device_get_softc(sc->rl_miibus);
3423                 error = ifmedia_ioctl(ifp, ifr, &mii->mii_media, command);
3424                 break;
3425         case SIOCSIFCAP:
3426             {
3427                 int mask, reinit;
3428
3429                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
3430                 reinit = 0;
3431 #ifdef DEVICE_POLLING
3432                 if (mask & IFCAP_POLLING) {
3433                         if (ifr->ifr_reqcap & IFCAP_POLLING) {
3434                                 error = ether_poll_register(re_poll, ifp);
3435                                 if (error)
3436                                         return (error);
3437                                 RL_LOCK(sc);
3438                                 /* Disable interrupts */
3439                                 CSR_WRITE_2(sc, RL_IMR, 0x0000);
3440                                 ifp->if_capenable |= IFCAP_POLLING;
3441                                 RL_UNLOCK(sc);
3442                         } else {
3443                                 error = ether_poll_deregister(ifp);
3444                                 /* Enable interrupts. */
3445                                 RL_LOCK(sc);
3446                                 CSR_WRITE_2(sc, RL_IMR, RL_INTRS_CPLUS);
3447                                 ifp->if_capenable &= ~IFCAP_POLLING;
3448                                 RL_UNLOCK(sc);
3449                         }
3450                 }
3451 #endif /* DEVICE_POLLING */
3452                 RL_LOCK(sc);
3453                 if ((mask & IFCAP_TXCSUM) != 0 &&
3454                     (ifp->if_capabilities & IFCAP_TXCSUM) != 0) {
3455                         ifp->if_capenable ^= IFCAP_TXCSUM;
3456                         if ((ifp->if_capenable & IFCAP_TXCSUM) != 0)
3457                                 ifp->if_hwassist |= RE_CSUM_FEATURES;
3458                         else
3459                                 ifp->if_hwassist &= ~RE_CSUM_FEATURES;
3460                         reinit = 1;
3461                 }
3462                 if ((mask & IFCAP_RXCSUM) != 0 &&
3463                     (ifp->if_capabilities & IFCAP_RXCSUM) != 0) {
3464                         ifp->if_capenable ^= IFCAP_RXCSUM;
3465                         reinit = 1;
3466                 }
3467                 if ((mask & IFCAP_TSO4) != 0 &&
3468                     (ifp->if_capabilities & IFCAP_TSO4) != 0) {
3469                         ifp->if_capenable ^= IFCAP_TSO4;
3470                         if ((IFCAP_TSO4 & ifp->if_capenable) != 0)
3471                                 ifp->if_hwassist |= CSUM_TSO;
3472                         else
3473                                 ifp->if_hwassist &= ~CSUM_TSO;
3474                         if (ifp->if_mtu > RL_TSO_MTU &&
3475                             (ifp->if_capenable & IFCAP_TSO4) != 0) {
3476                                 ifp->if_capenable &= ~IFCAP_TSO4;
3477                                 ifp->if_hwassist &= ~CSUM_TSO;
3478                         }
3479                 }
3480                 if ((mask & IFCAP_VLAN_HWTSO) != 0 &&
3481                     (ifp->if_capabilities & IFCAP_VLAN_HWTSO) != 0)
3482                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
3483                 if ((mask & IFCAP_VLAN_HWTAGGING) != 0 &&
3484                     (ifp->if_capabilities & IFCAP_VLAN_HWTAGGING) != 0) {
3485                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
3486                         /* TSO over VLAN requires VLAN hardware tagging. */
3487                         if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) == 0)
3488                                 ifp->if_capenable &= ~IFCAP_VLAN_HWTSO;
3489                         reinit = 1;
3490                 }
3491                 if ((sc->rl_flags & RL_FLAG_JUMBOV2) != 0 &&
3492                     (mask & (IFCAP_HWCSUM | IFCAP_TSO4 |
3493                     IFCAP_VLAN_HWTSO)) != 0)
3494                                 reinit = 1;
3495                 if ((mask & IFCAP_WOL) != 0 &&
3496                     (ifp->if_capabilities & IFCAP_WOL) != 0) {
3497                         if ((mask & IFCAP_WOL_UCAST) != 0)
3498                                 ifp->if_capenable ^= IFCAP_WOL_UCAST;
3499                         if ((mask & IFCAP_WOL_MCAST) != 0)
3500                                 ifp->if_capenable ^= IFCAP_WOL_MCAST;
3501                         if ((mask & IFCAP_WOL_MAGIC) != 0)
3502                                 ifp->if_capenable ^= IFCAP_WOL_MAGIC;
3503                 }
3504                 if (reinit && ifp->if_drv_flags & IFF_DRV_RUNNING) {
3505                         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3506                         re_init_locked(sc);
3507                 }
3508                 RL_UNLOCK(sc);
3509                 VLAN_CAPABILITIES(ifp);
3510             }
3511                 break;
3512         default:
3513                 error = ether_ioctl(ifp, command, data);
3514                 break;
3515         }
3516
3517         return (error);
3518 }
3519
3520 static void
3521 re_watchdog(struct rl_softc *sc)
3522 {
3523         struct ifnet            *ifp;
3524
3525         RL_LOCK_ASSERT(sc);
3526
3527         if (sc->rl_watchdog_timer == 0 || --sc->rl_watchdog_timer != 0)
3528                 return;
3529
3530         ifp = sc->rl_ifp;
3531         re_txeof(sc);
3532         if (sc->rl_ldata.rl_tx_free == sc->rl_ldata.rl_tx_desc_cnt) {
3533                 if_printf(ifp, "watchdog timeout (missed Tx interrupts) "
3534                     "-- recovering\n");
3535                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
3536                         re_start_locked(ifp);
3537                 return;
3538         }
3539
3540         if_printf(ifp, "watchdog timeout\n");
3541         ifp->if_oerrors++;
3542
3543         re_rxeof(sc, NULL);
3544         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3545         re_init_locked(sc);
3546         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
3547                 re_start_locked(ifp);
3548 }
3549
3550 /*
3551  * Stop the adapter and free any mbufs allocated to the
3552  * RX and TX lists.
3553  */
3554 static void
3555 re_stop(struct rl_softc *sc)
3556 {
3557         int                     i;
3558         struct ifnet            *ifp;
3559         struct rl_txdesc        *txd;
3560         struct rl_rxdesc        *rxd;
3561
3562         RL_LOCK_ASSERT(sc);
3563
3564         ifp = sc->rl_ifp;
3565
3566         sc->rl_watchdog_timer = 0;
3567         callout_stop(&sc->rl_stat_callout);
3568         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
3569
3570         /*
3571          * Disable accepting frames to put RX MAC into idle state.
3572          * Otherwise it's possible to get frames while stop command
3573          * execution is in progress and controller can DMA the frame
3574          * to already freed RX buffer during that period.
3575          */
3576         CSR_WRITE_4(sc, RL_RXCFG, CSR_READ_4(sc, RL_RXCFG) &
3577             ~(RL_RXCFG_RX_ALLPHYS | RL_RXCFG_RX_INDIV | RL_RXCFG_RX_MULTI |
3578             RL_RXCFG_RX_BROAD));
3579
3580         if ((sc->rl_flags & RL_FLAG_WAIT_TXPOLL) != 0) {
3581                 for (i = RL_TIMEOUT; i > 0; i--) {
3582                         if ((CSR_READ_1(sc, sc->rl_txstart) &
3583                             RL_TXSTART_START) == 0)
3584                                 break;
3585                         DELAY(20);
3586                 }
3587                 if (i == 0)
3588                         device_printf(sc->rl_dev,
3589                             "stopping TX poll timed out!\n");
3590                 CSR_WRITE_1(sc, RL_COMMAND, 0x00);
3591         } else if ((sc->rl_flags & RL_FLAG_CMDSTOP) != 0) {
3592                 CSR_WRITE_1(sc, RL_COMMAND, RL_CMD_STOPREQ | RL_CMD_TX_ENB |
3593                     RL_CMD_RX_ENB);
3594                 if ((sc->rl_flags & RL_FLAG_CMDSTOP_WAIT_TXQ) != 0) {
3595                         for (i = RL_TIMEOUT; i > 0; i--) {
3596                                 if ((CSR_READ_4(sc, RL_TXCFG) &
3597                                     RL_TXCFG_QUEUE_EMPTY) != 0)
3598                                         break;
3599                                 DELAY(100);
3600                         }
3601                         if (i == 0)
3602                                 device_printf(sc->rl_dev,
3603                                    "stopping TXQ timed out!\n");
3604                 }
3605         } else
3606                 CSR_WRITE_1(sc, RL_COMMAND, 0x00);
3607         DELAY(1000);
3608         CSR_WRITE_2(sc, RL_IMR, 0x0000);
3609         CSR_WRITE_2(sc, RL_ISR, 0xFFFF);
3610
3611         if (sc->rl_head != NULL) {
3612                 m_freem(sc->rl_head);
3613                 sc->rl_head = sc->rl_tail = NULL;
3614         }
3615
3616         /* Free the TX list buffers. */
3617         for (i = 0; i < sc->rl_ldata.rl_tx_desc_cnt; i++) {
3618                 txd = &sc->rl_ldata.rl_tx_desc[i];
3619                 if (txd->tx_m != NULL) {
3620                         bus_dmamap_sync(sc->rl_ldata.rl_tx_mtag,
3621                             txd->tx_dmamap, BUS_DMASYNC_POSTWRITE);
3622                         bus_dmamap_unload(sc->rl_ldata.rl_tx_mtag,
3623                             txd->tx_dmamap);
3624                         m_freem(txd->tx_m);
3625                         txd->tx_m = NULL;
3626                 }
3627         }
3628
3629         /* Free the RX list buffers. */
3630         for (i = 0; i < sc->rl_ldata.rl_rx_desc_cnt; i++) {
3631                 rxd = &sc->rl_ldata.rl_rx_desc[i];
3632                 if (rxd->rx_m != NULL) {
3633                         bus_dmamap_sync(sc->rl_ldata.rl_rx_mtag,
3634                             rxd->rx_dmamap, BUS_DMASYNC_POSTREAD);
3635                         bus_dmamap_unload(sc->rl_ldata.rl_rx_mtag,
3636                             rxd->rx_dmamap);
3637                         m_freem(rxd->rx_m);
3638                         rxd->rx_m = NULL;
3639                 }
3640         }
3641
3642         if ((sc->rl_flags & RL_FLAG_JUMBOV2) != 0) {
3643                 for (i = 0; i < sc->rl_ldata.rl_rx_desc_cnt; i++) {
3644                         rxd = &sc->rl_ldata.rl_jrx_desc[i];
3645                         if (rxd->rx_m != NULL) {
3646                                 bus_dmamap_sync(sc->rl_ldata.rl_jrx_mtag,
3647                                     rxd->rx_dmamap, BUS_DMASYNC_POSTREAD);
3648                                 bus_dmamap_unload(sc->rl_ldata.rl_jrx_mtag,
3649                                     rxd->rx_dmamap);
3650                                 m_freem(rxd->rx_m);
3651                                 rxd->rx_m = NULL;
3652                         }
3653                 }
3654         }
3655 }
3656
3657 /*
3658  * Device suspend routine.  Stop the interface and save some PCI
3659  * settings in case the BIOS doesn't restore them properly on
3660  * resume.
3661  */
3662 static int
3663 re_suspend(device_t dev)
3664 {
3665         struct rl_softc         *sc;
3666
3667         sc = device_get_softc(dev);
3668
3669         RL_LOCK(sc);
3670         re_stop(sc);
3671         re_setwol(sc);
3672         sc->suspended = 1;
3673         RL_UNLOCK(sc);
3674
3675         return (0);
3676 }
3677
3678 /*
3679  * Device resume routine.  Restore some PCI settings in case the BIOS
3680  * doesn't, re-enable busmastering, and restart the interface if
3681  * appropriate.
3682  */
3683 static int
3684 re_resume(device_t dev)
3685 {
3686         struct rl_softc         *sc;
3687         struct ifnet            *ifp;
3688
3689         sc = device_get_softc(dev);
3690
3691         RL_LOCK(sc);
3692
3693         ifp = sc->rl_ifp;
3694         /* Take controller out of sleep mode. */
3695         if ((sc->rl_flags & RL_FLAG_MACSLEEP) != 0) {
3696                 if ((CSR_READ_1(sc, RL_MACDBG) & 0x80) == 0x80)
3697                         CSR_WRITE_1(sc, RL_GPIO,
3698                             CSR_READ_1(sc, RL_GPIO) | 0x01);
3699         }
3700
3701         /*
3702          * Clear WOL matching such that normal Rx filtering
3703          * wouldn't interfere with WOL patterns.
3704          */
3705         re_clrwol(sc);
3706
3707         /* reinitialize interface if necessary */
3708         if (ifp->if_flags & IFF_UP)
3709                 re_init_locked(sc);
3710
3711         sc->suspended = 0;
3712         RL_UNLOCK(sc);
3713
3714         return (0);
3715 }
3716
3717 /*
3718  * Stop all chip I/O so that the kernel's probe routines don't
3719  * get confused by errant DMAs when rebooting.
3720  */
3721 static int
3722 re_shutdown(device_t dev)
3723 {
3724         struct rl_softc         *sc;
3725
3726         sc = device_get_softc(dev);
3727
3728         RL_LOCK(sc);
3729         re_stop(sc);
3730         /*
3731          * Mark interface as down since otherwise we will panic if
3732          * interrupt comes in later on, which can happen in some
3733          * cases.
3734          */
3735         sc->rl_ifp->if_flags &= ~IFF_UP;
3736         re_setwol(sc);
3737         RL_UNLOCK(sc);
3738
3739         return (0);
3740 }
3741
3742 static void
3743 re_set_linkspeed(struct rl_softc *sc)
3744 {
3745         struct mii_softc *miisc;
3746         struct mii_data *mii;
3747         int aneg, i, phyno;
3748
3749         RL_LOCK_ASSERT(sc);
3750
3751         mii = device_get_softc(sc->rl_miibus);
3752         mii_pollstat(mii);
3753         aneg = 0;
3754         if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) ==
3755             (IFM_ACTIVE | IFM_AVALID)) {
3756                 switch IFM_SUBTYPE(mii->mii_media_active) {
3757                 case IFM_10_T:
3758                 case IFM_100_TX:
3759                         return;
3760                 case IFM_1000_T:
3761                         aneg++;
3762                         break;
3763                 default:
3764                         break;
3765                 }
3766         }
3767         miisc = LIST_FIRST(&mii->mii_phys);
3768         phyno = miisc->mii_phy;
3769         LIST_FOREACH(miisc, &mii->mii_phys, mii_list)
3770                 PHY_RESET(miisc);
3771         re_miibus_writereg(sc->rl_dev, phyno, MII_100T2CR, 0);
3772         re_miibus_writereg(sc->rl_dev, phyno,
3773             MII_ANAR, ANAR_TX_FD | ANAR_TX | ANAR_10_FD | ANAR_10 | ANAR_CSMA);
3774         re_miibus_writereg(sc->rl_dev, phyno,
3775             MII_BMCR, BMCR_AUTOEN | BMCR_STARTNEG);
3776         DELAY(1000);
3777         if (aneg != 0) {
3778                 /*
3779                  * Poll link state until re(4) get a 10/100Mbps link.
3780                  */
3781                 for (i = 0; i < MII_ANEGTICKS_GIGE; i++) {
3782                         mii_pollstat(mii);
3783                         if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID))
3784                             == (IFM_ACTIVE | IFM_AVALID)) {
3785                                 switch (IFM_SUBTYPE(mii->mii_media_active)) {
3786                                 case IFM_10_T:
3787                                 case IFM_100_TX:
3788                                         return;
3789                                 default:
3790                                         break;
3791                                 }
3792                         }
3793                         RL_UNLOCK(sc);
3794                         pause("relnk", hz);
3795                         RL_LOCK(sc);
3796                 }
3797                 if (i == MII_ANEGTICKS_GIGE)
3798                         device_printf(sc->rl_dev,
3799                             "establishing a link failed, WOL may not work!");
3800         }
3801         /*
3802          * No link, force MAC to have 100Mbps, full-duplex link.
3803          * MAC does not require reprogramming on resolved speed/duplex,
3804          * so this is just for completeness.
3805          */
3806         mii->mii_media_status = IFM_AVALID | IFM_ACTIVE;
3807         mii->mii_media_active = IFM_ETHER | IFM_100_TX | IFM_FDX;
3808 }
3809
3810 static void
3811 re_setwol(struct rl_softc *sc)
3812 {
3813         struct ifnet            *ifp;
3814         int                     pmc;
3815         uint16_t                pmstat;
3816         uint8_t                 v;
3817
3818         RL_LOCK_ASSERT(sc);
3819
3820         if (pci_find_cap(sc->rl_dev, PCIY_PMG, &pmc) != 0)
3821                 return;
3822
3823         ifp = sc->rl_ifp;
3824         /* Put controller into sleep mode. */
3825         if ((sc->rl_flags & RL_FLAG_MACSLEEP) != 0) {
3826                 if ((CSR_READ_1(sc, RL_MACDBG) & 0x80) == 0x80)
3827                         CSR_WRITE_1(sc, RL_GPIO,
3828                             CSR_READ_1(sc, RL_GPIO) & ~0x01);
3829         }
3830         if ((ifp->if_capenable & IFCAP_WOL) != 0) {
3831                 re_set_rxmode(sc);
3832                 if ((sc->rl_flags & RL_FLAG_WOL_MANLINK) != 0)
3833                         re_set_linkspeed(sc);
3834                 if ((sc->rl_flags & RL_FLAG_WOLRXENB) != 0)
3835                         CSR_WRITE_1(sc, RL_COMMAND, RL_CMD_RX_ENB);
3836         }
3837         /* Enable config register write. */
3838         CSR_WRITE_1(sc, RL_EECMD, RL_EE_MODE);
3839
3840         /* Enable PME. */
3841         v = CSR_READ_1(sc, sc->rl_cfg1);
3842         v &= ~RL_CFG1_PME;
3843         if ((ifp->if_capenable & IFCAP_WOL) != 0)
3844                 v |= RL_CFG1_PME;
3845         CSR_WRITE_1(sc, sc->rl_cfg1, v);
3846
3847         v = CSR_READ_1(sc, sc->rl_cfg3);
3848         v &= ~(RL_CFG3_WOL_LINK | RL_CFG3_WOL_MAGIC);
3849         if ((ifp->if_capenable & IFCAP_WOL_MAGIC) != 0)
3850                 v |= RL_CFG3_WOL_MAGIC;
3851         CSR_WRITE_1(sc, sc->rl_cfg3, v);
3852
3853         v = CSR_READ_1(sc, sc->rl_cfg5);
3854         v &= ~(RL_CFG5_WOL_BCAST | RL_CFG5_WOL_MCAST | RL_CFG5_WOL_UCAST |
3855             RL_CFG5_WOL_LANWAKE);
3856         if ((ifp->if_capenable & IFCAP_WOL_UCAST) != 0)
3857                 v |= RL_CFG5_WOL_UCAST;
3858         if ((ifp->if_capenable & IFCAP_WOL_MCAST) != 0)
3859                 v |= RL_CFG5_WOL_MCAST | RL_CFG5_WOL_BCAST;
3860         if ((ifp->if_capenable & IFCAP_WOL) != 0)
3861                 v |= RL_CFG5_WOL_LANWAKE;
3862         CSR_WRITE_1(sc, sc->rl_cfg5, v);
3863
3864         /* Config register write done. */
3865         CSR_WRITE_1(sc, RL_EECMD, RL_EEMODE_OFF);
3866
3867         if ((ifp->if_capenable & IFCAP_WOL) == 0 &&
3868             (sc->rl_flags & RL_FLAG_PHYWAKE_PM) != 0)
3869                 CSR_WRITE_1(sc, RL_PMCH, CSR_READ_1(sc, RL_PMCH) & ~0x80);
3870         /*
3871          * It seems that hardware resets its link speed to 100Mbps in
3872          * power down mode so switching to 100Mbps in driver is not
3873          * needed.
3874          */
3875
3876         /* Request PME if WOL is requested. */
3877         pmstat = pci_read_config(sc->rl_dev, pmc + PCIR_POWER_STATUS, 2);
3878         pmstat &= ~(PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE);
3879         if ((ifp->if_capenable & IFCAP_WOL) != 0)
3880                 pmstat |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
3881         pci_write_config(sc->rl_dev, pmc + PCIR_POWER_STATUS, pmstat, 2);
3882 }
3883
3884 static void
3885 re_clrwol(struct rl_softc *sc)
3886 {
3887         int                     pmc;
3888         uint8_t                 v;
3889
3890         RL_LOCK_ASSERT(sc);
3891
3892         if (pci_find_cap(sc->rl_dev, PCIY_PMG, &pmc) != 0)
3893                 return;
3894
3895         /* Enable config register write. */
3896         CSR_WRITE_1(sc, RL_EECMD, RL_EE_MODE);
3897
3898         v = CSR_READ_1(sc, sc->rl_cfg3);
3899         v &= ~(RL_CFG3_WOL_LINK | RL_CFG3_WOL_MAGIC);
3900         CSR_WRITE_1(sc, sc->rl_cfg3, v);
3901
3902         /* Config register write done. */
3903         CSR_WRITE_1(sc, RL_EECMD, RL_EEMODE_OFF);
3904
3905         v = CSR_READ_1(sc, sc->rl_cfg5);
3906         v &= ~(RL_CFG5_WOL_BCAST | RL_CFG5_WOL_MCAST | RL_CFG5_WOL_UCAST);
3907         v &= ~RL_CFG5_WOL_LANWAKE;
3908         CSR_WRITE_1(sc, sc->rl_cfg5, v);
3909 }
3910
3911 static void
3912 re_add_sysctls(struct rl_softc *sc)
3913 {
3914         struct sysctl_ctx_list  *ctx;
3915         struct sysctl_oid_list  *children;
3916         int                     error;
3917
3918         ctx = device_get_sysctl_ctx(sc->rl_dev);
3919         children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->rl_dev));
3920
3921         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "stats",
3922             CTLTYPE_INT | CTLFLAG_RW, sc, 0, re_sysctl_stats, "I",
3923             "Statistics Information");
3924         if ((sc->rl_flags & (RL_FLAG_MSI | RL_FLAG_MSIX)) == 0)
3925                 return;
3926
3927         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "int_rx_mod",
3928             CTLTYPE_INT | CTLFLAG_RW, &sc->rl_int_rx_mod, 0,
3929             sysctl_hw_re_int_mod, "I", "re RX interrupt moderation");
3930         /* Pull in device tunables. */
3931         sc->rl_int_rx_mod = RL_TIMER_DEFAULT;
3932         error = resource_int_value(device_get_name(sc->rl_dev),
3933             device_get_unit(sc->rl_dev), "int_rx_mod", &sc->rl_int_rx_mod);
3934         if (error == 0) {
3935                 if (sc->rl_int_rx_mod < RL_TIMER_MIN ||
3936                     sc->rl_int_rx_mod > RL_TIMER_MAX) {
3937                         device_printf(sc->rl_dev, "int_rx_mod value out of "
3938                             "range; using default: %d\n",
3939                             RL_TIMER_DEFAULT);
3940                         sc->rl_int_rx_mod = RL_TIMER_DEFAULT;
3941                 }
3942         }
3943
3944 }
3945
3946 static int
3947 re_sysctl_stats(SYSCTL_HANDLER_ARGS)
3948 {
3949         struct rl_softc         *sc;
3950         struct rl_stats         *stats;
3951         int                     error, i, result;
3952
3953         result = -1;
3954         error = sysctl_handle_int(oidp, &result, 0, req);
3955         if (error || req->newptr == NULL)
3956                 return (error);
3957
3958         if (result == 1) {
3959                 sc = (struct rl_softc *)arg1;
3960                 RL_LOCK(sc);
3961                 if ((sc->rl_ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
3962                         RL_UNLOCK(sc);
3963                         goto done;
3964                 }
3965                 bus_dmamap_sync(sc->rl_ldata.rl_stag,
3966                     sc->rl_ldata.rl_smap, BUS_DMASYNC_PREREAD);
3967                 CSR_WRITE_4(sc, RL_DUMPSTATS_HI,
3968                     RL_ADDR_HI(sc->rl_ldata.rl_stats_addr));
3969                 CSR_WRITE_4(sc, RL_DUMPSTATS_LO,
3970                     RL_ADDR_LO(sc->rl_ldata.rl_stats_addr));
3971                 CSR_WRITE_4(sc, RL_DUMPSTATS_LO,
3972                     RL_ADDR_LO(sc->rl_ldata.rl_stats_addr |
3973                     RL_DUMPSTATS_START));
3974                 for (i = RL_TIMEOUT; i > 0; i--) {
3975                         if ((CSR_READ_4(sc, RL_DUMPSTATS_LO) &
3976                             RL_DUMPSTATS_START) == 0)
3977                                 break;
3978                         DELAY(1000);
3979                 }
3980                 bus_dmamap_sync(sc->rl_ldata.rl_stag,
3981                     sc->rl_ldata.rl_smap, BUS_DMASYNC_POSTREAD);
3982                 RL_UNLOCK(sc);
3983                 if (i == 0) {
3984                         device_printf(sc->rl_dev,
3985                             "DUMP statistics request timed out\n");
3986                         return (ETIMEDOUT);
3987                 }
3988 done:
3989                 stats = sc->rl_ldata.rl_stats;
3990                 printf("%s statistics:\n", device_get_nameunit(sc->rl_dev));
3991                 printf("Tx frames : %ju\n",
3992                     (uintmax_t)le64toh(stats->rl_tx_pkts));
3993                 printf("Rx frames : %ju\n",
3994                     (uintmax_t)le64toh(stats->rl_rx_pkts));
3995                 printf("Tx errors : %ju\n",
3996                     (uintmax_t)le64toh(stats->rl_tx_errs));
3997                 printf("Rx errors : %u\n",
3998                     le32toh(stats->rl_rx_errs));
3999                 printf("Rx missed frames : %u\n",
4000                     (uint32_t)le16toh(stats->rl_missed_pkts));
4001                 printf("Rx frame alignment errs : %u\n",
4002                     (uint32_t)le16toh(stats->rl_rx_framealign_errs));
4003                 printf("Tx single collisions : %u\n",
4004                     le32toh(stats->rl_tx_onecoll));
4005                 printf("Tx multiple collisions : %u\n",
4006                     le32toh(stats->rl_tx_multicolls));
4007                 printf("Rx unicast frames : %ju\n",
4008                     (uintmax_t)le64toh(stats->rl_rx_ucasts));
4009                 printf("Rx broadcast frames : %ju\n",
4010                     (uintmax_t)le64toh(stats->rl_rx_bcasts));
4011                 printf("Rx multicast frames : %u\n",
4012                     le32toh(stats->rl_rx_mcasts));
4013                 printf("Tx aborts : %u\n",
4014                     (uint32_t)le16toh(stats->rl_tx_aborts));
4015                 printf("Tx underruns : %u\n",
4016                     (uint32_t)le16toh(stats->rl_rx_underruns));
4017         }
4018
4019         return (error);
4020 }
4021
4022 static int
4023 sysctl_int_range(SYSCTL_HANDLER_ARGS, int low, int high)
4024 {
4025         int error, value;
4026
4027         if (arg1 == NULL)
4028                 return (EINVAL);
4029         value = *(int *)arg1;
4030         error = sysctl_handle_int(oidp, &value, 0, req);
4031         if (error || req->newptr == NULL)
4032                 return (error);
4033         if (value < low || value > high)
4034                 return (EINVAL);
4035         *(int *)arg1 = value;
4036
4037         return (0);
4038 }
4039
4040 static int
4041 sysctl_hw_re_int_mod(SYSCTL_HANDLER_ARGS)
4042 {
4043
4044         return (sysctl_int_range(oidp, arg1, arg2, req, RL_TIMER_MIN,
4045             RL_TIMER_MAX));
4046 }