sys/dev/re/if_re.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-4-Clause
   3  *
   4  * Copyright (c) 1997, 1998-2003
   5  *      Bill Paul <wpaul@windriver.com>.  All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  * 3. All advertising materials mentioning features or use of this software
  16  *    must display the following acknowledgement:
  17  *      This product includes software developed by Bill Paul.
  18  * 4. Neither the name of the author nor the names of any co-contributors
  19  *    may be used to endorse or promote products derived from this software
  20  *    without specific prior written permission.
  21  *
  22  * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND
  23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  25  * ARE DISCLAIMED.  IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD
  26  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  27  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  28  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  29  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  30  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  31  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  32  * THE POSSIBILITY OF SUCH DAMAGE.
  33  */
  34
  35 #include <sys/cdefs.h>
  36 __FBSDID("$FreeBSD$");
  37
  38 /*
  39  * RealTek 8139C+/8169/8169S/8110S/8168/8111/8101E PCI NIC driver
  40  *
  41  * Written by Bill Paul <wpaul@windriver.com>
  42  * Senior Networking Software Engineer
  43  * Wind River Systems
  44  */
  45
  46 /*
  47  * This driver is designed to support RealTek's next generation of
  48  * 10/100 and 10/100/1000 PCI ethernet controllers. There are currently
  49  * seven devices in this family: the RTL8139C+, the RTL8169, the RTL8169S,
  50  * RTL8110S, the RTL8168, the RTL8111 and the RTL8101E.
  51  *
  52  * The 8139C+ is a 10/100 ethernet chip. It is backwards compatible
  53  * with the older 8139 family, however it also supports a special
  54  * C+ mode of operation that provides several new performance enhancing
  55  * features. These include:
  56  *
  57  *      o Descriptor based DMA mechanism. Each descriptor represents
  58  *        a single packet fragment. Data buffers may be aligned on
  59  *        any byte boundary.
  60  *
  61  *      o 64-bit DMA
  62  *
  63  *      o TCP/IP checksum offload for both RX and TX
  64  *
  65  *      o High and normal priority transmit DMA rings
  66  *
  67  *      o VLAN tag insertion and extraction
  68  *
  69  *      o TCP large send (segmentation offload)
  70  *
  71  * Like the 8139, the 8139C+ also has a built-in 10/100 PHY. The C+
  72  * programming API is fairly straightforward. The RX filtering, EEPROM
  73  * access and PHY access is the same as it is on the older 8139 series
  74  * chips.
  75  *
  76  * The 8169 is a 64-bit 10/100/1000 gigabit ethernet MAC. It has almost the
  77  * same programming API and feature set as the 8139C+ with the following
  78  * differences and additions:
  79  *
  80  *      o 1000Mbps mode
  81  *
  82  *      o Jumbo frames
  83  *
  84  *      o GMII and TBI ports/registers for interfacing with copper
  85  *        or fiber PHYs
  86  *
  87  *      o RX and TX DMA rings can have up to 1024 descriptors
  88  *        (the 8139C+ allows a maximum of 64)
  89  *
  90  *      o Slight differences in register layout from the 8139C+
  91  *
  92  * The TX start and timer interrupt registers are at different locations
  93  * on the 8169 than they are on the 8139C+. Also, the status word in the
  94  * RX descriptor has a slightly different bit layout. The 8169 does not
  95  * have a built-in PHY. Most reference boards use a Marvell 88E1000 'Alaska'
  96  * copper gigE PHY.
  97  *
  98  * The 8169S/8110S 10/100/1000 devices have built-in copper gigE PHYs
  99  * (the 'S' stands for 'single-chip'). These devices have the same
 100  * programming API as the older 8169, but also have some vendor-specific
 101  * registers for the on-board PHY. The 8110S is a LAN-on-motherboard
 102  * part designed to be pin-compatible with the RealTek 8100 10/100 chip.
 103  *
 104  * This driver takes advantage of the RX and TX checksum offload and
 105  * VLAN tag insertion/extraction features. It also implements TX
 106  * interrupt moderation using the timer interrupt registers, which
 107  * significantly reduces TX interrupt load. There is also support
 108  * for jumbo frames, however the 8169/8169S/8110S can not transmit
 109  * jumbo frames larger than 7440, so the max MTU possible with this
 110  * driver is 7422 bytes.
 111  */
 112
 113 #ifdef HAVE_KERNEL_OPTION_HEADERS
 114 #include "opt_device_polling.h"
 115 #endif
 116
 117 #include <sys/param.h>
 118 #include <sys/endian.h>
 119 #include <sys/systm.h>
 120 #include <sys/sockio.h>
 121 #include <sys/mbuf.h>
 122 #include <sys/malloc.h>
 123 #include <sys/module.h>
 124 #include <sys/kernel.h>
 125 #include <sys/socket.h>
 126 #include <sys/lock.h>
 127 #include <sys/mutex.h>
 128 #include <sys/sysctl.h>
 129 #include <sys/taskqueue.h>
 130
 131 #include <net/if.h>
 132 #include <net/if_var.h>
 133 #include <net/if_arp.h>
 134 #include <net/ethernet.h>
 135 #include <net/if_dl.h>
 136 #include <net/if_media.h>
 137 #include <net/if_types.h>
 138 #include <net/if_vlan_var.h>
 139
 140 #include <net/bpf.h>
 141
 142 #include <machine/bus.h>
 143 #include <machine/resource.h>
 144 #include <sys/bus.h>
 145 #include <sys/rman.h>
 146
 147 #include <dev/mii/mii.h>
 148 #include <dev/mii/miivar.h>
 149
 150 #include <dev/pci/pcireg.h>
 151 #include <dev/pci/pcivar.h>
 152
 153 #include <dev/rl/if_rlreg.h>
 154
 155 MODULE_DEPEND(re, pci, 1, 1, 1);
 156 MODULE_DEPEND(re, ether, 1, 1, 1);
 157 MODULE_DEPEND(re, miibus, 1, 1, 1);
 158
 159 /* "device miibus" required.  See GENERIC if you get errors here. */
 160 #include "miibus_if.h"
 161
 162 /* Tunables. */
 163 static int intr_filter = 0;
 164 TUNABLE_INT("hw.re.intr_filter", &intr_filter);
 165 static int msi_disable = 0;
 166 TUNABLE_INT("hw.re.msi_disable", &msi_disable);
 167 static int msix_disable = 0;
 168 TUNABLE_INT("hw.re.msix_disable", &msix_disable);
 169 static int prefer_iomap = 0;
 170 TUNABLE_INT("hw.re.prefer_iomap", &prefer_iomap);
 171
 172 #define RE_CSUM_FEATURES    (CSUM_IP | CSUM_TCP | CSUM_UDP)
 173
 174 /*
 175  * Various supported device vendors/types and their names.
 176  */
 177 static const struct rl_type re_devs[] = {
 178         { DLINK_VENDORID, DLINK_DEVICEID_528T, 0,
 179             "D-Link DGE-528(T) Gigabit Ethernet Adapter" },
 180         { DLINK_VENDORID, DLINK_DEVICEID_530T_REVC, 0,
 181             "D-Link DGE-530(T) Gigabit Ethernet Adapter" },
 182         { RT_VENDORID, RT_DEVICEID_8139, 0,
 183             "RealTek 8139C+ 10/100BaseTX" },
 184         { RT_VENDORID, RT_DEVICEID_8101E, 0,
 185             "RealTek 810xE PCIe 10/100baseTX" },
 186         { RT_VENDORID, RT_DEVICEID_8168, 0,
 187             "RealTek 8168/8111 B/C/CP/D/DP/E/F/G PCIe Gigabit Ethernet" },
 188         { NCUBE_VENDORID, RT_DEVICEID_8168, 0,
 189             "TP-Link TG-3468 v2 (RTL8168) Gigabit Ethernet" },
 190         { RT_VENDORID, RT_DEVICEID_8169, 0,
 191             "RealTek 8169/8169S/8169SB(L)/8110S/8110SB(L) Gigabit Ethernet" },
 192         { RT_VENDORID, RT_DEVICEID_8169SC, 0,
 193             "RealTek 8169SC/8110SC Single-chip Gigabit Ethernet" },
 194         { COREGA_VENDORID, COREGA_DEVICEID_CGLAPCIGT, 0,
 195             "Corega CG-LAPCIGT (RTL8169S) Gigabit Ethernet" },
 196         { LINKSYS_VENDORID, LINKSYS_DEVICEID_EG1032, 0,
 197             "Linksys EG1032 (RTL8169S) Gigabit Ethernet" },
 198         { USR_VENDORID, USR_DEVICEID_997902, 0,
 199             "US Robotics 997902 (RTL8169S) Gigabit Ethernet" }
 200 };
 201
 202 static const struct rl_hwrev re_hwrevs[] = {
 203         { RL_HWREV_8139, RL_8139, "", RL_MTU },
 204         { RL_HWREV_8139A, RL_8139, "A", RL_MTU },
 205         { RL_HWREV_8139AG, RL_8139, "A-G", RL_MTU },
 206         { RL_HWREV_8139B, RL_8139, "B", RL_MTU },
 207         { RL_HWREV_8130, RL_8139, "8130", RL_MTU },
 208         { RL_HWREV_8139C, RL_8139, "C", RL_MTU },
 209         { RL_HWREV_8139D, RL_8139, "8139D/8100B/8100C", RL_MTU },
 210         { RL_HWREV_8139CPLUS, RL_8139CPLUS, "C+", RL_MTU },
 211         { RL_HWREV_8168B_SPIN1, RL_8169, "8168", RL_JUMBO_MTU },
 212         { RL_HWREV_8169, RL_8169, "8169", RL_JUMBO_MTU },
 213         { RL_HWREV_8169S, RL_8169, "8169S", RL_JUMBO_MTU },
 214         { RL_HWREV_8110S, RL_8169, "8110S", RL_JUMBO_MTU },
 215         { RL_HWREV_8169_8110SB, RL_8169, "8169SB/8110SB", RL_JUMBO_MTU },
 216         { RL_HWREV_8169_8110SC, RL_8169, "8169SC/8110SC", RL_JUMBO_MTU },
 217         { RL_HWREV_8169_8110SBL, RL_8169, "8169SBL/8110SBL", RL_JUMBO_MTU },
 218         { RL_HWREV_8169_8110SCE, RL_8169, "8169SC/8110SC", RL_JUMBO_MTU },
 219         { RL_HWREV_8100, RL_8139, "8100", RL_MTU },
 220         { RL_HWREV_8101, RL_8139, "8101", RL_MTU },
 221         { RL_HWREV_8100E, RL_8169, "8100E", RL_MTU },
 222         { RL_HWREV_8101E, RL_8169, "8101E", RL_MTU },
 223         { RL_HWREV_8102E, RL_8169, "8102E", RL_MTU },
 224         { RL_HWREV_8102EL, RL_8169, "8102EL", RL_MTU },
 225         { RL_HWREV_8102EL_SPIN1, RL_8169, "8102EL", RL_MTU },
 226         { RL_HWREV_8103E, RL_8169, "8103E", RL_MTU },
 227         { RL_HWREV_8401E, RL_8169, "8401E", RL_MTU },
 228         { RL_HWREV_8402, RL_8169, "8402", RL_MTU },
 229         { RL_HWREV_8105E, RL_8169, "8105E", RL_MTU },
 230         { RL_HWREV_8105E_SPIN1, RL_8169, "8105E", RL_MTU },
 231         { RL_HWREV_8106E, RL_8169, "8106E", RL_MTU },
 232         { RL_HWREV_8168B_SPIN2, RL_8169, "8168", RL_JUMBO_MTU },
 233         { RL_HWREV_8168B_SPIN3, RL_8169, "8168", RL_JUMBO_MTU },
 234         { RL_HWREV_8168C, RL_8169, "8168C/8111C", RL_JUMBO_MTU_6K },
 235         { RL_HWREV_8168C_SPIN2, RL_8169, "8168C/8111C", RL_JUMBO_MTU_6K },
 236         { RL_HWREV_8168CP, RL_8169, "8168CP/8111CP", RL_JUMBO_MTU_6K },
 237         { RL_HWREV_8168D, RL_8169, "8168D/8111D", RL_JUMBO_MTU_9K },
 238         { RL_HWREV_8168DP, RL_8169, "8168DP/8111DP", RL_JUMBO_MTU_9K },
 239         { RL_HWREV_8168E, RL_8169, "8168E/8111E", RL_JUMBO_MTU_9K},
 240         { RL_HWREV_8168E_VL, RL_8169, "8168E/8111E-VL", RL_JUMBO_MTU_6K},
 241         { RL_HWREV_8168EP, RL_8169, "8168EP/8111EP", RL_JUMBO_MTU_9K},
 242         { RL_HWREV_8168F, RL_8169, "8168F/8111F", RL_JUMBO_MTU_9K},
 243         { RL_HWREV_8168G, RL_8169, "8168G/8111G", RL_JUMBO_MTU_9K},
 244         { RL_HWREV_8168GU, RL_8169, "8168GU/8111GU", RL_JUMBO_MTU_9K},
 245         { RL_HWREV_8168H, RL_8169, "8168H/8111H", RL_JUMBO_MTU_9K},
 246         { RL_HWREV_8411, RL_8169, "8411", RL_JUMBO_MTU_9K},
 247         { RL_HWREV_8411B, RL_8169, "8411B", RL_JUMBO_MTU_9K},
 248         { 0, 0, NULL, 0 }
 249 };
 250
 251 static int re_probe             (device_t);
 252 static int re_attach            (device_t);
 253 static int re_detach            (device_t);
 254
 255 static int re_encap             (struct rl_softc *, struct mbuf **);
 256
 257 static void re_dma_map_addr     (void *, bus_dma_segment_t *, int, int);
 258 static int re_allocmem          (device_t, struct rl_softc *);
 259 static __inline void re_discard_rxbuf
 260                                 (struct rl_softc *, int);
 261 static int re_newbuf            (struct rl_softc *, int);
 262 static int re_jumbo_newbuf      (struct rl_softc *, int);
 263 static int re_rx_list_init      (struct rl_softc *);
 264 static int re_jrx_list_init     (struct rl_softc *);
 265 static int re_tx_list_init      (struct rl_softc *);
 266 #ifdef RE_FIXUP_RX
 267 static __inline void re_fixup_rx
 268                                 (struct mbuf *);
 269 #endif
 270 static int re_rxeof             (struct rl_softc *, int *);
 271 static void re_txeof            (struct rl_softc *);
 272 #ifdef DEVICE_POLLING
 273 static int re_poll              (struct ifnet *, enum poll_cmd, int);
 274 static int re_poll_locked       (struct ifnet *, enum poll_cmd, int);
 275 #endif
 276 static int re_intr              (void *);
 277 static void re_intr_msi         (void *);
 278 static void re_tick             (void *);
 279 static void re_int_task         (void *, int);
 280 static void re_start            (struct ifnet *);
 281 static void re_start_locked     (struct ifnet *);
 282 static int re_ioctl             (struct ifnet *, u_long, caddr_t);
 283 static void re_init             (void *);
 284 static void re_init_locked      (struct rl_softc *);
 285 static void re_stop             (struct rl_softc *);
 286 static void re_watchdog         (struct rl_softc *);
 287 static int re_suspend           (device_t);
 288 static int re_resume            (device_t);
 289 static int re_shutdown          (device_t);
 290 static int re_ifmedia_upd       (struct ifnet *);
 291 static void re_ifmedia_sts      (struct ifnet *, struct ifmediareq *);
 292
 293 static void re_eeprom_putbyte   (struct rl_softc *, int);
 294 static void re_eeprom_getword   (struct rl_softc *, int, u_int16_t *);
 295 static void re_read_eeprom      (struct rl_softc *, caddr_t, int, int);
 296 static int re_gmii_readreg      (device_t, int, int);
 297 static int re_gmii_writereg     (device_t, int, int, int);
 298
 299 static int re_miibus_readreg    (device_t, int, int);
 300 static int re_miibus_writereg   (device_t, int, int, int);
 301 static void re_miibus_statchg   (device_t);
 302
 303 static void re_set_jumbo        (struct rl_softc *, int);
 304 static void re_set_rxmode               (struct rl_softc *);
 305 static void re_reset            (struct rl_softc *);
 306 static void re_setwol           (struct rl_softc *);
 307 static void re_clrwol           (struct rl_softc *);
 308 static void re_set_linkspeed    (struct rl_softc *);
 309
 310 #ifdef DEV_NETMAP       /* see ixgbe.c for details */
 311 #include <dev/netmap/if_re_netmap.h>
 312 MODULE_DEPEND(re, netmap, 1, 1, 1);
 313 #endif /* !DEV_NETMAP */
 314
 315 #ifdef RE_DIAG
 316 static int re_diag              (struct rl_softc *);
 317 #endif
 318
 319 static void re_add_sysctls      (struct rl_softc *);
 320 static int re_sysctl_stats      (SYSCTL_HANDLER_ARGS);
 321 static int sysctl_int_range     (SYSCTL_HANDLER_ARGS, int, int);
 322 static int sysctl_hw_re_int_mod (SYSCTL_HANDLER_ARGS);
 323
 324 static device_method_t re_methods[] = {
 325         /* Device interface */
 326         DEVMETHOD(device_probe,         re_probe),
 327         DEVMETHOD(device_attach,        re_attach),
 328         DEVMETHOD(device_detach,        re_detach),
 329         DEVMETHOD(device_suspend,       re_suspend),
 330         DEVMETHOD(device_resume,        re_resume),
 331         DEVMETHOD(device_shutdown,      re_shutdown),
 332
 333         /* MII interface */
 334         DEVMETHOD(miibus_readreg,       re_miibus_readreg),
 335         DEVMETHOD(miibus_writereg,      re_miibus_writereg),
 336         DEVMETHOD(miibus_statchg,       re_miibus_statchg),
 337
 338         DEVMETHOD_END
 339 };
 340
 341 static driver_t re_driver = {
 342         "re",
 343         re_methods,
 344         sizeof(struct rl_softc)
 345 };
 346
 347 static devclass_t re_devclass;
 348
 349 DRIVER_MODULE(re, pci, re_driver, re_devclass, 0, 0);
 350 DRIVER_MODULE(miibus, re, miibus_driver, miibus_devclass, 0, 0);
 351
 352 #define EE_SET(x)                                       \
 353         CSR_WRITE_1(sc, RL_EECMD,                       \
 354                 CSR_READ_1(sc, RL_EECMD) | x)
 355
 356 #define EE_CLR(x)                                       \
 357         CSR_WRITE_1(sc, RL_EECMD,                       \
 358                 CSR_READ_1(sc, RL_EECMD) & ~x)
 359
 360 /*
 361  * Send a read command and address to the EEPROM, check for ACK.
 362  */
 363 static void
 364 re_eeprom_putbyte(struct rl_softc *sc, int addr)
 365 {
 366         int                     d, i;
 367
 368         d = addr | (RL_9346_READ << sc->rl_eewidth);
 369
 370         /*
 371          * Feed in each bit and strobe the clock.
 372          */
 373
 374         for (i = 1 << (sc->rl_eewidth + 3); i; i >>= 1) {
 375                 if (d & i) {
 376                         EE_SET(RL_EE_DATAIN);
 377                 } else {
 378                         EE_CLR(RL_EE_DATAIN);
 379                 }
 380                 DELAY(100);
 381                 EE_SET(RL_EE_CLK);
 382                 DELAY(150);
 383                 EE_CLR(RL_EE_CLK);
 384                 DELAY(100);
 385         }
 386 }
 387
 388 /*
 389  * Read a word of data stored in the EEPROM at address 'addr.'
 390  */
 391 static void
 392 re_eeprom_getword(struct rl_softc *sc, int addr, u_int16_t *dest)
 393 {
 394         int                     i;
 395         u_int16_t               word = 0;
 396
 397         /*
 398          * Send address of word we want to read.
 399          */
 400         re_eeprom_putbyte(sc, addr);
 401
 402         /*
 403          * Start reading bits from EEPROM.
 404          */
 405         for (i = 0x8000; i; i >>= 1) {
 406                 EE_SET(RL_EE_CLK);
 407                 DELAY(100);
 408                 if (CSR_READ_1(sc, RL_EECMD) & RL_EE_DATAOUT)
 409                         word |= i;
 410                 EE_CLR(RL_EE_CLK);
 411                 DELAY(100);
 412         }
 413
 414         *dest = word;
 415 }
 416
 417 /*
 418  * Read a sequence of words from the EEPROM.
 419  */
 420 static void
 421 re_read_eeprom(struct rl_softc *sc, caddr_t dest, int off, int cnt)
 422 {
 423         int                     i;
 424         u_int16_t               word = 0, *ptr;
 425
 426         CSR_SETBIT_1(sc, RL_EECMD, RL_EEMODE_PROGRAM);
 427
 428         DELAY(100);
 429
 430         for (i = 0; i < cnt; i++) {
 431                 CSR_SETBIT_1(sc, RL_EECMD, RL_EE_SEL);
 432                 re_eeprom_getword(sc, off + i, &word);
 433                 CSR_CLRBIT_1(sc, RL_EECMD, RL_EE_SEL);
 434                 ptr = (u_int16_t *)(dest + (i * 2));
 435                 *ptr = word;
 436         }
 437
 438         CSR_CLRBIT_1(sc, RL_EECMD, RL_EEMODE_PROGRAM);
 439 }
 440
 441 static int
 442 re_gmii_readreg(device_t dev, int phy, int reg)
 443 {
 444         struct rl_softc         *sc;
 445         u_int32_t               rval;
 446         int                     i;
 447
 448         sc = device_get_softc(dev);
 449
 450         /* Let the rgephy driver read the GMEDIASTAT register */
 451
 452         if (reg == RL_GMEDIASTAT) {
 453                 rval = CSR_READ_1(sc, RL_GMEDIASTAT);
 454                 return (rval);
 455         }
 456
 457         CSR_WRITE_4(sc, RL_PHYAR, reg << 16);
 458
 459         for (i = 0; i < RL_PHY_TIMEOUT; i++) {
 460                 rval = CSR_READ_4(sc, RL_PHYAR);
 461                 if (rval & RL_PHYAR_BUSY)
 462                         break;
 463                 DELAY(25);
 464         }
 465
 466         if (i == RL_PHY_TIMEOUT) {
 467                 device_printf(sc->rl_dev, "PHY read failed\n");
 468                 return (0);
 469         }
 470
 471         /*
 472          * Controller requires a 20us delay to process next MDIO request.
 473          */
 474         DELAY(20);
 475
 476         return (rval & RL_PHYAR_PHYDATA);
 477 }
 478
 479 static int
 480 re_gmii_writereg(device_t dev, int phy, int reg, int data)
 481 {
 482         struct rl_softc         *sc;
 483         u_int32_t               rval;
 484         int                     i;
 485
 486         sc = device_get_softc(dev);
 487
 488         CSR_WRITE_4(sc, RL_PHYAR, (reg << 16) |
 489             (data & RL_PHYAR_PHYDATA) | RL_PHYAR_BUSY);
 490
 491         for (i = 0; i < RL_PHY_TIMEOUT; i++) {
 492                 rval = CSR_READ_4(sc, RL_PHYAR);
 493                 if (!(rval & RL_PHYAR_BUSY))
 494                         break;
 495                 DELAY(25);
 496         }
 497
 498         if (i == RL_PHY_TIMEOUT) {
 499                 device_printf(sc->rl_dev, "PHY write failed\n");
 500                 return (0);
 501         }
 502
 503         /*
 504          * Controller requires a 20us delay to process next MDIO request.
 505          */
 506         DELAY(20);
 507
 508         return (0);
 509 }
 510
 511 static int
 512 re_miibus_readreg(device_t dev, int phy, int reg)
 513 {
 514         struct rl_softc         *sc;
 515         u_int16_t               rval = 0;
 516         u_int16_t               re8139_reg = 0;
 517
 518         sc = device_get_softc(dev);
 519
 520         if (sc->rl_type == RL_8169) {
 521                 rval = re_gmii_readreg(dev, phy, reg);
 522                 return (rval);
 523         }
 524
 525         switch (reg) {
 526         case MII_BMCR:
 527                 re8139_reg = RL_BMCR;
 528                 break;
 529         case MII_BMSR:
 530                 re8139_reg = RL_BMSR;
 531                 break;
 532         case MII_ANAR:
 533                 re8139_reg = RL_ANAR;
 534                 break;
 535         case MII_ANER:
 536                 re8139_reg = RL_ANER;
 537                 break;
 538         case MII_ANLPAR:
 539                 re8139_reg = RL_LPAR;
 540                 break;
 541         case MII_PHYIDR1:
 542         case MII_PHYIDR2:
 543                 return (0);
 544         /*
 545          * Allow the rlphy driver to read the media status
 546          * register. If we have a link partner which does not
 547          * support NWAY, this is the register which will tell
 548          * us the results of parallel detection.
 549          */
 550         case RL_MEDIASTAT:
 551                 rval = CSR_READ_1(sc, RL_MEDIASTAT);
 552                 return (rval);
 553         default:
 554                 device_printf(sc->rl_dev, "bad phy register\n");
 555                 return (0);
 556         }
 557         rval = CSR_READ_2(sc, re8139_reg);
 558         if (sc->rl_type == RL_8139CPLUS && re8139_reg == RL_BMCR) {
 559                 /* 8139C+ has different bit layout. */
 560                 rval &= ~(BMCR_LOOP | BMCR_ISO);
 561         }
 562         return (rval);
 563 }
 564
 565 static int
 566 re_miibus_writereg(device_t dev, int phy, int reg, int data)
 567 {
 568         struct rl_softc         *sc;
 569         u_int16_t               re8139_reg = 0;
 570         int                     rval = 0;
 571
 572         sc = device_get_softc(dev);
 573
 574         if (sc->rl_type == RL_8169) {
 575                 rval = re_gmii_writereg(dev, phy, reg, data);
 576                 return (rval);
 577         }
 578
 579         switch (reg) {
 580         case MII_BMCR:
 581                 re8139_reg = RL_BMCR;
 582                 if (sc->rl_type == RL_8139CPLUS) {
 583                         /* 8139C+ has different bit layout. */
 584                         data &= ~(BMCR_LOOP | BMCR_ISO);
 585                 }
 586                 break;
 587         case MII_BMSR:
 588                 re8139_reg = RL_BMSR;
 589                 break;
 590         case MII_ANAR:
 591                 re8139_reg = RL_ANAR;
 592                 break;
 593         case MII_ANER:
 594                 re8139_reg = RL_ANER;
 595                 break;
 596         case MII_ANLPAR:
 597                 re8139_reg = RL_LPAR;
 598                 break;
 599         case MII_PHYIDR1:
 600         case MII_PHYIDR2:
 601                 return (0);
 602                 break;
 603         default:
 604                 device_printf(sc->rl_dev, "bad phy register\n");
 605                 return (0);
 606         }
 607         CSR_WRITE_2(sc, re8139_reg, data);
 608         return (0);
 609 }
 610
 611 static void
 612 re_miibus_statchg(device_t dev)
 613 {
 614         struct rl_softc         *sc;
 615         struct ifnet            *ifp;
 616         struct mii_data         *mii;
 617
 618         sc = device_get_softc(dev);
 619         mii = device_get_softc(sc->rl_miibus);
 620         ifp = sc->rl_ifp;
 621         if (mii == NULL || ifp == NULL ||
 622             (ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 623                 return;
 624
 625         sc->rl_flags &= ~RL_FLAG_LINK;
 626         if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) ==
 627             (IFM_ACTIVE | IFM_AVALID)) {
 628                 switch (IFM_SUBTYPE(mii->mii_media_active)) {
 629                 case IFM_10_T:
 630                 case IFM_100_TX:
 631                         sc->rl_flags |= RL_FLAG_LINK;
 632                         break;
 633                 case IFM_1000_T:
 634                         if ((sc->rl_flags & RL_FLAG_FASTETHER) != 0)
 635                                 break;
 636                         sc->rl_flags |= RL_FLAG_LINK;
 637                         break;
 638                 default:
 639                         break;
 640                 }
 641         }
 642         /*
 643          * RealTek controllers do not provide any interface to the RX/TX
 644          * MACs for resolved speed, duplex and flow-control parameters.
 645          */
 646 }
 647
 648 /*
 649  * Set the RX configuration and 64-bit multicast hash filter.
 650  */
 651 static void
 652 re_set_rxmode(struct rl_softc *sc)
 653 {
 654         struct ifnet            *ifp;
 655         struct ifmultiaddr      *ifma;
 656         uint32_t                hashes[2] = { 0, 0 };
 657         uint32_t                h, rxfilt;
 658
 659         RL_LOCK_ASSERT(sc);
 660
 661         ifp = sc->rl_ifp;
 662
 663         rxfilt = RL_RXCFG_CONFIG | RL_RXCFG_RX_INDIV | RL_RXCFG_RX_BROAD;
 664         if ((sc->rl_flags & RL_FLAG_EARLYOFF) != 0)
 665                 rxfilt |= RL_RXCFG_EARLYOFF;
 666         else if ((sc->rl_flags & RL_FLAG_8168G_PLUS) != 0)
 667                 rxfilt |= RL_RXCFG_EARLYOFFV2;
 668
 669         if (ifp->if_flags & (IFF_ALLMULTI | IFF_PROMISC)) {
 670                 if (ifp->if_flags & IFF_PROMISC)
 671                         rxfilt |= RL_RXCFG_RX_ALLPHYS;
 672                 /*
 673                  * Unlike other hardwares, we have to explicitly set
 674                  * RL_RXCFG_RX_MULTI to receive multicast frames in
 675                  * promiscuous mode.
 676                  */
 677                 rxfilt |= RL_RXCFG_RX_MULTI;
 678                 hashes[0] = hashes[1] = 0xffffffff;
 679                 goto done;
 680         }
 681
 682         if_maddr_rlock(ifp);
 683         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
 684                 if (ifma->ifma_addr->sa_family != AF_LINK)
 685                         continue;
 686                 h = ether_crc32_be(LLADDR((struct sockaddr_dl *)
 687                     ifma->ifma_addr), ETHER_ADDR_LEN) >> 26;
 688                 if (h < 32)
 689                         hashes[0] |= (1 << h);
 690                 else
 691                         hashes[1] |= (1 << (h - 32));
 692         }
 693         if_maddr_runlock(ifp);
 694
 695         if (hashes[0] != 0 || hashes[1] != 0) {
 696                 /*
 697                  * For some unfathomable reason, RealTek decided to
 698                  * reverse the order of the multicast hash registers
 699                  * in the PCI Express parts.  This means we have to
 700                  * write the hash pattern in reverse order for those
 701                  * devices.
 702                  */
 703                 if ((sc->rl_flags & RL_FLAG_PCIE) != 0) {
 704                         h = bswap32(hashes[0]);
 705                         hashes[0] = bswap32(hashes[1]);
 706                         hashes[1] = h;
 707                 }
 708                 rxfilt |= RL_RXCFG_RX_MULTI;
 709         }
 710
 711         if  (sc->rl_hwrev->rl_rev == RL_HWREV_8168F) {
 712                 /* Disable multicast filtering due to silicon bug. */
 713                 hashes[0] = 0xffffffff;
 714                 hashes[1] = 0xffffffff;
 715         }
 716
 717 done:
 718         CSR_WRITE_4(sc, RL_MAR0, hashes[0]);
 719         CSR_WRITE_4(sc, RL_MAR4, hashes[1]);
 720         CSR_WRITE_4(sc, RL_RXCFG, rxfilt);
 721 }
 722
 723 static void
 724 re_reset(struct rl_softc *sc)
 725 {
 726         int                     i;
 727
 728         RL_LOCK_ASSERT(sc);
 729
 730         CSR_WRITE_1(sc, RL_COMMAND, RL_CMD_RESET);
 731
 732         for (i = 0; i < RL_TIMEOUT; i++) {
 733                 DELAY(10);
 734                 if (!(CSR_READ_1(sc, RL_COMMAND) & RL_CMD_RESET))
 735                         break;
 736         }
 737         if (i == RL_TIMEOUT)
 738                 device_printf(sc->rl_dev, "reset never completed!\n");
 739
 740         if ((sc->rl_flags & RL_FLAG_MACRESET) != 0)
 741                 CSR_WRITE_1(sc, 0x82, 1);
 742         if (sc->rl_hwrev->rl_rev == RL_HWREV_8169S)
 743                 re_gmii_writereg(sc->rl_dev, 1, 0x0b, 0);
 744 }
 745
 746 #ifdef RE_DIAG
 747
 748 /*
 749  * The following routine is designed to test for a defect on some
 750  * 32-bit 8169 cards. Some of these NICs have the REQ64# and ACK64#
 751  * lines connected to the bus, however for a 32-bit only card, they
 752  * should be pulled high. The result of this defect is that the
 753  * NIC will not work right if you plug it into a 64-bit slot: DMA
 754  * operations will be done with 64-bit transfers, which will fail
 755  * because the 64-bit data lines aren't connected.
 756  *
 757  * There's no way to work around this (short of talking a soldering
 758  * iron to the board), however we can detect it. The method we use
 759  * here is to put the NIC into digital loopback mode, set the receiver
 760  * to promiscuous mode, and then try to send a frame. We then compare
 761  * the frame data we sent to what was received. If the data matches,
 762  * then the NIC is working correctly, otherwise we know the user has
 763  * a defective NIC which has been mistakenly plugged into a 64-bit PCI
 764  * slot. In the latter case, there's no way the NIC can work correctly,
 765  * so we print out a message on the console and abort the device attach.
 766  */
 767
 768 static int
 769 re_diag(struct rl_softc *sc)
 770 {
 771         struct ifnet            *ifp = sc->rl_ifp;
 772         struct mbuf             *m0;
 773         struct ether_header     *eh;
 774         struct rl_desc          *cur_rx;
 775         u_int16_t               status;
 776         u_int32_t               rxstat;
 777         int                     total_len, i, error = 0, phyaddr;
 778         u_int8_t                dst[] = { 0x00, 'h', 'e', 'l', 'l', 'o' };
 779         u_int8_t                src[] = { 0x00, 'w', 'o', 'r', 'l', 'd' };
 780
 781         /* Allocate a single mbuf */
 782         MGETHDR(m0, M_NOWAIT, MT_DATA);
 783         if (m0 == NULL)
 784                 return (ENOBUFS);
 785
 786         RL_LOCK(sc);
 787
 788         /*
 789          * Initialize the NIC in test mode. This sets the chip up
 790          * so that it can send and receive frames, but performs the
 791          * following special functions:
 792          * - Puts receiver in promiscuous mode
 793          * - Enables digital loopback mode
 794          * - Leaves interrupts turned off
 795          */
 796
 797         ifp->if_flags |= IFF_PROMISC;
 798         sc->rl_testmode = 1;
 799         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 800         re_init_locked(sc);
 801         sc->rl_flags |= RL_FLAG_LINK;
 802         if (sc->rl_type == RL_8169)
 803                 phyaddr = 1;
 804         else
 805                 phyaddr = 0;
 806
 807         re_miibus_writereg(sc->rl_dev, phyaddr, MII_BMCR, BMCR_RESET);
 808         for (i = 0; i < RL_TIMEOUT; i++) {
 809                 status = re_miibus_readreg(sc->rl_dev, phyaddr, MII_BMCR);
 810                 if (!(status & BMCR_RESET))
 811                         break;
 812         }
 813
 814         re_miibus_writereg(sc->rl_dev, phyaddr, MII_BMCR, BMCR_LOOP);
 815         CSR_WRITE_2(sc, RL_ISR, RL_INTRS);
 816
 817         DELAY(100000);
 818
 819         /* Put some data in the mbuf */
 820
 821         eh = mtod(m0, struct ether_header *);
 822         bcopy ((char *)&dst, eh->ether_dhost, ETHER_ADDR_LEN);
 823         bcopy ((char *)&src, eh->ether_shost, ETHER_ADDR_LEN);
 824         eh->ether_type = htons(ETHERTYPE_IP);
 825         m0->m_pkthdr.len = m0->m_len = ETHER_MIN_LEN - ETHER_CRC_LEN;
 826
 827         /*
 828          * Queue the packet, start transmission.
 829          * Note: IF_HANDOFF() ultimately calls re_start() for us.
 830          */
 831
 832         CSR_WRITE_2(sc, RL_ISR, 0xFFFF);
 833         RL_UNLOCK(sc);
 834         /* XXX: re_diag must not be called when in ALTQ mode */
 835         IF_HANDOFF(&ifp->if_snd, m0, ifp);
 836         RL_LOCK(sc);
 837         m0 = NULL;
 838
 839         /* Wait for it to propagate through the chip */
 840
 841         DELAY(100000);
 842         for (i = 0; i < RL_TIMEOUT; i++) {
 843                 status = CSR_READ_2(sc, RL_ISR);
 844                 CSR_WRITE_2(sc, RL_ISR, status);
 845                 if ((status & (RL_ISR_TIMEOUT_EXPIRED|RL_ISR_RX_OK)) ==
 846                     (RL_ISR_TIMEOUT_EXPIRED|RL_ISR_RX_OK))
 847                         break;
 848                 DELAY(10);
 849         }
 850
 851         if (i == RL_TIMEOUT) {
 852                 device_printf(sc->rl_dev,
 853                     "diagnostic failed, failed to receive packet in"
 854                     " loopback mode\n");
 855                 error = EIO;
 856                 goto done;
 857         }
 858
 859         /*
 860          * The packet should have been dumped into the first
 861          * entry in the RX DMA ring. Grab it from there.
 862          */
 863
 864         bus_dmamap_sync(sc->rl_ldata.rl_rx_list_tag,
 865             sc->rl_ldata.rl_rx_list_map,
 866             BUS_DMASYNC_POSTREAD);
 867         bus_dmamap_sync(sc->rl_ldata.rl_rx_mtag,
 868             sc->rl_ldata.rl_rx_desc[0].rx_dmamap,
 869             BUS_DMASYNC_POSTREAD);
 870         bus_dmamap_unload(sc->rl_ldata.rl_rx_mtag,
 871             sc->rl_ldata.rl_rx_desc[0].rx_dmamap);
 872
 873         m0 = sc->rl_ldata.rl_rx_desc[0].rx_m;
 874         sc->rl_ldata.rl_rx_desc[0].rx_m = NULL;
 875         eh = mtod(m0, struct ether_header *);
 876
 877         cur_rx = &sc->rl_ldata.rl_rx_list[0];
 878         total_len = RL_RXBYTES(cur_rx);
 879         rxstat = le32toh(cur_rx->rl_cmdstat);
 880
 881         if (total_len != ETHER_MIN_LEN) {
 882                 device_printf(sc->rl_dev,
 883                     "diagnostic failed, received short packet\n");
 884                 error = EIO;
 885                 goto done;
 886         }
 887
 888         /* Test that the received packet data matches what we sent. */
 889
 890         if (bcmp((char *)&eh->ether_dhost, (char *)&dst, ETHER_ADDR_LEN) ||
 891             bcmp((char *)&eh->ether_shost, (char *)&src, ETHER_ADDR_LEN) ||
 892             ntohs(eh->ether_type) != ETHERTYPE_IP) {
 893                 device_printf(sc->rl_dev, "WARNING, DMA FAILURE!\n");
 894                 device_printf(sc->rl_dev, "expected TX data: %6D/%6D/0x%x\n",
 895                     dst, ":", src, ":", ETHERTYPE_IP);
 896                 device_printf(sc->rl_dev, "received RX data: %6D/%6D/0x%x\n",
 897                     eh->ether_dhost, ":", eh->ether_shost, ":",
 898                     ntohs(eh->ether_type));
 899                 device_printf(sc->rl_dev, "You may have a defective 32-bit "
 900                     "NIC plugged into a 64-bit PCI slot.\n");
 901                 device_printf(sc->rl_dev, "Please re-install the NIC in a "
 902                     "32-bit slot for proper operation.\n");
 903                 device_printf(sc->rl_dev, "Read the re(4) man page for more "
 904                     "details.\n");
 905                 error = EIO;
 906         }
 907
 908 done:
 909         /* Turn interface off, release resources */
 910
 911         sc->rl_testmode = 0;
 912         sc->rl_flags &= ~RL_FLAG_LINK;
 913         ifp->if_flags &= ~IFF_PROMISC;
 914         re_stop(sc);
 915         if (m0 != NULL)
 916                 m_freem(m0);
 917
 918         RL_UNLOCK(sc);
 919
 920         return (error);
 921 }
 922
 923 #endif
 924
 925 /*
 926  * Probe for a RealTek 8139C+/8169/8110 chip. Check the PCI vendor and device
 927  * IDs against our list and return a device name if we find a match.
 928  */
 929 static int
 930 re_probe(device_t dev)
 931 {
 932         const struct rl_type    *t;
 933         uint16_t                devid, vendor;
 934         uint16_t                revid, sdevid;
 935         int                     i;
 936
 937         vendor = pci_get_vendor(dev);
 938         devid = pci_get_device(dev);
 939         revid = pci_get_revid(dev);
 940         sdevid = pci_get_subdevice(dev);
 941
 942         if (vendor == LINKSYS_VENDORID && devid == LINKSYS_DEVICEID_EG1032) {
 943                 if (sdevid != LINKSYS_SUBDEVICE_EG1032_REV3) {
 944                         /*
 945                          * Only attach to rev. 3 of the Linksys EG1032 adapter.
 946                          * Rev. 2 is supported by sk(4).
 947                          */
 948                         return (ENXIO);
 949                 }
 950         }
 951
 952         if (vendor == RT_VENDORID && devid == RT_DEVICEID_8139) {
 953                 if (revid != 0x20) {
 954                         /* 8139, let rl(4) take care of this device. */
 955                         return (ENXIO);
 956                 }
 957         }
 958
 959         t = re_devs;
 960         for (i = 0; i < nitems(re_devs); i++, t++) {
 961                 if (vendor == t->rl_vid && devid == t->rl_did) {
 962                         device_set_desc(dev, t->rl_name);
 963                         return (BUS_PROBE_DEFAULT);
 964                 }
 965         }
 966
 967         return (ENXIO);
 968 }
 969
 970 /*
 971  * Map a single buffer address.
 972  */
 973
 974 static void
 975 re_dma_map_addr(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 976 {
 977         bus_addr_t              *addr;
 978
 979         if (error)
 980                 return;
 981
 982         KASSERT(nseg == 1, ("too many DMA segments, %d should be 1", nseg));
 983         addr = arg;
 984         *addr = segs->ds_addr;
 985 }
 986
 987 static int
 988 re_allocmem(device_t dev, struct rl_softc *sc)
 989 {
 990         bus_addr_t              lowaddr;
 991         bus_size_t              rx_list_size, tx_list_size;
 992         int                     error;
 993         int                     i;
 994
 995         rx_list_size = sc->rl_ldata.rl_rx_desc_cnt * sizeof(struct rl_desc);
 996         tx_list_size = sc->rl_ldata.rl_tx_desc_cnt * sizeof(struct rl_desc);
 997
 998         /*
 999          * Allocate the parent bus DMA tag appropriate for PCI.
1000          * In order to use DAC, RL_CPLUSCMD_PCI_DAC bit of RL_CPLUS_CMD
1001          * register should be set. However some RealTek chips are known
1002          * to be buggy on DAC handling, therefore disable DAC by limiting
1003          * DMA address space to 32bit. PCIe variants of RealTek chips
1004          * may not have the limitation.
1005          */
1006         lowaddr = BUS_SPACE_MAXADDR;
1007         if ((sc->rl_flags & RL_FLAG_PCIE) == 0)
1008                 lowaddr = BUS_SPACE_MAXADDR_32BIT;
1009         error = bus_dma_tag_create(bus_get_dma_tag(dev), 1, 0,
1010             lowaddr, BUS_SPACE_MAXADDR, NULL, NULL,
1011             BUS_SPACE_MAXSIZE_32BIT, 0, BUS_SPACE_MAXSIZE_32BIT, 0,
1012             NULL, NULL, &sc->rl_parent_tag);
1013         if (error) {
1014                 device_printf(dev, "could not allocate parent DMA tag\n");
1015                 return (error);
1016         }
1017
1018         /*
1019          * Allocate map for TX mbufs.
1020          */
1021         error = bus_dma_tag_create(sc->rl_parent_tag, 1, 0,
1022             BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL,
1023             NULL, MCLBYTES * RL_NTXSEGS, RL_NTXSEGS, 4096, 0,
1024             NULL, NULL, &sc->rl_ldata.rl_tx_mtag);
1025         if (error) {
1026                 device_printf(dev, "could not allocate TX DMA tag\n");
1027                 return (error);
1028         }
1029
1030         /*
1031          * Allocate map for RX mbufs.
1032          */
1033
1034         if ((sc->rl_flags & RL_FLAG_JUMBOV2) != 0) {
1035                 error = bus_dma_tag_create(sc->rl_parent_tag, sizeof(uint64_t),
1036                     0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
1037                     MJUM9BYTES, 1, MJUM9BYTES, 0, NULL, NULL,
1038                     &sc->rl_ldata.rl_jrx_mtag);
1039                 if (error) {
1040                         device_printf(dev,
1041                             "could not allocate jumbo RX DMA tag\n");
1042                         return (error);
1043                 }
1044         }
1045         error = bus_dma_tag_create(sc->rl_parent_tag, sizeof(uint64_t), 0,
1046             BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
1047             MCLBYTES, 1, MCLBYTES, 0, NULL, NULL, &sc->rl_ldata.rl_rx_mtag);
1048         if (error) {
1049                 device_printf(dev, "could not allocate RX DMA tag\n");
1050                 return (error);
1051         }
1052
1053         /*
1054          * Allocate map for TX descriptor list.
1055          */
1056         error = bus_dma_tag_create(sc->rl_parent_tag, RL_RING_ALIGN,
1057             0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL,
1058             NULL, tx_list_size, 1, tx_list_size, 0,
1059             NULL, NULL, &sc->rl_ldata.rl_tx_list_tag);
1060         if (error) {
1061                 device_printf(dev, "could not allocate TX DMA ring tag\n");
1062                 return (error);
1063         }
1064
1065         /* Allocate DMA'able memory for the TX ring */
1066
1067         error = bus_dmamem_alloc(sc->rl_ldata.rl_tx_list_tag,
1068             (void **)&sc->rl_ldata.rl_tx_list,
1069             BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO,
1070             &sc->rl_ldata.rl_tx_list_map);
1071         if (error) {
1072                 device_printf(dev, "could not allocate TX DMA ring\n");
1073                 return (error);
1074         }
1075
1076         /* Load the map for the TX ring. */
1077
1078         sc->rl_ldata.rl_tx_list_addr = 0;
1079         error = bus_dmamap_load(sc->rl_ldata.rl_tx_list_tag,
1080              sc->rl_ldata.rl_tx_list_map, sc->rl_ldata.rl_tx_list,
1081              tx_list_size, re_dma_map_addr,
1082              &sc->rl_ldata.rl_tx_list_addr, BUS_DMA_NOWAIT);
1083         if (error != 0 || sc->rl_ldata.rl_tx_list_addr == 0) {
1084                 device_printf(dev, "could not load TX DMA ring\n");
1085                 return (ENOMEM);
1086         }
1087
1088         /* Create DMA maps for TX buffers */
1089
1090         for (i = 0; i < sc->rl_ldata.rl_tx_desc_cnt; i++) {
1091                 error = bus_dmamap_create(sc->rl_ldata.rl_tx_mtag, 0,
1092                     &sc->rl_ldata.rl_tx_desc[i].tx_dmamap);
1093                 if (error) {
1094                         device_printf(dev, "could not create DMA map for TX\n");
1095                         return (error);
1096                 }
1097         }
1098
1099         /*
1100          * Allocate map for RX descriptor list.
1101          */
1102         error = bus_dma_tag_create(sc->rl_parent_tag, RL_RING_ALIGN,
1103             0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL,
1104             NULL, rx_list_size, 1, rx_list_size, 0,
1105             NULL, NULL, &sc->rl_ldata.rl_rx_list_tag);
1106         if (error) {
1107                 device_printf(dev, "could not create RX DMA ring tag\n");
1108                 return (error);
1109         }
1110
1111         /* Allocate DMA'able memory for the RX ring */
1112
1113         error = bus_dmamem_alloc(sc->rl_ldata.rl_rx_list_tag,
1114             (void **)&sc->rl_ldata.rl_rx_list,
1115             BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO,
1116             &sc->rl_ldata.rl_rx_list_map);
1117         if (error) {
1118                 device_printf(dev, "could not allocate RX DMA ring\n");
1119                 return (error);
1120         }
1121
1122         /* Load the map for the RX ring. */
1123
1124         sc->rl_ldata.rl_rx_list_addr = 0;
1125         error = bus_dmamap_load(sc->rl_ldata.rl_rx_list_tag,
1126              sc->rl_ldata.rl_rx_list_map, sc->rl_ldata.rl_rx_list,
1127              rx_list_size, re_dma_map_addr,
1128              &sc->rl_ldata.rl_rx_list_addr, BUS_DMA_NOWAIT);
1129         if (error != 0 || sc->rl_ldata.rl_rx_list_addr == 0) {
1130                 device_printf(dev, "could not load RX DMA ring\n");
1131                 return (ENOMEM);
1132         }
1133
1134         /* Create DMA maps for RX buffers */
1135
1136         if ((sc->rl_flags & RL_FLAG_JUMBOV2) != 0) {
1137                 error = bus_dmamap_create(sc->rl_ldata.rl_jrx_mtag, 0,
1138                     &sc->rl_ldata.rl_jrx_sparemap);
1139                 if (error) {
1140                         device_printf(dev,
1141                             "could not create spare DMA map for jumbo RX\n");
1142                         return (error);
1143                 }
1144                 for (i = 0; i < sc->rl_ldata.rl_rx_desc_cnt; i++) {
1145                         error = bus_dmamap_create(sc->rl_ldata.rl_jrx_mtag, 0,
1146                             &sc->rl_ldata.rl_jrx_desc[i].rx_dmamap);
1147                         if (error) {
1148                                 device_printf(dev,
1149                                     "could not create DMA map for jumbo RX\n");
1150                                 return (error);
1151                         }
1152                 }
1153         }
1154         error = bus_dmamap_create(sc->rl_ldata.rl_rx_mtag, 0,
1155             &sc->rl_ldata.rl_rx_sparemap);
1156         if (error) {
1157                 device_printf(dev, "could not create spare DMA map for RX\n");
1158                 return (error);
1159         }
1160         for (i = 0; i < sc->rl_ldata.rl_rx_desc_cnt; i++) {
1161                 error = bus_dmamap_create(sc->rl_ldata.rl_rx_mtag, 0,
1162                     &sc->rl_ldata.rl_rx_desc[i].rx_dmamap);
1163                 if (error) {
1164                         device_printf(dev, "could not create DMA map for RX\n");
1165                         return (error);
1166                 }
1167         }
1168
1169         /* Create DMA map for statistics. */
1170         error = bus_dma_tag_create(sc->rl_parent_tag, RL_DUMP_ALIGN, 0,
1171             BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
1172             sizeof(struct rl_stats), 1, sizeof(struct rl_stats), 0, NULL, NULL,
1173             &sc->rl_ldata.rl_stag);
1174         if (error) {
1175                 device_printf(dev, "could not create statistics DMA tag\n");
1176                 return (error);
1177         }
1178         /* Allocate DMA'able memory for statistics. */
1179         error = bus_dmamem_alloc(sc->rl_ldata.rl_stag,
1180             (void **)&sc->rl_ldata.rl_stats,
1181             BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO,
1182             &sc->rl_ldata.rl_smap);
1183         if (error) {
1184                 device_printf(dev,
1185                     "could not allocate statistics DMA memory\n");
1186                 return (error);
1187         }
1188         /* Load the map for statistics. */
1189         sc->rl_ldata.rl_stats_addr = 0;
1190         error = bus_dmamap_load(sc->rl_ldata.rl_stag, sc->rl_ldata.rl_smap,
1191             sc->rl_ldata.rl_stats, sizeof(struct rl_stats), re_dma_map_addr,
1192              &sc->rl_ldata.rl_stats_addr, BUS_DMA_NOWAIT);
1193         if (error != 0 || sc->rl_ldata.rl_stats_addr == 0) {
1194                 device_printf(dev, "could not load statistics DMA memory\n");
1195                 return (ENOMEM);
1196         }
1197
1198         return (0);
1199 }
1200
1201 /*
1202  * Attach the interface. Allocate softc structures, do ifmedia
1203  * setup and ethernet/BPF attach.
1204  */
1205 static int
1206 re_attach(device_t dev)
1207 {
1208         u_char                  eaddr[ETHER_ADDR_LEN];
1209         u_int16_t               as[ETHER_ADDR_LEN / 2];
1210         struct rl_softc         *sc;
1211         struct ifnet            *ifp;
1212         const struct rl_hwrev   *hw_rev;
1213         int                     capmask, error = 0, hwrev, i, msic, msixc,
1214                                 phy, reg, rid;
1215         u_int32_t               cap, ctl;
1216         u_int16_t               devid, re_did = 0;
1217         uint8_t                 cfg;
1218
1219         sc = device_get_softc(dev);
1220         sc->rl_dev = dev;
1221
1222         mtx_init(&sc->rl_mtx, device_get_nameunit(dev), MTX_NETWORK_LOCK,
1223             MTX_DEF);
1224         callout_init_mtx(&sc->rl_stat_callout, &sc->rl_mtx, 0);
1225
1226         /*
1227          * Map control/status registers.
1228          */
1229         pci_enable_busmaster(dev);
1230
1231         devid = pci_get_device(dev);
1232         /*
1233          * Prefer memory space register mapping over IO space.
1234          * Because RTL8169SC does not seem to work when memory mapping
1235          * is used always activate io mapping.
1236          */
1237         if (devid == RT_DEVICEID_8169SC)
1238                 prefer_iomap = 1;
1239         if (prefer_iomap == 0) {
1240                 sc->rl_res_id = PCIR_BAR(1);
1241                 sc->rl_res_type = SYS_RES_MEMORY;
1242                 /* RTL8168/8101E seems to use different BARs. */
1243                 if (devid == RT_DEVICEID_8168 || devid == RT_DEVICEID_8101E)
1244                         sc->rl_res_id = PCIR_BAR(2);
1245         } else {
1246                 sc->rl_res_id = PCIR_BAR(0);
1247                 sc->rl_res_type = SYS_RES_IOPORT;
1248         }
1249         sc->rl_res = bus_alloc_resource_any(dev, sc->rl_res_type,
1250             &sc->rl_res_id, RF_ACTIVE);
1251         if (sc->rl_res == NULL && prefer_iomap == 0) {
1252                 sc->rl_res_id = PCIR_BAR(0);
1253                 sc->rl_res_type = SYS_RES_IOPORT;
1254                 sc->rl_res = bus_alloc_resource_any(dev, sc->rl_res_type,
1255                     &sc->rl_res_id, RF_ACTIVE);
1256         }
1257         if (sc->rl_res == NULL) {
1258                 device_printf(dev, "couldn't map ports/memory\n");
1259                 error = ENXIO;
1260                 goto fail;
1261         }
1262
1263         sc->rl_btag = rman_get_bustag(sc->rl_res);
1264         sc->rl_bhandle = rman_get_bushandle(sc->rl_res);
1265
1266         msic = pci_msi_count(dev);
1267         msixc = pci_msix_count(dev);
1268         if (pci_find_cap(dev, PCIY_EXPRESS, &reg) == 0) {
1269                 sc->rl_flags |= RL_FLAG_PCIE;
1270                 sc->rl_expcap = reg;
1271         }
1272         if (bootverbose) {
1273                 device_printf(dev, "MSI count : %d\n", msic);
1274                 device_printf(dev, "MSI-X count : %d\n", msixc);
1275         }
1276         if (msix_disable > 0)
1277                 msixc = 0;
1278         if (msi_disable > 0)
1279                 msic = 0;
1280         /* Prefer MSI-X to MSI. */
1281         if (msixc > 0) {
1282                 msixc = RL_MSI_MESSAGES;
1283                 rid = PCIR_BAR(4);
1284                 sc->rl_res_pba = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
1285                     &rid, RF_ACTIVE);
1286                 if (sc->rl_res_pba == NULL) {
1287                         device_printf(sc->rl_dev,
1288                             "could not allocate MSI-X PBA resource\n");
1289                 }
1290                 if (sc->rl_res_pba != NULL &&
1291                     pci_alloc_msix(dev, &msixc) == 0) {
1292                         if (msixc == RL_MSI_MESSAGES) {
1293                                 device_printf(dev, "Using %d MSI-X message\n",
1294                                     msixc);
1295                                 sc->rl_flags |= RL_FLAG_MSIX;
1296                         } else
1297                                 pci_release_msi(dev);
1298                 }
1299                 if ((sc->rl_flags & RL_FLAG_MSIX) == 0) {
1300                         if (sc->rl_res_pba != NULL)
1301                                 bus_release_resource(dev, SYS_RES_MEMORY, rid,
1302                                     sc->rl_res_pba);
1303                         sc->rl_res_pba = NULL;
1304                         msixc = 0;
1305                 }
1306         }
1307         /* Prefer MSI to INTx. */
1308         if (msixc == 0 && msic > 0) {
1309                 msic = RL_MSI_MESSAGES;
1310                 if (pci_alloc_msi(dev, &msic) == 0) {
1311                         if (msic == RL_MSI_MESSAGES) {
1312                                 device_printf(dev, "Using %d MSI message\n",
1313                                     msic);
1314                                 sc->rl_flags |= RL_FLAG_MSI;
1315                                 /* Explicitly set MSI enable bit. */
1316                                 CSR_WRITE_1(sc, RL_EECMD, RL_EE_MODE);
1317                                 cfg = CSR_READ_1(sc, RL_CFG2);
1318                                 cfg |= RL_CFG2_MSI;
1319                                 CSR_WRITE_1(sc, RL_CFG2, cfg);
1320                                 CSR_WRITE_1(sc, RL_EECMD, RL_EEMODE_OFF);
1321                         } else
1322                                 pci_release_msi(dev);
1323                 }
1324                 if ((sc->rl_flags & RL_FLAG_MSI) == 0)
1325                         msic = 0;
1326         }
1327
1328         /* Allocate interrupt */
1329         if ((sc->rl_flags & (RL_FLAG_MSI | RL_FLAG_MSIX)) == 0) {
1330                 rid = 0;
1331                 sc->rl_irq[0] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
1332                     RF_SHAREABLE | RF_ACTIVE);
1333                 if (sc->rl_irq[0] == NULL) {
1334                         device_printf(dev, "couldn't allocate IRQ resources\n");
1335                         error = ENXIO;
1336                         goto fail;
1337                 }
1338         } else {
1339                 for (i = 0, rid = 1; i < RL_MSI_MESSAGES; i++, rid++) {
1340                         sc->rl_irq[i] = bus_alloc_resource_any(dev,
1341                             SYS_RES_IRQ, &rid, RF_ACTIVE);
1342                         if (sc->rl_irq[i] == NULL) {
1343                                 device_printf(dev,
1344                                     "couldn't allocate IRQ resources for "
1345                                     "message %d\n", rid);
1346                                 error = ENXIO;
1347                                 goto fail;
1348                         }
1349                 }
1350         }
1351
1352         if ((sc->rl_flags & RL_FLAG_MSI) == 0) {
1353                 CSR_WRITE_1(sc, RL_EECMD, RL_EE_MODE);
1354                 cfg = CSR_READ_1(sc, RL_CFG2);
1355                 if ((cfg & RL_CFG2_MSI) != 0) {
1356                         device_printf(dev, "turning off MSI enable bit.\n");
1357                         cfg &= ~RL_CFG2_MSI;
1358                         CSR_WRITE_1(sc, RL_CFG2, cfg);
1359                 }
1360                 CSR_WRITE_1(sc, RL_EECMD, RL_EEMODE_OFF);
1361         }
1362
1363         /* Disable ASPM L0S/L1 and CLKREQ. */
1364         if (sc->rl_expcap != 0) {
1365                 cap = pci_read_config(dev, sc->rl_expcap +
1366                     PCIER_LINK_CAP, 2);
1367                 if ((cap & PCIEM_LINK_CAP_ASPM) != 0) {
1368                         ctl = pci_read_config(dev, sc->rl_expcap +
1369                             PCIER_LINK_CTL, 2);
1370                         if ((ctl & (PCIEM_LINK_CTL_ECPM |
1371                             PCIEM_LINK_CTL_ASPMC))!= 0) {
1372                                 ctl &= ~(PCIEM_LINK_CTL_ECPM |
1373                                     PCIEM_LINK_CTL_ASPMC);
1374                                 pci_write_config(dev, sc->rl_expcap +
1375                                     PCIER_LINK_CTL, ctl, 2);
1376                                 device_printf(dev, "ASPM disabled\n");
1377                         }
1378                 } else
1379                         device_printf(dev, "no ASPM capability\n");
1380         }
1381
1382         hw_rev = re_hwrevs;
1383         hwrev = CSR_READ_4(sc, RL_TXCFG);
1384         switch (hwrev & 0x70000000) {
1385         case 0x00000000:
1386         case 0x10000000:
1387                 device_printf(dev, "Chip rev. 0x%08x\n", hwrev & 0xfc800000);
1388                 hwrev &= (RL_TXCFG_HWREV | 0x80000000);
1389                 break;
1390         default:
1391                 device_printf(dev, "Chip rev. 0x%08x\n", hwrev & 0x7c800000);
1392                 sc->rl_macrev = hwrev & 0x00700000;
1393                 hwrev &= RL_TXCFG_HWREV;
1394                 break;
1395         }
1396         device_printf(dev, "MAC rev. 0x%08x\n", sc->rl_macrev);
1397         while (hw_rev->rl_desc != NULL) {
1398                 if (hw_rev->rl_rev == hwrev) {
1399                         sc->rl_type = hw_rev->rl_type;
1400                         sc->rl_hwrev = hw_rev;
1401                         break;
1402                 }
1403                 hw_rev++;
1404         }
1405         if (hw_rev->rl_desc == NULL) {
1406                 device_printf(dev, "Unknown H/W revision: 0x%08x\n", hwrev);
1407                 error = ENXIO;
1408                 goto fail;
1409         }
1410
1411         switch (hw_rev->rl_rev) {
1412         case RL_HWREV_8139CPLUS:
1413                 sc->rl_flags |= RL_FLAG_FASTETHER | RL_FLAG_AUTOPAD;
1414                 break;
1415         case RL_HWREV_8100E:
1416         case RL_HWREV_8101E:
1417                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_FASTETHER;
1418                 break;
1419         case RL_HWREV_8102E:
1420         case RL_HWREV_8102EL:
1421         case RL_HWREV_8102EL_SPIN1:
1422                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PAR | RL_FLAG_DESCV2 |
1423                     RL_FLAG_MACSTAT | RL_FLAG_FASTETHER | RL_FLAG_CMDSTOP |
1424                     RL_FLAG_AUTOPAD;
1425                 break;
1426         case RL_HWREV_8103E:
1427                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PAR | RL_FLAG_DESCV2 |
1428                     RL_FLAG_MACSTAT | RL_FLAG_FASTETHER | RL_FLAG_CMDSTOP |
1429                     RL_FLAG_AUTOPAD | RL_FLAG_MACSLEEP;
1430                 break;
1431         case RL_HWREV_8401E:
1432         case RL_HWREV_8105E:
1433         case RL_HWREV_8105E_SPIN1:
1434         case RL_HWREV_8106E:
1435                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PHYWAKE_PM |
1436                     RL_FLAG_PAR | RL_FLAG_DESCV2 | RL_FLAG_MACSTAT |
1437                     RL_FLAG_FASTETHER | RL_FLAG_CMDSTOP | RL_FLAG_AUTOPAD;
1438                 break;
1439         case RL_HWREV_8402:
1440                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PHYWAKE_PM |
1441                     RL_FLAG_PAR | RL_FLAG_DESCV2 | RL_FLAG_MACSTAT |
1442                     RL_FLAG_FASTETHER | RL_FLAG_CMDSTOP | RL_FLAG_AUTOPAD |
1443                     RL_FLAG_CMDSTOP_WAIT_TXQ;
1444                 break;
1445         case RL_HWREV_8168B_SPIN1:
1446         case RL_HWREV_8168B_SPIN2:
1447                 sc->rl_flags |= RL_FLAG_WOLRXENB;
1448                 /* FALLTHROUGH */
1449         case RL_HWREV_8168B_SPIN3:
1450                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_MACSTAT;
1451                 break;
1452         case RL_HWREV_8168C_SPIN2:
1453                 sc->rl_flags |= RL_FLAG_MACSLEEP;
1454                 /* FALLTHROUGH */
1455         case RL_HWREV_8168C:
1456                 if (sc->rl_macrev == 0x00200000)
1457                         sc->rl_flags |= RL_FLAG_MACSLEEP;
1458                 /* FALLTHROUGH */
1459         case RL_HWREV_8168CP:
1460                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PAR |
1461                     RL_FLAG_DESCV2 | RL_FLAG_MACSTAT | RL_FLAG_CMDSTOP |
1462                     RL_FLAG_AUTOPAD | RL_FLAG_JUMBOV2 | RL_FLAG_WOL_MANLINK;
1463                 break;
1464         case RL_HWREV_8168D:
1465                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PHYWAKE_PM |
1466                     RL_FLAG_PAR | RL_FLAG_DESCV2 | RL_FLAG_MACSTAT |
1467                     RL_FLAG_CMDSTOP | RL_FLAG_AUTOPAD | RL_FLAG_JUMBOV2 |
1468                     RL_FLAG_WOL_MANLINK;
1469                 break;
1470         case RL_HWREV_8168DP:
1471                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PAR |
1472                     RL_FLAG_DESCV2 | RL_FLAG_MACSTAT | RL_FLAG_AUTOPAD |
1473                     RL_FLAG_JUMBOV2 | RL_FLAG_WAIT_TXPOLL | RL_FLAG_WOL_MANLINK;
1474                 break;
1475         case RL_HWREV_8168E:
1476                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PHYWAKE_PM |
1477                     RL_FLAG_PAR | RL_FLAG_DESCV2 | RL_FLAG_MACSTAT |
1478                     RL_FLAG_CMDSTOP | RL_FLAG_AUTOPAD | RL_FLAG_JUMBOV2 |
1479                     RL_FLAG_WOL_MANLINK;
1480                 break;
1481         case RL_HWREV_8168E_VL:
1482         case RL_HWREV_8168F:
1483                 sc->rl_flags |= RL_FLAG_EARLYOFF;
1484                 /* FALLTHROUGH */
1485         case RL_HWREV_8411:
1486                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PAR |
1487                     RL_FLAG_DESCV2 | RL_FLAG_MACSTAT | RL_FLAG_CMDSTOP |
1488                     RL_FLAG_AUTOPAD | RL_FLAG_JUMBOV2 |
1489                     RL_FLAG_CMDSTOP_WAIT_TXQ | RL_FLAG_WOL_MANLINK;
1490                 break;
1491         case RL_HWREV_8168EP:
1492         case RL_HWREV_8168G:
1493         case RL_HWREV_8411B:
1494                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PAR |
1495                     RL_FLAG_DESCV2 | RL_FLAG_MACSTAT | RL_FLAG_CMDSTOP |
1496                     RL_FLAG_AUTOPAD | RL_FLAG_JUMBOV2 |
1497                     RL_FLAG_CMDSTOP_WAIT_TXQ | RL_FLAG_WOL_MANLINK |
1498                     RL_FLAG_8168G_PLUS;
1499                 break;
1500         case RL_HWREV_8168GU:
1501         case RL_HWREV_8168H:
1502                 if (pci_get_device(dev) == RT_DEVICEID_8101E) {
1503                         /* RTL8106E(US), RTL8107E */
1504                         sc->rl_flags |= RL_FLAG_FASTETHER;
1505                 } else
1506                         sc->rl_flags |= RL_FLAG_JUMBOV2 | RL_FLAG_WOL_MANLINK;
1507
1508                 sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PAR |
1509                     RL_FLAG_DESCV2 | RL_FLAG_MACSTAT | RL_FLAG_CMDSTOP |
1510                     RL_FLAG_AUTOPAD | RL_FLAG_CMDSTOP_WAIT_TXQ |
1511                     RL_FLAG_8168G_PLUS;
1512                 break;
1513         case RL_HWREV_8169_8110SB:
1514         case RL_HWREV_8169_8110SBL:
1515         case RL_HWREV_8169_8110SC:
1516         case RL_HWREV_8169_8110SCE:
1517                 sc->rl_flags |= RL_FLAG_PHYWAKE;
1518                 /* FALLTHROUGH */
1519         case RL_HWREV_8169:
1520         case RL_HWREV_8169S:
1521         case RL_HWREV_8110S:
1522                 sc->rl_flags |= RL_FLAG_MACRESET;
1523                 break;
1524         default:
1525                 break;
1526         }
1527
1528         if (sc->rl_hwrev->rl_rev == RL_HWREV_8139CPLUS) {
1529                 sc->rl_cfg0 = RL_8139_CFG0;
1530                 sc->rl_cfg1 = RL_8139_CFG1;
1531                 sc->rl_cfg2 = 0;
1532                 sc->rl_cfg3 = RL_8139_CFG3;
1533                 sc->rl_cfg4 = RL_8139_CFG4;
1534                 sc->rl_cfg5 = RL_8139_CFG5;
1535         } else {
1536                 sc->rl_cfg0 = RL_CFG0;
1537                 sc->rl_cfg1 = RL_CFG1;
1538                 sc->rl_cfg2 = RL_CFG2;
1539                 sc->rl_cfg3 = RL_CFG3;
1540                 sc->rl_cfg4 = RL_CFG4;
1541                 sc->rl_cfg5 = RL_CFG5;
1542         }
1543
1544         /* Reset the adapter. */
1545         RL_LOCK(sc);
1546         re_reset(sc);
1547         RL_UNLOCK(sc);
1548
1549         /* Enable PME. */
1550         CSR_WRITE_1(sc, RL_EECMD, RL_EE_MODE);
1551         cfg = CSR_READ_1(sc, sc->rl_cfg1);
1552         cfg |= RL_CFG1_PME;
1553         CSR_WRITE_1(sc, sc->rl_cfg1, cfg);
1554         cfg = CSR_READ_1(sc, sc->rl_cfg5);
1555         cfg &= RL_CFG5_PME_STS;
1556         CSR_WRITE_1(sc, sc->rl_cfg5, cfg);
1557         CSR_WRITE_1(sc, RL_EECMD, RL_EEMODE_OFF);
1558
1559         if ((sc->rl_flags & RL_FLAG_PAR) != 0) {
1560                 /*
1561                  * XXX Should have a better way to extract station
1562                  * address from EEPROM.
1563                  */
1564                 for (i = 0; i < ETHER_ADDR_LEN; i++)
1565                         eaddr[i] = CSR_READ_1(sc, RL_IDR0 + i);
1566         } else {
1567                 sc->rl_eewidth = RL_9356_ADDR_LEN;
1568                 re_read_eeprom(sc, (caddr_t)&re_did, 0, 1);
1569                 if (re_did != 0x8129)
1570                         sc->rl_eewidth = RL_9346_ADDR_LEN;
1571
1572                 /*
1573                  * Get station address from the EEPROM.
1574                  */
1575                 re_read_eeprom(sc, (caddr_t)as, RL_EE_EADDR, 3);
1576                 for (i = 0; i < ETHER_ADDR_LEN / 2; i++)
1577                         as[i] = le16toh(as[i]);
1578                 bcopy(as, eaddr, ETHER_ADDR_LEN);
1579         }
1580
1581         if (sc->rl_type == RL_8169) {
1582                 /* Set RX length mask and number of descriptors. */
1583                 sc->rl_rxlenmask = RL_RDESC_STAT_GFRAGLEN;
1584                 sc->rl_txstart = RL_GTXSTART;
1585                 sc->rl_ldata.rl_tx_desc_cnt = RL_8169_TX_DESC_CNT;
1586                 sc->rl_ldata.rl_rx_desc_cnt = RL_8169_RX_DESC_CNT;
1587         } else {
1588                 /* Set RX length mask and number of descriptors. */
1589                 sc->rl_rxlenmask = RL_RDESC_STAT_FRAGLEN;
1590                 sc->rl_txstart = RL_TXSTART;
1591                 sc->rl_ldata.rl_tx_desc_cnt = RL_8139_TX_DESC_CNT;
1592                 sc->rl_ldata.rl_rx_desc_cnt = RL_8139_RX_DESC_CNT;
1593         }
1594
1595         error = re_allocmem(dev, sc);
1596         if (error)
1597                 goto fail;
1598         re_add_sysctls(sc);
1599
1600         ifp = sc->rl_ifp = if_alloc(IFT_ETHER);
1601         if (ifp == NULL) {
1602                 device_printf(dev, "can not if_alloc()\n");
1603                 error = ENOSPC;
1604                 goto fail;
1605         }
1606
1607         /* Take controller out of deep sleep mode. */
1608         if ((sc->rl_flags & RL_FLAG_MACSLEEP) != 0) {
1609                 if ((CSR_READ_1(sc, RL_MACDBG) & 0x80) == 0x80)
1610                         CSR_WRITE_1(sc, RL_GPIO,
1611                             CSR_READ_1(sc, RL_GPIO) | 0x01);
1612                 else
1613                         CSR_WRITE_1(sc, RL_GPIO,
1614                             CSR_READ_1(sc, RL_GPIO) & ~0x01);
1615         }
1616
1617         /* Take PHY out of power down mode. */
1618         if ((sc->rl_flags & RL_FLAG_PHYWAKE_PM) != 0) {
1619                 CSR_WRITE_1(sc, RL_PMCH, CSR_READ_1(sc, RL_PMCH) | 0x80);
1620                 if (hw_rev->rl_rev == RL_HWREV_8401E)
1621                         CSR_WRITE_1(sc, 0xD1, CSR_READ_1(sc, 0xD1) & ~0x08);
1622         }
1623         if ((sc->rl_flags & RL_FLAG_PHYWAKE) != 0) {
1624                 re_gmii_writereg(dev, 1, 0x1f, 0);
1625                 re_gmii_writereg(dev, 1, 0x0e, 0);
1626         }
1627
1628         ifp->if_softc = sc;
1629         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1630         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1631         ifp->if_ioctl = re_ioctl;
1632         ifp->if_start = re_start;
1633         /*
1634          * RTL8168/8111C generates wrong IP checksummed frame if the
1635          * packet has IP options so disable TX checksum offloading.
1636          */
1637         if (sc->rl_hwrev->rl_rev == RL_HWREV_8168C ||
1638             sc->rl_hwrev->rl_rev == RL_HWREV_8168C_SPIN2 ||
1639             sc->rl_hwrev->rl_rev == RL_HWREV_8168CP) {
1640                 ifp->if_hwassist = 0;
1641                 ifp->if_capabilities = IFCAP_RXCSUM | IFCAP_TSO4;
1642         } else {
1643                 ifp->if_hwassist = CSUM_IP | CSUM_TCP | CSUM_UDP;
1644                 ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_TSO4;
1645         }
1646         ifp->if_hwassist |= CSUM_TSO;
1647         ifp->if_capenable = ifp->if_capabilities;
1648         ifp->if_init = re_init;
1649         IFQ_SET_MAXLEN(&ifp->if_snd, RL_IFQ_MAXLEN);
1650         ifp->if_snd.ifq_drv_maxlen = RL_IFQ_MAXLEN;
1651         IFQ_SET_READY(&ifp->if_snd);
1652
1653         TASK_INIT(&sc->rl_inttask, 0, re_int_task, sc);
1654
1655 #define RE_PHYAD_INTERNAL        0
1656
1657         /* Do MII setup. */
1658         phy = RE_PHYAD_INTERNAL;
1659         if (sc->rl_type == RL_8169)
1660                 phy = 1;
1661         capmask = BMSR_DEFCAPMASK;
1662         if ((sc->rl_flags & RL_FLAG_FASTETHER) != 0)
1663                  capmask &= ~BMSR_EXTSTAT;
1664         error = mii_attach(dev, &sc->rl_miibus, ifp, re_ifmedia_upd,
1665             re_ifmedia_sts, capmask, phy, MII_OFFSET_ANY, MIIF_DOPAUSE);
1666         if (error != 0) {
1667                 device_printf(dev, "attaching PHYs failed\n");
1668                 goto fail;
1669         }
1670
1671         /*
1672          * Call MI attach routine.
1673          */
1674         ether_ifattach(ifp, eaddr);
1675
1676         /* VLAN capability setup */
1677         ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING;
1678         if (ifp->if_capabilities & IFCAP_HWCSUM)
1679                 ifp->if_capabilities |= IFCAP_VLAN_HWCSUM;
1680         /* Enable WOL if PM is supported. */
1681         if (pci_find_cap(sc->rl_dev, PCIY_PMG, &reg) == 0)
1682                 ifp->if_capabilities |= IFCAP_WOL;
1683         ifp->if_capenable = ifp->if_capabilities;
1684         ifp->if_capenable &= ~(IFCAP_WOL_UCAST | IFCAP_WOL_MCAST);
1685         /*
1686          * Don't enable TSO by default.  It is known to generate
1687          * corrupted TCP segments(bad TCP options) under certain
1688          * circumstances.
1689          */
1690         ifp->if_hwassist &= ~CSUM_TSO;
1691         ifp->if_capenable &= ~(IFCAP_TSO4 | IFCAP_VLAN_HWTSO);
1692 #ifdef DEVICE_POLLING
1693         ifp->if_capabilities |= IFCAP_POLLING;
1694 #endif
1695         /*
1696          * Tell the upper layer(s) we support long frames.
1697          * Must appear after the call to ether_ifattach() because
1698          * ether_ifattach() sets ifi_hdrlen to the default value.
1699          */
1700         ifp->if_hdrlen = sizeof(struct ether_vlan_header);
1701
1702 #ifdef DEV_NETMAP
1703         re_netmap_attach(sc);
1704 #endif /* DEV_NETMAP */
1705
1706 #ifdef RE_DIAG
1707         /*
1708          * Perform hardware diagnostic on the original RTL8169.
1709          * Some 32-bit cards were incorrectly wired and would
1710          * malfunction if plugged into a 64-bit slot.
1711          */
1712         if (hwrev == RL_HWREV_8169) {
1713                 error = re_diag(sc);
1714                 if (error) {
1715                         device_printf(dev,
1716                         "attach aborted due to hardware diag failure\n");
1717                         ether_ifdetach(ifp);
1718                         goto fail;
1719                 }
1720         }
1721 #endif
1722
1723 #ifdef RE_TX_MODERATION
1724         intr_filter = 1;
1725 #endif
1726         /* Hook interrupt last to avoid having to lock softc */
1727         if ((sc->rl_flags & (RL_FLAG_MSI | RL_FLAG_MSIX)) != 0 &&
1728             intr_filter == 0) {
1729                 error = bus_setup_intr(dev, sc->rl_irq[0],
1730                     INTR_TYPE_NET | INTR_MPSAFE, NULL, re_intr_msi, sc,
1731                     &sc->rl_intrhand[0]);
1732         } else {
1733                 error = bus_setup_intr(dev, sc->rl_irq[0],
1734                     INTR_TYPE_NET | INTR_MPSAFE, re_intr, NULL, sc,
1735                     &sc->rl_intrhand[0]);
1736         }
1737         if (error) {
1738                 device_printf(dev, "couldn't set up irq\n");
1739                 ether_ifdetach(ifp);
1740         }
1741
1742 fail:
1743         if (error)
1744                 re_detach(dev);
1745
1746         return (error);
1747 }
1748
1749 /*
1750  * Shutdown hardware and free up resources. This can be called any
1751  * time after the mutex has been initialized. It is called in both
1752  * the error case in attach and the normal detach case so it needs
1753  * to be careful about only freeing resources that have actually been
1754  * allocated.
1755  */
1756 static int
1757 re_detach(device_t dev)
1758 {
1759         struct rl_softc         *sc;
1760         struct ifnet            *ifp;
1761         int                     i, rid;
1762
1763         sc = device_get_softc(dev);
1764         ifp = sc->rl_ifp;
1765         KASSERT(mtx_initialized(&sc->rl_mtx), ("re mutex not initialized"));
1766
1767         /* These should only be active if attach succeeded */
1768         if (device_is_attached(dev)) {
1769 #ifdef DEVICE_POLLING
1770                 if (ifp->if_capenable & IFCAP_POLLING)
1771                         ether_poll_deregister(ifp);
1772 #endif
1773                 RL_LOCK(sc);
1774 #if 0
1775                 sc->suspended = 1;
1776 #endif
1777                 re_stop(sc);
1778                 RL_UNLOCK(sc);
1779                 callout_drain(&sc->rl_stat_callout);
1780                 taskqueue_drain(taskqueue_fast, &sc->rl_inttask);
1781                 /*
1782                  * Force off the IFF_UP flag here, in case someone
1783                  * still had a BPF descriptor attached to this
1784                  * interface. If they do, ether_ifdetach() will cause
1785                  * the BPF code to try and clear the promisc mode
1786                  * flag, which will bubble down to re_ioctl(),
1787                  * which will try to call re_init() again. This will
1788                  * turn the NIC back on and restart the MII ticker,
1789                  * which will panic the system when the kernel tries
1790                  * to invoke the re_tick() function that isn't there
1791                  * anymore.
1792                  */
1793                 ifp->if_flags &= ~IFF_UP;
1794                 ether_ifdetach(ifp);
1795         }
1796         if (sc->rl_miibus)
1797                 device_delete_child(dev, sc->rl_miibus);
1798         bus_generic_detach(dev);
1799
1800         /*
1801          * The rest is resource deallocation, so we should already be
1802          * stopped here.
1803          */
1804
1805         if (sc->rl_intrhand[0] != NULL) {
1806                 bus_teardown_intr(dev, sc->rl_irq[0], sc->rl_intrhand[0]);
1807                 sc->rl_intrhand[0] = NULL;
1808         }
1809         if (ifp != NULL) {
1810 #ifdef DEV_NETMAP
1811                 netmap_detach(ifp);
1812 #endif /* DEV_NETMAP */
1813                 if_free(ifp);
1814         }
1815         if ((sc->rl_flags & (RL_FLAG_MSI | RL_FLAG_MSIX)) == 0)
1816                 rid = 0;
1817         else
1818                 rid = 1;
1819         if (sc->rl_irq[0] != NULL) {
1820                 bus_release_resource(dev, SYS_RES_IRQ, rid, sc->rl_irq[0]);
1821                 sc->rl_irq[0] = NULL;
1822         }
1823         if ((sc->rl_flags & (RL_FLAG_MSI | RL_FLAG_MSIX)) != 0)
1824                 pci_release_msi(dev);
1825         if (sc->rl_res_pba) {
1826                 rid = PCIR_BAR(4);
1827                 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->rl_res_pba);
1828         }
1829         if (sc->rl_res)
1830                 bus_release_resource(dev, sc->rl_res_type, sc->rl_res_id,
1831                     sc->rl_res);
1832
1833         /* Unload and free the RX DMA ring memory and map */
1834
1835         if (sc->rl_ldata.rl_rx_list_tag) {
1836                 if (sc->rl_ldata.rl_rx_list_addr)
1837                         bus_dmamap_unload(sc->rl_ldata.rl_rx_list_tag,
1838                             sc->rl_ldata.rl_rx_list_map);
1839                 if (sc->rl_ldata.rl_rx_list)
1840                         bus_dmamem_free(sc->rl_ldata.rl_rx_list_tag,
1841                             sc->rl_ldata.rl_rx_list,
1842                             sc->rl_ldata.rl_rx_list_map);
1843                 bus_dma_tag_destroy(sc->rl_ldata.rl_rx_list_tag);
1844         }
1845
1846         /* Unload and free the TX DMA ring memory and map */
1847
1848         if (sc->rl_ldata.rl_tx_list_tag) {
1849                 if (sc->rl_ldata.rl_tx_list_addr)
1850                         bus_dmamap_unload(sc->rl_ldata.rl_tx_list_tag,
1851                             sc->rl_ldata.rl_tx_list_map);
1852                 if (sc->rl_ldata.rl_tx_list)
1853                         bus_dmamem_free(sc->rl_ldata.rl_tx_list_tag,
1854                             sc->rl_ldata.rl_tx_list,
1855                             sc->rl_ldata.rl_tx_list_map);
1856                 bus_dma_tag_destroy(sc->rl_ldata.rl_tx_list_tag);
1857         }
1858
1859         /* Destroy all the RX and TX buffer maps */
1860
1861         if (sc->rl_ldata.rl_tx_mtag) {
1862                 for (i = 0; i < sc->rl_ldata.rl_tx_desc_cnt; i++) {
1863                         if (sc->rl_ldata.rl_tx_desc[i].tx_dmamap)
1864                                 bus_dmamap_destroy(sc->rl_ldata.rl_tx_mtag,
1865                                     sc->rl_ldata.rl_tx_desc[i].tx_dmamap);
1866                 }
1867                 bus_dma_tag_destroy(sc->rl_ldata.rl_tx_mtag);
1868         }
1869         if (sc->rl_ldata.rl_rx_mtag) {
1870                 for (i = 0; i < sc->rl_ldata.rl_rx_desc_cnt; i++) {
1871                         if (sc->rl_ldata.rl_rx_desc[i].rx_dmamap)
1872                                 bus_dmamap_destroy(sc->rl_ldata.rl_rx_mtag,
1873                                     sc->rl_ldata.rl_rx_desc[i].rx_dmamap);
1874                 }
1875                 if (sc->rl_ldata.rl_rx_sparemap)
1876                         bus_dmamap_destroy(sc->rl_ldata.rl_rx_mtag,
1877                             sc->rl_ldata.rl_rx_sparemap);
1878                 bus_dma_tag_destroy(sc->rl_ldata.rl_rx_mtag);
1879         }
1880         if (sc->rl_ldata.rl_jrx_mtag) {
1881                 for (i = 0; i < sc->rl_ldata.rl_rx_desc_cnt; i++) {
1882                         if (sc->rl_ldata.rl_jrx_desc[i].rx_dmamap)
1883                                 bus_dmamap_destroy(sc->rl_ldata.rl_jrx_mtag,
1884                                     sc->rl_ldata.rl_jrx_desc[i].rx_dmamap);
1885                 }
1886                 if (sc->rl_ldata.rl_jrx_sparemap)
1887                         bus_dmamap_destroy(sc->rl_ldata.rl_jrx_mtag,
1888                             sc->rl_ldata.rl_jrx_sparemap);
1889                 bus_dma_tag_destroy(sc->rl_ldata.rl_jrx_mtag);
1890         }
1891         /* Unload and free the stats buffer and map */
1892
1893         if (sc->rl_ldata.rl_stag) {
1894                 if (sc->rl_ldata.rl_stats_addr)
1895                         bus_dmamap_unload(sc->rl_ldata.rl_stag,
1896                             sc->rl_ldata.rl_smap);
1897                 if (sc->rl_ldata.rl_stats)
1898                         bus_dmamem_free(sc->rl_ldata.rl_stag,
1899                             sc->rl_ldata.rl_stats, sc->rl_ldata.rl_smap);
1900                 bus_dma_tag_destroy(sc->rl_ldata.rl_stag);
1901         }
1902
1903         if (sc->rl_parent_tag)
1904                 bus_dma_tag_destroy(sc->rl_parent_tag);
1905
1906         mtx_destroy(&sc->rl_mtx);
1907
1908         return (0);
1909 }
1910
1911 static __inline void
1912 re_discard_rxbuf(struct rl_softc *sc, int idx)
1913 {
1914         struct rl_desc          *desc;
1915         struct rl_rxdesc        *rxd;
1916         uint32_t                cmdstat;
1917
1918         if (sc->rl_ifp->if_mtu > RL_MTU &&
1919             (sc->rl_flags & RL_FLAG_JUMBOV2) != 0)
1920                 rxd = &sc->rl_ldata.rl_jrx_desc[idx];
1921         else
1922                 rxd = &sc->rl_ldata.rl_rx_desc[idx];
1923         desc = &sc->rl_ldata.rl_rx_list[idx];
1924         desc->rl_vlanctl = 0;
1925         cmdstat = rxd->rx_size;
1926         if (idx == sc->rl_ldata.rl_rx_desc_cnt - 1)
1927                 cmdstat |= RL_RDESC_CMD_EOR;
1928         desc->rl_cmdstat = htole32(cmdstat | RL_RDESC_CMD_OWN);
1929 }
1930
1931 static int
1932 re_newbuf(struct rl_softc *sc, int idx)
1933 {
1934         struct mbuf             *m;
1935         struct rl_rxdesc        *rxd;
1936         bus_dma_segment_t       segs[1];
1937         bus_dmamap_t            map;
1938         struct rl_desc          *desc;
1939         uint32_t                cmdstat;
1940         int                     error, nsegs;
1941
1942         m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
1943         if (m == NULL)
1944                 return (ENOBUFS);
1945
1946         m->m_len = m->m_pkthdr.len = MCLBYTES;
1947 #ifdef RE_FIXUP_RX
1948         /*
1949          * This is part of an evil trick to deal with non-x86 platforms.
1950          * The RealTek chip requires RX buffers to be aligned on 64-bit
1951          * boundaries, but that will hose non-x86 machines. To get around
1952          * this, we leave some empty space at the start of each buffer
1953          * and for non-x86 hosts, we copy the buffer back six bytes
1954          * to achieve word alignment. This is slightly more efficient
1955          * than allocating a new buffer, copying the contents, and
1956          * discarding the old buffer.
1957          */
1958         m_adj(m, RE_ETHER_ALIGN);
1959 #endif
1960         error = bus_dmamap_load_mbuf_sg(sc->rl_ldata.rl_rx_mtag,
1961             sc->rl_ldata.rl_rx_sparemap, m, segs, &nsegs, BUS_DMA_NOWAIT);
1962         if (error != 0) {
1963                 m_freem(m);
1964                 return (ENOBUFS);
1965         }
1966         KASSERT(nsegs == 1, ("%s: %d segment returned!", __func__, nsegs));
1967
1968         rxd = &sc->rl_ldata.rl_rx_desc[idx];
1969         if (rxd->rx_m != NULL) {
1970                 bus_dmamap_sync(sc->rl_ldata.rl_rx_mtag, rxd->rx_dmamap,
1971                     BUS_DMASYNC_POSTREAD);
1972                 bus_dmamap_unload(sc->rl_ldata.rl_rx_mtag, rxd->rx_dmamap);
1973         }
1974
1975         rxd->rx_m = m;
1976         map = rxd->rx_dmamap;
1977         rxd->rx_dmamap = sc->rl_ldata.rl_rx_sparemap;
1978         rxd->rx_size = segs[0].ds_len;
1979         sc->rl_ldata.rl_rx_sparemap = map;
1980         bus_dmamap_sync(sc->rl_ldata.rl_rx_mtag, rxd->rx_dmamap,
1981             BUS_DMASYNC_PREREAD);
1982
1983         desc = &sc->rl_ldata.rl_rx_list[idx];
1984         desc->rl_vlanctl = 0;
1985         desc->rl_bufaddr_lo = htole32(RL_ADDR_LO(segs[0].ds_addr));
1986         desc->rl_bufaddr_hi = htole32(RL_ADDR_HI(segs[0].ds_addr));
1987         cmdstat = segs[0].ds_len;
1988         if (idx == sc->rl_ldata.rl_rx_desc_cnt - 1)
1989                 cmdstat |= RL_RDESC_CMD_EOR;
1990         desc->rl_cmdstat = htole32(cmdstat | RL_RDESC_CMD_OWN);
1991
1992         return (0);
1993 }
1994
1995 static int
1996 re_jumbo_newbuf(struct rl_softc *sc, int idx)
1997 {
1998         struct mbuf             *m;
1999         struct rl_rxdesc        *rxd;
2000         bus_dma_segment_t       segs[1];
2001         bus_dmamap_t            map;
2002         struct rl_desc          *desc;
2003         uint32_t                cmdstat;
2004         int                     error, nsegs;
2005
2006         m = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, MJUM9BYTES);
2007         if (m == NULL)
2008                 return (ENOBUFS);
2009         m->m_len = m->m_pkthdr.len = MJUM9BYTES;
2010 #ifdef RE_FIXUP_RX
2011         m_adj(m, RE_ETHER_ALIGN);
2012 #endif
2013         error = bus_dmamap_load_mbuf_sg(sc->rl_ldata.rl_jrx_mtag,
2014             sc->rl_ldata.rl_jrx_sparemap, m, segs, &nsegs, BUS_DMA_NOWAIT);
2015         if (error != 0) {
2016                 m_freem(m);
2017                 return (ENOBUFS);
2018         }
2019         KASSERT(nsegs == 1, ("%s: %d segment returned!", __func__, nsegs));
2020
2021         rxd = &sc->rl_ldata.rl_jrx_desc[idx];
2022         if (rxd->rx_m != NULL) {
2023                 bus_dmamap_sync(sc->rl_ldata.rl_jrx_mtag, rxd->rx_dmamap,
2024                     BUS_DMASYNC_POSTREAD);
2025                 bus_dmamap_unload(sc->rl_ldata.rl_jrx_mtag, rxd->rx_dmamap);
2026         }
2027
2028         rxd->rx_m = m;
2029         map = rxd->rx_dmamap;
2030         rxd->rx_dmamap = sc->rl_ldata.rl_jrx_sparemap;
2031         rxd->rx_size = segs[0].ds_len;
2032         sc->rl_ldata.rl_jrx_sparemap = map;
2033         bus_dmamap_sync(sc->rl_ldata.rl_jrx_mtag, rxd->rx_dmamap,
2034             BUS_DMASYNC_PREREAD);
2035
2036         desc = &sc->rl_ldata.rl_rx_list[idx];
2037         desc->rl_vlanctl = 0;
2038         desc->rl_bufaddr_lo = htole32(RL_ADDR_LO(segs[0].ds_addr));
2039         desc->rl_bufaddr_hi = htole32(RL_ADDR_HI(segs[0].ds_addr));
2040         cmdstat = segs[0].ds_len;
2041         if (idx == sc->rl_ldata.rl_rx_desc_cnt - 1)
2042                 cmdstat |= RL_RDESC_CMD_EOR;
2043         desc->rl_cmdstat = htole32(cmdstat | RL_RDESC_CMD_OWN);
2044
2045         return (0);
2046 }
2047
2048 #ifdef RE_FIXUP_RX
2049 static __inline void
2050 re_fixup_rx(struct mbuf *m)
2051 {
2052         int                     i;
2053         uint16_t                *src, *dst;
2054
2055         src = mtod(m, uint16_t *);
2056         dst = src - (RE_ETHER_ALIGN - ETHER_ALIGN) / sizeof *src;
2057
2058         for (i = 0; i < (m->m_len / sizeof(uint16_t) + 1); i++)
2059                 *dst++ = *src++;
2060
2061         m->m_data -= RE_ETHER_ALIGN - ETHER_ALIGN;
2062 }
2063 #endif
2064
2065 static int
2066 re_tx_list_init(struct rl_softc *sc)
2067 {
2068         struct rl_desc          *desc;
2069         int                     i;
2070
2071         RL_LOCK_ASSERT(sc);
2072
2073         bzero(sc->rl_ldata.rl_tx_list,
2074             sc->rl_ldata.rl_tx_desc_cnt * sizeof(struct rl_desc));
2075         for (i = 0; i < sc->rl_ldata.rl_tx_desc_cnt; i++)
2076                 sc->rl_ldata.rl_tx_desc[i].tx_m = NULL;
2077 #ifdef DEV_NETMAP
2078         re_netmap_tx_init(sc);
2079 #endif /* DEV_NETMAP */
2080         /* Set EOR. */
2081         desc = &sc->rl_ldata.rl_tx_list[sc->rl_ldata.rl_tx_desc_cnt - 1];
2082         desc->rl_cmdstat |= htole32(RL_TDESC_CMD_EOR);
2083
2084         bus_dmamap_sync(sc->rl_ldata.rl_tx_list_tag,
2085             sc->rl_ldata.rl_tx_list_map,
2086             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2087
2088         sc->rl_ldata.rl_tx_prodidx = 0;
2089         sc->rl_ldata.rl_tx_considx = 0;
2090         sc->rl_ldata.rl_tx_free = sc->rl_ldata.rl_tx_desc_cnt;
2091
2092         return (0);
2093 }
2094
2095 static int
2096 re_rx_list_init(struct rl_softc *sc)
2097 {
2098         int                     error, i;
2099
2100         bzero(sc->rl_ldata.rl_rx_list,
2101             sc->rl_ldata.rl_rx_desc_cnt * sizeof(struct rl_desc));
2102         for (i = 0; i < sc->rl_ldata.rl_rx_desc_cnt; i++) {
2103                 sc->rl_ldata.rl_rx_desc[i].rx_m = NULL;
2104                 if ((error = re_newbuf(sc, i)) != 0)
2105                         return (error);
2106         }
2107 #ifdef DEV_NETMAP
2108         re_netmap_rx_init(sc);
2109 #endif /* DEV_NETMAP */
2110
2111         /* Flush the RX descriptors */
2112
2113         bus_dmamap_sync(sc->rl_ldata.rl_rx_list_tag,
2114             sc->rl_ldata.rl_rx_list_map,
2115             BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD);
2116
2117         sc->rl_ldata.rl_rx_prodidx = 0;
2118         sc->rl_head = sc->rl_tail = NULL;
2119         sc->rl_int_rx_act = 0;
2120
2121         return (0);
2122 }
2123
2124 static int
2125 re_jrx_list_init(struct rl_softc *sc)
2126 {
2127         int                     error, i;
2128
2129         bzero(sc->rl_ldata.rl_rx_list,
2130             sc->rl_ldata.rl_rx_desc_cnt * sizeof(struct rl_desc));
2131         for (i = 0; i < sc->rl_ldata.rl_rx_desc_cnt; i++) {
2132                 sc->rl_ldata.rl_jrx_desc[i].rx_m = NULL;
2133                 if ((error = re_jumbo_newbuf(sc, i)) != 0)
2134                         return (error);
2135         }
2136
2137         bus_dmamap_sync(sc->rl_ldata.rl_rx_list_tag,
2138             sc->rl_ldata.rl_rx_list_map,
2139             BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2140
2141         sc->rl_ldata.rl_rx_prodidx = 0;
2142         sc->rl_head = sc->rl_tail = NULL;
2143         sc->rl_int_rx_act = 0;
2144
2145         return (0);
2146 }
2147
2148 /*
2149  * RX handler for C+ and 8169. For the gigE chips, we support
2150  * the reception of jumbo frames that have been fragmented
2151  * across multiple 2K mbuf cluster buffers.
2152  */
2153 static int
2154 re_rxeof(struct rl_softc *sc, int *rx_npktsp)
2155 {
2156         struct mbuf             *m;
2157         struct ifnet            *ifp;
2158         int                     i, rxerr, total_len;
2159         struct rl_desc          *cur_rx;
2160         u_int32_t               rxstat, rxvlan;
2161         int                     jumbo, maxpkt = 16, rx_npkts = 0;
2162
2163         RL_LOCK_ASSERT(sc);
2164
2165         ifp = sc->rl_ifp;
2166 #ifdef DEV_NETMAP
2167         if (netmap_rx_irq(ifp, 0, &rx_npkts))
2168                 return 0;
2169 #endif /* DEV_NETMAP */
2170         if (ifp->if_mtu > RL_MTU && (sc->rl_flags & RL_FLAG_JUMBOV2) != 0)
2171                 jumbo = 1;
2172         else
2173                 jumbo = 0;
2174
2175         /* Invalidate the descriptor memory */
2176
2177         bus_dmamap_sync(sc->rl_ldata.rl_rx_list_tag,
2178             sc->rl_ldata.rl_rx_list_map,
2179             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2180
2181         for (i = sc->rl_ldata.rl_rx_prodidx; maxpkt > 0;
2182             i = RL_RX_DESC_NXT(sc, i)) {
2183                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
2184                         break;
2185                 cur_rx = &sc->rl_ldata.rl_rx_list[i];
2186                 rxstat = le32toh(cur_rx->rl_cmdstat);
2187                 if ((rxstat & RL_RDESC_STAT_OWN) != 0)
2188                         break;
2189                 total_len = rxstat & sc->rl_rxlenmask;
2190                 rxvlan = le32toh(cur_rx->rl_vlanctl);
2191                 if (jumbo != 0)
2192                         m = sc->rl_ldata.rl_jrx_desc[i].rx_m;
2193                 else
2194                         m = sc->rl_ldata.rl_rx_desc[i].rx_m;
2195
2196                 if ((sc->rl_flags & RL_FLAG_JUMBOV2) != 0 &&
2197                     (rxstat & (RL_RDESC_STAT_SOF | RL_RDESC_STAT_EOF)) !=
2198                     (RL_RDESC_STAT_SOF | RL_RDESC_STAT_EOF)) {
2199                         /*
2200                          * RTL8168C or later controllers do not
2201                          * support multi-fragment packet.
2202                          */
2203                         re_discard_rxbuf(sc, i);
2204                         continue;
2205                 } else if ((rxstat & RL_RDESC_STAT_EOF) == 0) {
2206                         if (re_newbuf(sc, i) != 0) {
2207                                 /*
2208                                  * If this is part of a multi-fragment packet,
2209                                  * discard all the pieces.
2210                                  */
2211                                 if (sc->rl_head != NULL) {
2212                                         m_freem(sc->rl_head);
2213                                         sc->rl_head = sc->rl_tail = NULL;
2214                                 }
2215                                 re_discard_rxbuf(sc, i);
2216                                 continue;
2217                         }
2218                         m->m_len = RE_RX_DESC_BUFLEN;
2219                         if (sc->rl_head == NULL)
2220                                 sc->rl_head = sc->rl_tail = m;
2221                         else {
2222                                 m->m_flags &= ~M_PKTHDR;
2223                                 sc->rl_tail->m_next = m;
2224                                 sc->rl_tail = m;
2225                         }
2226                         continue;
2227                 }
2228
2229                 /*
2230                  * NOTE: for the 8139C+, the frame length field
2231                  * is always 12 bits in size, but for the gigE chips,
2232                  * it is 13 bits (since the max RX frame length is 16K).
2233                  * Unfortunately, all 32 bits in the status word
2234                  * were already used, so to make room for the extra
2235                  * length bit, RealTek took out the 'frame alignment
2236                  * error' bit and shifted the other status bits
2237                  * over one slot. The OWN, EOR, FS and LS bits are
2238                  * still in the same places. We have already extracted
2239                  * the frame length and checked the OWN bit, so rather
2240                  * than using an alternate bit mapping, we shift the
2241                  * status bits one space to the right so we can evaluate
2242                  * them using the 8169 status as though it was in the
2243                  * same format as that of the 8139C+.
2244                  */
2245                 if (sc->rl_type == RL_8169)
2246                         rxstat >>= 1;
2247
2248                 /*
2249                  * if total_len > 2^13-1, both _RXERRSUM and _GIANT will be
2250                  * set, but if CRC is clear, it will still be a valid frame.
2251                  */
2252                 if ((rxstat & RL_RDESC_STAT_RXERRSUM) != 0) {
2253                         rxerr = 1;
2254                         if ((sc->rl_flags & RL_FLAG_JUMBOV2) == 0 &&
2255                             total_len > 8191 &&
2256                             (rxstat & RL_RDESC_STAT_ERRS) == RL_RDESC_STAT_GIANT)
2257                                 rxerr = 0;
2258                         if (rxerr != 0) {
2259                                 if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
2260                                 /*
2261                                  * If this is part of a multi-fragment packet,
2262                                  * discard all the pieces.
2263                                  */
2264                                 if (sc->rl_head != NULL) {
2265                                         m_freem(sc->rl_head);
2266                                         sc->rl_head = sc->rl_tail = NULL;
2267                                 }
2268                                 re_discard_rxbuf(sc, i);
2269                                 continue;
2270                         }
2271                 }
2272
2273                 /*
2274                  * If allocating a replacement mbuf fails,
2275                  * reload the current one.
2276                  */
2277                 if (jumbo != 0)
2278                         rxerr = re_jumbo_newbuf(sc, i);
2279                 else
2280                         rxerr = re_newbuf(sc, i);
2281                 if (rxerr != 0) {
2282                         if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1);
2283                         if (sc->rl_head != NULL) {
2284                                 m_freem(sc->rl_head);
2285                                 sc->rl_head = sc->rl_tail = NULL;
2286                         }
2287                         re_discard_rxbuf(sc, i);
2288                         continue;
2289                 }
2290
2291                 if (sc->rl_head != NULL) {
2292                         if (jumbo != 0)
2293                                 m->m_len = total_len;
2294                         else {
2295                                 m->m_len = total_len % RE_RX_DESC_BUFLEN;
2296                                 if (m->m_len == 0)
2297                                         m->m_len = RE_RX_DESC_BUFLEN;
2298                         }
2299                         /*
2300                          * Special case: if there's 4 bytes or less
2301                          * in this buffer, the mbuf can be discarded:
2302                          * the last 4 bytes is the CRC, which we don't
2303                          * care about anyway.
2304                          */
2305                         if (m->m_len <= ETHER_CRC_LEN) {
2306                                 sc->rl_tail->m_len -=
2307                                     (ETHER_CRC_LEN - m->m_len);
2308                                 m_freem(m);
2309                         } else {
2310                                 m->m_len -= ETHER_CRC_LEN;
2311                                 m->m_flags &= ~M_PKTHDR;
2312                                 sc->rl_tail->m_next = m;
2313                         }
2314                         m = sc->rl_head;
2315                         sc->rl_head = sc->rl_tail = NULL;
2316                         m->m_pkthdr.len = total_len - ETHER_CRC_LEN;
2317                 } else
2318                         m->m_pkthdr.len = m->m_len =
2319                             (total_len - ETHER_CRC_LEN);
2320
2321 #ifdef RE_FIXUP_RX
2322                 re_fixup_rx(m);
2323 #endif
2324                 if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);
2325                 m->m_pkthdr.rcvif = ifp;
2326
2327                 /* Do RX checksumming if enabled */
2328
2329                 if (ifp->if_capenable & IFCAP_RXCSUM) {
2330                         if ((sc->rl_flags & RL_FLAG_DESCV2) == 0) {
2331                                 /* Check IP header checksum */
2332                                 if (rxstat & RL_RDESC_STAT_PROTOID)
2333                                         m->m_pkthdr.csum_flags |=
2334                                             CSUM_IP_CHECKED;
2335                                 if (!(rxstat & RL_RDESC_STAT_IPSUMBAD))
2336                                         m->m_pkthdr.csum_flags |=
2337                                             CSUM_IP_VALID;
2338
2339                                 /* Check TCP/UDP checksum */
2340                                 if ((RL_TCPPKT(rxstat) &&
2341                                     !(rxstat & RL_RDESC_STAT_TCPSUMBAD)) ||
2342                                     (RL_UDPPKT(rxstat) &&
2343                                      !(rxstat & RL_RDESC_STAT_UDPSUMBAD))) {
2344                                         m->m_pkthdr.csum_flags |=
2345                                                 CSUM_DATA_VALID|CSUM_PSEUDO_HDR;
2346                                         m->m_pkthdr.csum_data = 0xffff;
2347                                 }
2348                         } else {
2349                                 /*
2350                                  * RTL8168C/RTL816CP/RTL8111C/RTL8111CP
2351                                  */
2352                                 if ((rxstat & RL_RDESC_STAT_PROTOID) &&
2353                                     (rxvlan & RL_RDESC_IPV4))
2354                                         m->m_pkthdr.csum_flags |=
2355                                             CSUM_IP_CHECKED;
2356                                 if (!(rxstat & RL_RDESC_STAT_IPSUMBAD) &&
2357                                     (rxvlan & RL_RDESC_IPV4))
2358                                         m->m_pkthdr.csum_flags |=
2359                                             CSUM_IP_VALID;
2360                                 if (((rxstat & RL_RDESC_STAT_TCP) &&
2361                                     !(rxstat & RL_RDESC_STAT_TCPSUMBAD)) ||
2362                                     ((rxstat & RL_RDESC_STAT_UDP) &&
2363                                     !(rxstat & RL_RDESC_STAT_UDPSUMBAD))) {
2364                                         m->m_pkthdr.csum_flags |=
2365                                                 CSUM_DATA_VALID|CSUM_PSEUDO_HDR;
2366                                         m->m_pkthdr.csum_data = 0xffff;
2367                                 }
2368                         }
2369                 }
2370                 maxpkt--;
2371                 if (rxvlan & RL_RDESC_VLANCTL_TAG) {
2372                         m->m_pkthdr.ether_vtag =
2373                             bswap16((rxvlan & RL_RDESC_VLANCTL_DATA));
2374                         m->m_flags |= M_VLANTAG;
2375                 }
2376                 RL_UNLOCK(sc);
2377                 (*ifp->if_input)(ifp, m);
2378                 RL_LOCK(sc);
2379                 rx_npkts++;
2380         }
2381
2382         /* Flush the RX DMA ring */
2383
2384         bus_dmamap_sync(sc->rl_ldata.rl_rx_list_tag,
2385             sc->rl_ldata.rl_rx_list_map,
2386             BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD);
2387
2388         sc->rl_ldata.rl_rx_prodidx = i;
2389
2390         if (rx_npktsp != NULL)
2391                 *rx_npktsp = rx_npkts;
2392         if (maxpkt)
2393                 return (EAGAIN);
2394
2395         return (0);
2396 }
2397
2398 static void
2399 re_txeof(struct rl_softc *sc)
2400 {
2401         struct ifnet            *ifp;
2402         struct rl_txdesc        *txd;
2403         u_int32_t               txstat;
2404         int                     cons;
2405
2406         cons = sc->rl_ldata.rl_tx_considx;
2407         if (cons == sc->rl_ldata.rl_tx_prodidx)
2408                 return;
2409
2410         ifp = sc->rl_ifp;
2411 #ifdef DEV_NETMAP
2412         if (netmap_tx_irq(ifp, 0))
2413                 return;
2414 #endif /* DEV_NETMAP */
2415         /* Invalidate the TX descriptor list */
2416         bus_dmamap_sync(sc->rl_ldata.rl_tx_list_tag,
2417             sc->rl_ldata.rl_tx_list_map,
2418             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2419
2420         for (; cons != sc->rl_ldata.rl_tx_prodidx;
2421             cons = RL_TX_DESC_NXT(sc, cons)) {
2422                 txstat = le32toh(sc->rl_ldata.rl_tx_list[cons].rl_cmdstat);
2423                 if (txstat & RL_TDESC_STAT_OWN)
2424                         break;
2425                 /*
2426                  * We only stash mbufs in the last descriptor
2427                  * in a fragment chain, which also happens to
2428                  * be the only place where the TX status bits
2429                  * are valid.
2430                  */
2431                 if (txstat & RL_TDESC_CMD_EOF) {
2432                         txd = &sc->rl_ldata.rl_tx_desc[cons];
2433                         bus_dmamap_sync(sc->rl_ldata.rl_tx_mtag,
2434                             txd->tx_dmamap, BUS_DMASYNC_POSTWRITE);
2435                         bus_dmamap_unload(sc->rl_ldata.rl_tx_mtag,
2436                             txd->tx_dmamap);
2437                         KASSERT(txd->tx_m != NULL,
2438                             ("%s: freeing NULL mbufs!", __func__));
2439                         m_freem(txd->tx_m);
2440                         txd->tx_m = NULL;
2441                         if (txstat & (RL_TDESC_STAT_EXCESSCOL|
2442                             RL_TDESC_STAT_COLCNT))
2443                                 if_inc_counter(ifp, IFCOUNTER_COLLISIONS, 1);
2444                         if (txstat & RL_TDESC_STAT_TXERRSUM)
2445                                 if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
2446                         else
2447                                 if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1);
2448                 }
2449                 sc->rl_ldata.rl_tx_free++;
2450                 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
2451         }
2452         sc->rl_ldata.rl_tx_considx = cons;
2453
2454         /* No changes made to the TX ring, so no flush needed */
2455
2456         if (sc->rl_ldata.rl_tx_free != sc->rl_ldata.rl_tx_desc_cnt) {
2457 #ifdef RE_TX_MODERATION
2458                 /*
2459                  * If not all descriptors have been reaped yet, reload
2460                  * the timer so that we will eventually get another
2461                  * interrupt that will cause us to re-enter this routine.
2462                  * This is done in case the transmitter has gone idle.
2463                  */
2464                 CSR_WRITE_4(sc, RL_TIMERCNT, 1);
2465 #endif
2466         } else
2467                 sc->rl_watchdog_timer = 0;
2468 }
2469
2470 static void
2471 re_tick(void *xsc)
2472 {
2473         struct rl_softc         *sc;
2474         struct mii_data         *mii;
2475
2476         sc = xsc;
2477
2478         RL_LOCK_ASSERT(sc);
2479
2480         mii = device_get_softc(sc->rl_miibus);
2481         mii_tick(mii);
2482         if ((sc->rl_flags & RL_FLAG_LINK) == 0)
2483                 re_miibus_statchg(sc->rl_dev);
2484         /*
2485          * Reclaim transmitted frames here. Technically it is not
2486          * necessary to do here but it ensures periodic reclamation
2487          * regardless of Tx completion interrupt which seems to be
2488          * lost on PCIe based controllers under certain situations.
2489          */
2490         re_txeof(sc);
2491         re_watchdog(sc);
2492         callout_reset(&sc->rl_stat_callout, hz, re_tick, sc);
2493 }
2494
2495 #ifdef DEVICE_POLLING
2496 static int
2497 re_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
2498 {
2499         struct rl_softc *sc = ifp->if_softc;
2500         int rx_npkts = 0;
2501
2502         RL_LOCK(sc);
2503         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
2504                 rx_npkts = re_poll_locked(ifp, cmd, count);
2505         RL_UNLOCK(sc);
2506         return (rx_npkts);
2507 }
2508
2509 static int
2510 re_poll_locked(struct ifnet *ifp, enum poll_cmd cmd, int count)
2511 {
2512         struct rl_softc *sc = ifp->if_softc;
2513         int rx_npkts;
2514
2515         RL_LOCK_ASSERT(sc);
2516
2517         sc->rxcycles = count;
2518         re_rxeof(sc, &rx_npkts);
2519         re_txeof(sc);
2520
2521         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
2522                 re_start_locked(ifp);
2523
2524         if (cmd == POLL_AND_CHECK_STATUS) { /* also check status register */
2525                 u_int16_t       status;
2526
2527                 status = CSR_READ_2(sc, RL_ISR);
2528                 if (status == 0xffff)
2529                         return (rx_npkts);
2530                 if (status)
2531                         CSR_WRITE_2(sc, RL_ISR, status);
2532                 if ((status & (RL_ISR_TX_OK | RL_ISR_TX_DESC_UNAVAIL)) &&
2533                     (sc->rl_flags & RL_FLAG_PCIE))
2534                         CSR_WRITE_1(sc, sc->rl_txstart, RL_TXSTART_START);
2535
2536                 /*
2537                  * XXX check behaviour on receiver stalls.
2538                  */
2539
2540                 if (status & RL_ISR_SYSTEM_ERR) {
2541                         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2542                         re_init_locked(sc);
2543                 }
2544         }
2545         return (rx_npkts);
2546 }
2547 #endif /* DEVICE_POLLING */
2548
2549 static int
2550 re_intr(void *arg)
2551 {
2552         struct rl_softc         *sc;
2553         uint16_t                status;
2554
2555         sc = arg;
2556
2557         status = CSR_READ_2(sc, RL_ISR);
2558         if (status == 0xFFFF || (status & RL_INTRS_CPLUS) == 0)
2559                 return (FILTER_STRAY);
2560         CSR_WRITE_2(sc, RL_IMR, 0);
2561
2562         taskqueue_enqueue(taskqueue_fast, &sc->rl_inttask);
2563
2564         return (FILTER_HANDLED);
2565 }
2566
2567 static void
2568 re_int_task(void *arg, int npending)
2569 {
2570         struct rl_softc         *sc;
2571         struct ifnet            *ifp;
2572         u_int16_t               status;
2573         int                     rval = 0;
2574
2575         sc = arg;
2576         ifp = sc->rl_ifp;
2577
2578         RL_LOCK(sc);
2579
2580         status = CSR_READ_2(sc, RL_ISR);
2581         CSR_WRITE_2(sc, RL_ISR, status);
2582
2583         if (sc->suspended ||
2584             (ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
2585                 RL_UNLOCK(sc);
2586                 return;
2587         }
2588
2589 #ifdef DEVICE_POLLING
2590         if  (ifp->if_capenable & IFCAP_POLLING) {
2591                 RL_UNLOCK(sc);
2592                 return;
2593         }
2594 #endif
2595
2596         if (status & (RL_ISR_RX_OK|RL_ISR_RX_ERR|RL_ISR_FIFO_OFLOW))
2597                 rval = re_rxeof(sc, NULL);
2598
2599         /*
2600          * Some chips will ignore a second TX request issued
2601          * while an existing transmission is in progress. If
2602          * the transmitter goes idle but there are still
2603          * packets waiting to be sent, we need to restart the
2604          * channel here to flush them out. This only seems to
2605          * be required with the PCIe devices.
2606          */
2607         if ((status & (RL_ISR_TX_OK | RL_ISR_TX_DESC_UNAVAIL)) &&
2608             (sc->rl_flags & RL_FLAG_PCIE))
2609                 CSR_WRITE_1(sc, sc->rl_txstart, RL_TXSTART_START);
2610         if (status & (
2611 #ifdef RE_TX_MODERATION
2612             RL_ISR_TIMEOUT_EXPIRED|
2613 #else
2614             RL_ISR_TX_OK|
2615 #endif
2616             RL_ISR_TX_ERR|RL_ISR_TX_DESC_UNAVAIL))
2617                 re_txeof(sc);
2618
2619         if (status & RL_ISR_SYSTEM_ERR) {
2620                 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2621                 re_init_locked(sc);
2622         }
2623
2624         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
2625                 re_start_locked(ifp);
2626
2627         RL_UNLOCK(sc);
2628
2629         if ((CSR_READ_2(sc, RL_ISR) & RL_INTRS_CPLUS) || rval) {
2630                 taskqueue_enqueue(taskqueue_fast, &sc->rl_inttask);
2631                 return;
2632         }
2633
2634         CSR_WRITE_2(sc, RL_IMR, RL_INTRS_CPLUS);
2635 }
2636
2637 static void
2638 re_intr_msi(void *xsc)
2639 {
2640         struct rl_softc         *sc;
2641         struct ifnet            *ifp;
2642         uint16_t                intrs, status;
2643
2644         sc = xsc;
2645         RL_LOCK(sc);
2646
2647         ifp = sc->rl_ifp;
2648 #ifdef DEVICE_POLLING
2649         if (ifp->if_capenable & IFCAP_POLLING) {
2650                 RL_UNLOCK(sc);
2651                 return;
2652         }
2653 #endif
2654         /* Disable interrupts. */
2655         CSR_WRITE_2(sc, RL_IMR, 0);
2656         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
2657                 RL_UNLOCK(sc);
2658                 return;
2659         }
2660
2661         intrs = RL_INTRS_CPLUS;
2662         status = CSR_READ_2(sc, RL_ISR);
2663         CSR_WRITE_2(sc, RL_ISR, status);
2664         if (sc->rl_int_rx_act > 0) {
2665                 intrs &= ~(RL_ISR_RX_OK | RL_ISR_RX_ERR | RL_ISR_FIFO_OFLOW |
2666                     RL_ISR_RX_OVERRUN);
2667                 status &= ~(RL_ISR_RX_OK | RL_ISR_RX_ERR | RL_ISR_FIFO_OFLOW |
2668                     RL_ISR_RX_OVERRUN);
2669         }
2670
2671         if (status & (RL_ISR_TIMEOUT_EXPIRED | RL_ISR_RX_OK | RL_ISR_RX_ERR |
2672             RL_ISR_FIFO_OFLOW | RL_ISR_RX_OVERRUN)) {
2673                 re_rxeof(sc, NULL);
2674                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) {
2675                         if (sc->rl_int_rx_mod != 0 &&
2676                             (status & (RL_ISR_RX_OK | RL_ISR_RX_ERR |
2677                             RL_ISR_FIFO_OFLOW | RL_ISR_RX_OVERRUN)) != 0) {
2678                                 /* Rearm one-shot timer. */
2679                                 CSR_WRITE_4(sc, RL_TIMERCNT, 1);
2680                                 intrs &= ~(RL_ISR_RX_OK | RL_ISR_RX_ERR |
2681                                     RL_ISR_FIFO_OFLOW | RL_ISR_RX_OVERRUN);
2682                                 sc->rl_int_rx_act = 1;
2683                         } else {
2684                                 intrs |= RL_ISR_RX_OK | RL_ISR_RX_ERR |
2685                                     RL_ISR_FIFO_OFLOW | RL_ISR_RX_OVERRUN;
2686                                 sc->rl_int_rx_act = 0;
2687                         }
2688                 }
2689         }
2690
2691         /*
2692          * Some chips will ignore a second TX request issued
2693          * while an existing transmission is in progress. If
2694          * the transmitter goes idle but there are still
2695          * packets waiting to be sent, we need to restart the
2696          * channel here to flush them out. This only seems to
2697          * be required with the PCIe devices.
2698          */
2699         if ((status & (RL_ISR_TX_OK | RL_ISR_TX_DESC_UNAVAIL)) &&
2700             (sc->rl_flags & RL_FLAG_PCIE))
2701                 CSR_WRITE_1(sc, sc->rl_txstart, RL_TXSTART_START);
2702         if (status & (RL_ISR_TX_OK | RL_ISR_TX_ERR | RL_ISR_TX_DESC_UNAVAIL))
2703                 re_txeof(sc);
2704
2705         if (status & RL_ISR_SYSTEM_ERR) {
2706                 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2707                 re_init_locked(sc);
2708         }
2709
2710         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) {
2711                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
2712                         re_start_locked(ifp);
2713                 CSR_WRITE_2(sc, RL_IMR, intrs);
2714         }
2715         RL_UNLOCK(sc);
2716 }
2717
2718 static int
2719 re_encap(struct rl_softc *sc, struct mbuf **m_head)
2720 {
2721         struct rl_txdesc        *txd, *txd_last;
2722         bus_dma_segment_t       segs[RL_NTXSEGS];
2723         bus_dmamap_t            map;
2724         struct mbuf             *m_new;
2725         struct rl_desc          *desc;
2726         int                     nsegs, prod;
2727         int                     i, error, ei, si;
2728         int                     padlen;
2729         uint32_t                cmdstat, csum_flags, vlanctl;
2730
2731         RL_LOCK_ASSERT(sc);
2732         M_ASSERTPKTHDR((*m_head));
2733
2734         /*
2735          * With some of the RealTek chips, using the checksum offload
2736          * support in conjunction with the autopadding feature results
2737          * in the transmission of corrupt frames. For example, if we
2738          * need to send a really small IP fragment that's less than 60
2739          * bytes in size, and IP header checksumming is enabled, the
2740          * resulting ethernet frame that appears on the wire will
2741          * have garbled payload. To work around this, if TX IP checksum
2742          * offload is enabled, we always manually pad short frames out
2743          * to the minimum ethernet frame size.
2744          */
2745         if ((sc->rl_flags & RL_FLAG_AUTOPAD) == 0 &&
2746             (*m_head)->m_pkthdr.len < RL_IP4CSUMTX_PADLEN &&
2747             ((*m_head)->m_pkthdr.csum_flags & CSUM_IP) != 0) {
2748                 padlen = RL_MIN_FRAMELEN - (*m_head)->m_pkthdr.len;
2749                 if (M_WRITABLE(*m_head) == 0) {
2750                         /* Get a writable copy. */
2751                         m_new = m_dup(*m_head, M_NOWAIT);
2752                         m_freem(*m_head);
2753                         if (m_new == NULL) {
2754                                 *m_head = NULL;
2755                                 return (ENOBUFS);
2756                         }
2757                         *m_head = m_new;
2758                 }
2759                 if ((*m_head)->m_next != NULL ||
2760                     M_TRAILINGSPACE(*m_head) < padlen) {
2761                         m_new = m_defrag(*m_head, M_NOWAIT);
2762                         if (m_new == NULL) {
2763                                 m_freem(*m_head);
2764                                 *m_head = NULL;
2765                                 return (ENOBUFS);
2766                         }
2767                 } else
2768                         m_new = *m_head;
2769
2770                 /*
2771                  * Manually pad short frames, and zero the pad space
2772                  * to avoid leaking data.
2773                  */
2774                 bzero(mtod(m_new, char *) + m_new->m_pkthdr.len, padlen);
2775                 m_new->m_pkthdr.len += padlen;
2776                 m_new->m_len = m_new->m_pkthdr.len;
2777                 *m_head = m_new;
2778         }
2779
2780         prod = sc->rl_ldata.rl_tx_prodidx;
2781         txd = &sc->rl_ldata.rl_tx_desc[prod];
2782         error = bus_dmamap_load_mbuf_sg(sc->rl_ldata.rl_tx_mtag, txd->tx_dmamap,
2783             *m_head, segs, &nsegs, BUS_DMA_NOWAIT);
2784         if (error == EFBIG) {
2785                 m_new = m_collapse(*m_head, M_NOWAIT, RL_NTXSEGS);
2786                 if (m_new == NULL) {
2787                         m_freem(*m_head);
2788                         *m_head = NULL;
2789                         return (ENOBUFS);
2790                 }
2791                 *m_head = m_new;
2792                 error = bus_dmamap_load_mbuf_sg(sc->rl_ldata.rl_tx_mtag,
2793                     txd->tx_dmamap, *m_head, segs, &nsegs, BUS_DMA_NOWAIT);
2794                 if (error != 0) {
2795                         m_freem(*m_head);
2796                         *m_head = NULL;
2797                         return (error);
2798                 }
2799         } else if (error != 0)
2800                 return (error);
2801         if (nsegs == 0) {
2802                 m_freem(*m_head);
2803                 *m_head = NULL;
2804                 return (EIO);
2805         }
2806
2807         /* Check for number of available descriptors. */
2808         if (sc->rl_ldata.rl_tx_free - nsegs <= 1) {
2809                 bus_dmamap_unload(sc->rl_ldata.rl_tx_mtag, txd->tx_dmamap);
2810                 return (ENOBUFS);
2811         }
2812
2813         bus_dmamap_sync(sc->rl_ldata.rl_tx_mtag, txd->tx_dmamap,
2814             BUS_DMASYNC_PREWRITE);
2815
2816         /*
2817          * Set up checksum offload. Note: checksum offload bits must
2818          * appear in all descriptors of a multi-descriptor transmit
2819          * attempt. This is according to testing done with an 8169
2820          * chip. This is a requirement.
2821          */
2822         vlanctl = 0;
2823         csum_flags = 0;
2824         if (((*m_head)->m_pkthdr.csum_flags & CSUM_TSO) != 0) {
2825                 if ((sc->rl_flags & RL_FLAG_DESCV2) != 0) {
2826                         csum_flags |= RL_TDESC_CMD_LGSEND;
2827                         vlanctl |= ((uint32_t)(*m_head)->m_pkthdr.tso_segsz <<
2828                             RL_TDESC_CMD_MSSVALV2_SHIFT);
2829                 } else {
2830                         csum_flags |= RL_TDESC_CMD_LGSEND |
2831                             ((uint32_t)(*m_head)->m_pkthdr.tso_segsz <<
2832                             RL_TDESC_CMD_MSSVAL_SHIFT);
2833                 }
2834         } else {
2835                 /*
2836                  * Unconditionally enable IP checksum if TCP or UDP
2837                  * checksum is required. Otherwise, TCP/UDP checksum
2838                  * doesn't make effects.
2839                  */
2840                 if (((*m_head)->m_pkthdr.csum_flags & RE_CSUM_FEATURES) != 0) {
2841                         if ((sc->rl_flags & RL_FLAG_DESCV2) == 0) {
2842                                 csum_flags |= RL_TDESC_CMD_IPCSUM;
2843                                 if (((*m_head)->m_pkthdr.csum_flags &
2844                                     CSUM_TCP) != 0)
2845                                         csum_flags |= RL_TDESC_CMD_TCPCSUM;
2846                                 if (((*m_head)->m_pkthdr.csum_flags &
2847                                     CSUM_UDP) != 0)
2848                                         csum_flags |= RL_TDESC_CMD_UDPCSUM;
2849                         } else {
2850                                 vlanctl |= RL_TDESC_CMD_IPCSUMV2;
2851                                 if (((*m_head)->m_pkthdr.csum_flags &
2852                                     CSUM_TCP) != 0)
2853                                         vlanctl |= RL_TDESC_CMD_TCPCSUMV2;
2854                                 if (((*m_head)->m_pkthdr.csum_flags &
2855                                     CSUM_UDP) != 0)
2856                                         vlanctl |= RL_TDESC_CMD_UDPCSUMV2;
2857                         }
2858                 }
2859         }
2860
2861         /*
2862          * Set up hardware VLAN tagging. Note: vlan tag info must
2863          * appear in all descriptors of a multi-descriptor
2864          * transmission attempt.
2865          */
2866         if ((*m_head)->m_flags & M_VLANTAG)
2867                 vlanctl |= bswap16((*m_head)->m_pkthdr.ether_vtag) |
2868                     RL_TDESC_VLANCTL_TAG;
2869
2870         si = prod;
2871         for (i = 0; i < nsegs; i++, prod = RL_TX_DESC_NXT(sc, prod)) {
2872                 desc = &sc->rl_ldata.rl_tx_list[prod];
2873                 desc->rl_vlanctl = htole32(vlanctl);
2874                 desc->rl_bufaddr_lo = htole32(RL_ADDR_LO(segs[i].ds_addr));
2875                 desc->rl_bufaddr_hi = htole32(RL_ADDR_HI(segs[i].ds_addr));
2876                 cmdstat = segs[i].ds_len;
2877                 if (i != 0)
2878                         cmdstat |= RL_TDESC_CMD_OWN;
2879                 if (prod == sc->rl_ldata.rl_tx_desc_cnt - 1)
2880                         cmdstat |= RL_TDESC_CMD_EOR;
2881                 desc->rl_cmdstat = htole32(cmdstat | csum_flags);
2882                 sc->rl_ldata.rl_tx_free--;
2883         }
2884         /* Update producer index. */
2885         sc->rl_ldata.rl_tx_prodidx = prod;
2886
2887         /* Set EOF on the last descriptor. */
2888         ei = RL_TX_DESC_PRV(sc, prod);
2889         desc = &sc->rl_ldata.rl_tx_list[ei];
2890         desc->rl_cmdstat |= htole32(RL_TDESC_CMD_EOF);
2891
2892         desc = &sc->rl_ldata.rl_tx_list[si];
2893         /* Set SOF and transfer ownership of packet to the chip. */
2894         desc->rl_cmdstat |= htole32(RL_TDESC_CMD_OWN | RL_TDESC_CMD_SOF);
2895
2896         /*
2897          * Insure that the map for this transmission
2898          * is placed at the array index of the last descriptor
2899          * in this chain.  (Swap last and first dmamaps.)
2900          */
2901         txd_last = &sc->rl_ldata.rl_tx_desc[ei];
2902         map = txd->tx_dmamap;
2903         txd->tx_dmamap = txd_last->tx_dmamap;
2904         txd_last->tx_dmamap = map;
2905         txd_last->tx_m = *m_head;
2906
2907         return (0);
2908 }
2909
2910 static void
2911 re_start(struct ifnet *ifp)
2912 {
2913         struct rl_softc         *sc;
2914
2915         sc = ifp->if_softc;
2916         RL_LOCK(sc);
2917         re_start_locked(ifp);
2918         RL_UNLOCK(sc);
2919 }
2920
2921 /*
2922  * Main transmit routine for C+ and gigE NICs.
2923  */
2924 static void
2925 re_start_locked(struct ifnet *ifp)
2926 {
2927         struct rl_softc         *sc;
2928         struct mbuf             *m_head;
2929         int                     queued;
2930
2931         sc = ifp->if_softc;
2932
2933 #ifdef DEV_NETMAP
2934         /* XXX is this necessary ? */
2935         if (ifp->if_capenable & IFCAP_NETMAP) {
2936                 struct netmap_kring *kring = &NA(ifp)->tx_rings[0];
2937                 if (sc->rl_ldata.rl_tx_prodidx != kring->nr_hwcur) {
2938                         /* kick the tx unit */
2939                         CSR_WRITE_1(sc, sc->rl_txstart, RL_TXSTART_START);
2940 #ifdef RE_TX_MODERATION
2941                         CSR_WRITE_4(sc, RL_TIMERCNT, 1);
2942 #endif
2943                         sc->rl_watchdog_timer = 5;
2944                 }
2945                 return;
2946         }
2947 #endif /* DEV_NETMAP */
2948
2949         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
2950             IFF_DRV_RUNNING || (sc->rl_flags & RL_FLAG_LINK) == 0)
2951                 return;
2952
2953         for (queued = 0; !IFQ_DRV_IS_EMPTY(&ifp->if_snd) &&
2954             sc->rl_ldata.rl_tx_free > 1;) {
2955                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
2956                 if (m_head == NULL)
2957                         break;
2958
2959                 if (re_encap(sc, &m_head) != 0) {
2960                         if (m_head == NULL)
2961                                 break;
2962                         IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
2963                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
2964                         break;
2965                 }
2966
2967                 /*
2968                  * If there's a BPF listener, bounce a copy of this frame
2969                  * to him.
2970                  */
2971                 ETHER_BPF_MTAP(ifp, m_head);
2972
2973                 queued++;
2974         }
2975
2976         if (queued == 0) {
2977 #ifdef RE_TX_MODERATION
2978                 if (sc->rl_ldata.rl_tx_free != sc->rl_ldata.rl_tx_desc_cnt)
2979                         CSR_WRITE_4(sc, RL_TIMERCNT, 1);
2980 #endif
2981                 return;
2982         }
2983
2984         /* Flush the TX descriptors */
2985
2986         bus_dmamap_sync(sc->rl_ldata.rl_tx_list_tag,
2987             sc->rl_ldata.rl_tx_list_map,
2988             BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD);
2989
2990         CSR_WRITE_1(sc, sc->rl_txstart, RL_TXSTART_START);
2991
2992 #ifdef RE_TX_MODERATION
2993         /*
2994          * Use the countdown timer for interrupt moderation.
2995          * 'TX done' interrupts are disabled. Instead, we reset the
2996          * countdown timer, which will begin counting until it hits
2997          * the value in the TIMERINT register, and then trigger an
2998          * interrupt. Each time we write to the TIMERCNT register,
2999          * the timer count is reset to 0.
3000          */
3001         CSR_WRITE_4(sc, RL_TIMERCNT, 1);
3002 #endif
3003
3004         /*
3005          * Set a timeout in case the chip goes out to lunch.
3006          */
3007         sc->rl_watchdog_timer = 5;
3008 }
3009
3010 static void
3011 re_set_jumbo(struct rl_softc *sc, int jumbo)
3012 {
3013
3014         if (sc->rl_hwrev->rl_rev == RL_HWREV_8168E_VL) {
3015                 pci_set_max_read_req(sc->rl_dev, 4096);
3016                 return;
3017         }
3018
3019         CSR_WRITE_1(sc, RL_EECMD, RL_EEMODE_WRITECFG);
3020         if (jumbo != 0) {
3021                 CSR_WRITE_1(sc, sc->rl_cfg3, CSR_READ_1(sc, sc->rl_cfg3) |
3022                     RL_CFG3_JUMBO_EN0);
3023                 switch (sc->rl_hwrev->rl_rev) {
3024                 case RL_HWREV_8168DP:
3025                         break;
3026                 case RL_HWREV_8168E:
3027                         CSR_WRITE_1(sc, sc->rl_cfg4,
3028                             CSR_READ_1(sc, sc->rl_cfg4) | 0x01);
3029                         break;
3030                 default:
3031                         CSR_WRITE_1(sc, sc->rl_cfg4,
3032                             CSR_READ_1(sc, sc->rl_cfg4) | RL_CFG4_JUMBO_EN1);
3033                 }
3034         } else {
3035                 CSR_WRITE_1(sc, sc->rl_cfg3, CSR_READ_1(sc, sc->rl_cfg3) &
3036                     ~RL_CFG3_JUMBO_EN0);
3037                 switch (sc->rl_hwrev->rl_rev) {
3038                 case RL_HWREV_8168DP:
3039                         break;
3040                 case RL_HWREV_8168E:
3041                         CSR_WRITE_1(sc, sc->rl_cfg4,
3042                             CSR_READ_1(sc, sc->rl_cfg4) & ~0x01);
3043                         break;
3044                 default:
3045                         CSR_WRITE_1(sc, sc->rl_cfg4,
3046                             CSR_READ_1(sc, sc->rl_cfg4) & ~RL_CFG4_JUMBO_EN1);
3047                 }
3048         }
3049         CSR_WRITE_1(sc, RL_EECMD, RL_EEMODE_OFF);
3050
3051         switch (sc->rl_hwrev->rl_rev) {
3052         case RL_HWREV_8168DP:
3053                 pci_set_max_read_req(sc->rl_dev, 4096);
3054                 break;
3055         default:
3056                 if (jumbo != 0)
3057                         pci_set_max_read_req(sc->rl_dev, 512);
3058                 else
3059                         pci_set_max_read_req(sc->rl_dev, 4096);
3060         }
3061 }
3062
3063 static void
3064 re_init(void *xsc)
3065 {
3066         struct rl_softc         *sc = xsc;
3067
3068         RL_LOCK(sc);
3069         re_init_locked(sc);
3070         RL_UNLOCK(sc);
3071 }
3072
3073 static void
3074 re_init_locked(struct rl_softc *sc)
3075 {
3076         struct ifnet            *ifp = sc->rl_ifp;
3077         struct mii_data         *mii;
3078         uint32_t                reg;
3079         uint16_t                cfg;
3080         union {
3081                 uint32_t align_dummy;
3082                 u_char eaddr[ETHER_ADDR_LEN];
3083         } eaddr;
3084
3085         RL_LOCK_ASSERT(sc);
3086
3087         mii = device_get_softc(sc->rl_miibus);
3088
3089         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0)
3090                 return;
3091
3092         /*
3093          * Cancel pending I/O and free all RX/TX buffers.
3094          */
3095         re_stop(sc);
3096
3097         /* Put controller into known state. */
3098         re_reset(sc);
3099
3100         /*
3101          * For C+ mode, initialize the RX descriptors and mbufs.
3102          */
3103         if ((sc->rl_flags & RL_FLAG_JUMBOV2) != 0) {
3104                 if (ifp->if_mtu > RL_MTU) {
3105                         if (re_jrx_list_init(sc) != 0) {
3106                                 device_printf(sc->rl_dev,
3107                                     "no memory for jumbo RX buffers\n");
3108                                 re_stop(sc);
3109                                 return;
3110                         }
3111                         /* Disable checksum offloading for jumbo frames. */
3112                         ifp->if_capenable &= ~(IFCAP_HWCSUM | IFCAP_TSO4);
3113                         ifp->if_hwassist &= ~(RE_CSUM_FEATURES | CSUM_TSO);
3114                 } else {
3115                         if (re_rx_list_init(sc) != 0) {
3116                                 device_printf(sc->rl_dev,
3117                                     "no memory for RX buffers\n");
3118                                 re_stop(sc);
3119                                 return;
3120                         }
3121                 }
3122                 re_set_jumbo(sc, ifp->if_mtu > RL_MTU);
3123         } else {
3124                 if (re_rx_list_init(sc) != 0) {
3125                         device_printf(sc->rl_dev, "no memory for RX buffers\n");
3126                         re_stop(sc);
3127                         return;
3128                 }
3129                 if ((sc->rl_flags & RL_FLAG_PCIE) != 0 &&
3130                     pci_get_device(sc->rl_dev) != RT_DEVICEID_8101E) {
3131                         if (ifp->if_mtu > RL_MTU)
3132                                 pci_set_max_read_req(sc->rl_dev, 512);
3133                         else
3134                                 pci_set_max_read_req(sc->rl_dev, 4096);
3135                 }
3136         }
3137         re_tx_list_init(sc);
3138
3139         /*
3140          * Enable C+ RX and TX mode, as well as VLAN stripping and
3141          * RX checksum offload. We must configure the C+ register
3142          * before all others.
3143          */
3144         cfg = RL_CPLUSCMD_PCI_MRW;
3145         if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
3146                 cfg |= RL_CPLUSCMD_RXCSUM_ENB;
3147         if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0)
3148                 cfg |= RL_CPLUSCMD_VLANSTRIP;
3149         if ((sc->rl_flags & RL_FLAG_MACSTAT) != 0) {
3150                 cfg |= RL_CPLUSCMD_MACSTAT_DIS;
3151                 /* XXX magic. */
3152                 cfg |= 0x0001;
3153         } else
3154                 cfg |= RL_CPLUSCMD_RXENB | RL_CPLUSCMD_TXENB;
3155         CSR_WRITE_2(sc, RL_CPLUS_CMD, cfg);
3156         if (sc->rl_hwrev->rl_rev == RL_HWREV_8169_8110SC ||
3157             sc->rl_hwrev->rl_rev == RL_HWREV_8169_8110SCE) {
3158                 reg = 0x000fff00;
3159                 if ((CSR_READ_1(sc, sc->rl_cfg2) & RL_CFG2_PCI66MHZ) != 0)
3160                         reg |= 0x000000ff;
3161                 if (sc->rl_hwrev->rl_rev == RL_HWREV_8169_8110SCE)
3162                         reg |= 0x00f00000;
3163                 CSR_WRITE_4(sc, 0x7c, reg);
3164                 /* Disable interrupt mitigation. */
3165                 CSR_WRITE_2(sc, 0xe2, 0);
3166         }
3167         /*
3168          * Disable TSO if interface MTU size is greater than MSS
3169          * allowed in controller.
3170          */
3171         if (ifp->if_mtu > RL_TSO_MTU && (ifp->if_capenable & IFCAP_TSO4) != 0) {
3172                 ifp->if_capenable &= ~IFCAP_TSO4;
3173                 ifp->if_hwassist &= ~CSUM_TSO;
3174         }
3175
3176         /*
3177          * Init our MAC address.  Even though the chipset
3178          * documentation doesn't mention it, we need to enter "Config
3179          * register write enable" mode to modify the ID registers.
3180          */
3181         /* Copy MAC address on stack to align. */
3182         bcopy(IF_LLADDR(ifp), eaddr.eaddr, ETHER_ADDR_LEN);
3183         CSR_WRITE_1(sc, RL_EECMD, RL_EEMODE_WRITECFG);
3184         CSR_WRITE_4(sc, RL_IDR0,
3185             htole32(*(u_int32_t *)(&eaddr.eaddr[0])));
3186         CSR_WRITE_4(sc, RL_IDR4,
3187             htole32(*(u_int32_t *)(&eaddr.eaddr[4])));
3188         CSR_WRITE_1(sc, RL_EECMD, RL_EEMODE_OFF);
3189
3190         /*
3191          * Load the addresses of the RX and TX lists into the chip.
3192          */
3193
3194         CSR_WRITE_4(sc, RL_RXLIST_ADDR_HI,
3195             RL_ADDR_HI(sc->rl_ldata.rl_rx_list_addr));
3196         CSR_WRITE_4(sc, RL_RXLIST_ADDR_LO,
3197             RL_ADDR_LO(sc->rl_ldata.rl_rx_list_addr));
3198
3199         CSR_WRITE_4(sc, RL_TXLIST_ADDR_HI,
3200             RL_ADDR_HI(sc->rl_ldata.rl_tx_list_addr));
3201         CSR_WRITE_4(sc, RL_TXLIST_ADDR_LO,
3202             RL_ADDR_LO(sc->rl_ldata.rl_tx_list_addr));
3203
3204         if ((sc->rl_flags & RL_FLAG_8168G_PLUS) != 0) {
3205                 /* Disable RXDV gate. */
3206                 CSR_WRITE_4(sc, RL_MISC, CSR_READ_4(sc, RL_MISC) &
3207                     ~0x00080000);
3208         }
3209
3210         /*
3211          * Enable transmit and receive for pre-RTL8168G controllers.
3212          * RX/TX MACs should be enabled before RX/TX configuration.
3213          */
3214         if ((sc->rl_flags & RL_FLAG_8168G_PLUS) == 0)
3215                 CSR_WRITE_1(sc, RL_COMMAND, RL_CMD_TX_ENB | RL_CMD_RX_ENB);
3216
3217         /*
3218          * Set the initial TX configuration.
3219          */
3220         if (sc->rl_testmode) {
3221                 if (sc->rl_type == RL_8169)
3222                         CSR_WRITE_4(sc, RL_TXCFG,
3223                             RL_TXCFG_CONFIG|RL_LOOPTEST_ON);
3224                 else
3225                         CSR_WRITE_4(sc, RL_TXCFG,
3226                             RL_TXCFG_CONFIG|RL_LOOPTEST_ON_CPLUS);
3227         } else
3228                 CSR_WRITE_4(sc, RL_TXCFG, RL_TXCFG_CONFIG);
3229
3230         CSR_WRITE_1(sc, RL_EARLY_TX_THRESH, 16);
3231
3232         /*
3233          * Set the initial RX configuration.
3234          */
3235         re_set_rxmode(sc);
3236
3237         /* Configure interrupt moderation. */
3238         if (sc->rl_type == RL_8169) {
3239                 /* Magic from vendor. */
3240                 CSR_WRITE_2(sc, RL_INTRMOD, 0x5100);
3241         }
3242
3243         /*
3244          * Enable transmit and receive for RTL8168G and later controllers.
3245          * RX/TX MACs should be enabled after RX/TX configuration.
3246          */
3247         if ((sc->rl_flags & RL_FLAG_8168G_PLUS) != 0)
3248                 CSR_WRITE_1(sc, RL_COMMAND, RL_CMD_TX_ENB | RL_CMD_RX_ENB);
3249
3250 #ifdef DEVICE_POLLING
3251         /*
3252          * Disable interrupts if we are polling.
3253          */
3254         if (ifp->if_capenable & IFCAP_POLLING)
3255                 CSR_WRITE_2(sc, RL_IMR, 0);
3256         else    /* otherwise ... */
3257 #endif
3258
3259         /*
3260          * Enable interrupts.
3261          */
3262         if (sc->rl_testmode)
3263                 CSR_WRITE_2(sc, RL_IMR, 0);
3264         else
3265                 CSR_WRITE_2(sc, RL_IMR, RL_INTRS_CPLUS);
3266         CSR_WRITE_2(sc, RL_ISR, RL_INTRS_CPLUS);
3267
3268         /* Set initial TX threshold */
3269         sc->rl_txthresh = RL_TX_THRESH_INIT;
3270
3271         /* Start RX/TX process. */
3272         CSR_WRITE_4(sc, RL_MISSEDPKT, 0);
3273
3274         /*
3275          * Initialize the timer interrupt register so that
3276          * a timer interrupt will be generated once the timer
3277          * reaches a certain number of ticks. The timer is
3278          * reloaded on each transmit.
3279          */
3280 #ifdef RE_TX_MODERATION
3281         /*
3282          * Use timer interrupt register to moderate TX interrupt
3283          * moderation, which dramatically improves TX frame rate.
3284          */
3285         if (sc->rl_type == RL_8169)
3286                 CSR_WRITE_4(sc, RL_TIMERINT_8169, 0x800);
3287         else
3288                 CSR_WRITE_4(sc, RL_TIMERINT, 0x400);
3289 #else
3290         /*
3291          * Use timer interrupt register to moderate RX interrupt
3292          * moderation.
3293          */
3294         if ((sc->rl_flags & (RL_FLAG_MSI | RL_FLAG_MSIX)) != 0 &&
3295             intr_filter == 0) {
3296                 if (sc->rl_type == RL_8169)
3297                         CSR_WRITE_4(sc, RL_TIMERINT_8169,
3298                             RL_USECS(sc->rl_int_rx_mod));
3299         } else {
3300                 if (sc->rl_type == RL_8169)
3301                         CSR_WRITE_4(sc, RL_TIMERINT_8169, RL_USECS(0));
3302         }
3303 #endif
3304
3305         /*
3306          * For 8169 gigE NICs, set the max allowed RX packet
3307          * size so we can receive jumbo frames.
3308          */
3309         if (sc->rl_type == RL_8169) {
3310                 if ((sc->rl_flags & RL_FLAG_JUMBOV2) != 0) {
3311                         /*
3312                          * For controllers that use new jumbo frame scheme,
3313                          * set maximum size of jumbo frame depending on
3314                          * controller revisions.
3315                          */
3316                         if (ifp->if_mtu > RL_MTU)
3317                                 CSR_WRITE_2(sc, RL_MAXRXPKTLEN,
3318                                     sc->rl_hwrev->rl_max_mtu +
3319                                     ETHER_VLAN_ENCAP_LEN + ETHER_HDR_LEN +
3320                                     ETHER_CRC_LEN);
3321                         else
3322                                 CSR_WRITE_2(sc, RL_MAXRXPKTLEN,
3323                                     RE_RX_DESC_BUFLEN);
3324                 } else if ((sc->rl_flags & RL_FLAG_PCIE) != 0 &&
3325                     sc->rl_hwrev->rl_max_mtu == RL_MTU) {
3326                         /* RTL810x has no jumbo frame support. */
3327                         CSR_WRITE_2(sc, RL_MAXRXPKTLEN, RE_RX_DESC_BUFLEN);
3328                 } else
3329                         CSR_WRITE_2(sc, RL_MAXRXPKTLEN, 16383);
3330         }
3331
3332         if (sc->rl_testmode)
3333                 return;
3334
3335         CSR_WRITE_1(sc, sc->rl_cfg1, CSR_READ_1(sc, sc->rl_cfg1) |
3336             RL_CFG1_DRVLOAD);
3337
3338         ifp->if_drv_flags |= IFF_DRV_RUNNING;
3339         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
3340
3341         sc->rl_flags &= ~RL_FLAG_LINK;
3342         mii_mediachg(mii);
3343
3344         sc->rl_watchdog_timer = 0;
3345         callout_reset(&sc->rl_stat_callout, hz, re_tick, sc);
3346 }
3347
3348 /*
3349  * Set media options.
3350  */
3351 static int
3352 re_ifmedia_upd(struct ifnet *ifp)
3353 {
3354         struct rl_softc         *sc;
3355         struct mii_data         *mii;
3356         int                     error;
3357
3358         sc = ifp->if_softc;
3359         mii = device_get_softc(sc->rl_miibus);
3360         RL_LOCK(sc);
3361         error = mii_mediachg(mii);
3362         RL_UNLOCK(sc);
3363
3364         return (error);
3365 }
3366
3367 /*
3368  * Report current media status.
3369  */
3370 static void
3371 re_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
3372 {
3373         struct rl_softc         *sc;
3374         struct mii_data         *mii;
3375
3376         sc = ifp->if_softc;
3377         mii = device_get_softc(sc->rl_miibus);
3378
3379         RL_LOCK(sc);
3380         mii_pollstat(mii);
3381         ifmr->ifm_active = mii->mii_media_active;
3382         ifmr->ifm_status = mii->mii_media_status;
3383         RL_UNLOCK(sc);
3384 }
3385
3386 static int
3387 re_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
3388 {
3389         struct rl_softc         *sc = ifp->if_softc;
3390         struct ifreq            *ifr = (struct ifreq *) data;
3391         struct mii_data         *mii;
3392         int                     error = 0;
3393
3394         switch (command) {
3395         case SIOCSIFMTU:
3396                 if (ifr->ifr_mtu < ETHERMIN ||
3397                     ifr->ifr_mtu > sc->rl_hwrev->rl_max_mtu ||
3398                     ((sc->rl_flags & RL_FLAG_FASTETHER) != 0 &&
3399                     ifr->ifr_mtu > RL_MTU)) {
3400                         error = EINVAL;
3401                         break;
3402                 }
3403                 RL_LOCK(sc);
3404                 if (ifp->if_mtu != ifr->ifr_mtu) {
3405                         ifp->if_mtu = ifr->ifr_mtu;
3406                         if ((sc->rl_flags & RL_FLAG_JUMBOV2) != 0 &&
3407                             (ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) {
3408                                 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3409                                 re_init_locked(sc);
3410                         }
3411                         if (ifp->if_mtu > RL_TSO_MTU &&
3412                             (ifp->if_capenable & IFCAP_TSO4) != 0) {
3413                                 ifp->if_capenable &= ~(IFCAP_TSO4 |
3414                                     IFCAP_VLAN_HWTSO);
3415                                 ifp->if_hwassist &= ~CSUM_TSO;
3416                         }
3417                         VLAN_CAPABILITIES(ifp);
3418                 }
3419                 RL_UNLOCK(sc);
3420                 break;
3421         case SIOCSIFFLAGS:
3422                 RL_LOCK(sc);
3423                 if ((ifp->if_flags & IFF_UP) != 0) {
3424                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) {
3425                                 if (((ifp->if_flags ^ sc->rl_if_flags)
3426                                     & (IFF_PROMISC | IFF_ALLMULTI)) != 0)
3427                                         re_set_rxmode(sc);
3428                         } else
3429                                 re_init_locked(sc);
3430                 } else {
3431                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0)
3432                                 re_stop(sc);
3433                 }
3434                 sc->rl_if_flags = ifp->if_flags;
3435                 RL_UNLOCK(sc);
3436                 break;
3437         case SIOCADDMULTI:
3438         case SIOCDELMULTI:
3439                 RL_LOCK(sc);
3440                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0)
3441                         re_set_rxmode(sc);
3442                 RL_UNLOCK(sc);
3443                 break;
3444         case SIOCGIFMEDIA:
3445         case SIOCSIFMEDIA:
3446                 mii = device_get_softc(sc->rl_miibus);
3447                 error = ifmedia_ioctl(ifp, ifr, &mii->mii_media, command);
3448                 break;
3449         case SIOCSIFCAP:
3450             {
3451                 int mask, reinit;
3452
3453                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
3454                 reinit = 0;
3455 #ifdef DEVICE_POLLING
3456                 if (mask & IFCAP_POLLING) {
3457                         if (ifr->ifr_reqcap & IFCAP_POLLING) {
3458                                 error = ether_poll_register(re_poll, ifp);
3459                                 if (error)
3460                                         return (error);
3461                                 RL_LOCK(sc);
3462                                 /* Disable interrupts */
3463                                 CSR_WRITE_2(sc, RL_IMR, 0x0000);
3464                                 ifp->if_capenable |= IFCAP_POLLING;
3465                                 RL_UNLOCK(sc);
3466                         } else {
3467                                 error = ether_poll_deregister(ifp);
3468                                 /* Enable interrupts. */
3469                                 RL_LOCK(sc);
3470                                 CSR_WRITE_2(sc, RL_IMR, RL_INTRS_CPLUS);
3471                                 ifp->if_capenable &= ~IFCAP_POLLING;
3472                                 RL_UNLOCK(sc);
3473                         }
3474                 }
3475 #endif /* DEVICE_POLLING */
3476                 RL_LOCK(sc);
3477                 if ((mask & IFCAP_TXCSUM) != 0 &&
3478                     (ifp->if_capabilities & IFCAP_TXCSUM) != 0) {
3479                         ifp->if_capenable ^= IFCAP_TXCSUM;
3480                         if ((ifp->if_capenable & IFCAP_TXCSUM) != 0)
3481                                 ifp->if_hwassist |= RE_CSUM_FEATURES;
3482                         else
3483                                 ifp->if_hwassist &= ~RE_CSUM_FEATURES;
3484                         reinit = 1;
3485                 }
3486                 if ((mask & IFCAP_RXCSUM) != 0 &&
3487                     (ifp->if_capabilities & IFCAP_RXCSUM) != 0) {
3488                         ifp->if_capenable ^= IFCAP_RXCSUM;
3489                         reinit = 1;
3490                 }
3491                 if ((mask & IFCAP_TSO4) != 0 &&
3492                     (ifp->if_capabilities & IFCAP_TSO4) != 0) {
3493                         ifp->if_capenable ^= IFCAP_TSO4;
3494                         if ((IFCAP_TSO4 & ifp->if_capenable) != 0)
3495                                 ifp->if_hwassist |= CSUM_TSO;
3496                         else
3497                                 ifp->if_hwassist &= ~CSUM_TSO;
3498                         if (ifp->if_mtu > RL_TSO_MTU &&
3499                             (ifp->if_capenable & IFCAP_TSO4) != 0) {
3500                                 ifp->if_capenable &= ~IFCAP_TSO4;
3501                                 ifp->if_hwassist &= ~CSUM_TSO;
3502                         }
3503                 }
3504                 if ((mask & IFCAP_VLAN_HWTSO) != 0 &&
3505                     (ifp->if_capabilities & IFCAP_VLAN_HWTSO) != 0)
3506                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
3507                 if ((mask & IFCAP_VLAN_HWTAGGING) != 0 &&
3508                     (ifp->if_capabilities & IFCAP_VLAN_HWTAGGING) != 0) {
3509                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
3510                         /* TSO over VLAN requires VLAN hardware tagging. */
3511                         if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) == 0)
3512                                 ifp->if_capenable &= ~IFCAP_VLAN_HWTSO;
3513                         reinit = 1;
3514                 }
3515                 if ((sc->rl_flags & RL_FLAG_JUMBOV2) != 0 &&
3516                     (mask & (IFCAP_HWCSUM | IFCAP_TSO4 |
3517                     IFCAP_VLAN_HWTSO)) != 0)
3518                                 reinit = 1;
3519                 if ((mask & IFCAP_WOL) != 0 &&
3520                     (ifp->if_capabilities & IFCAP_WOL) != 0) {
3521                         if ((mask & IFCAP_WOL_UCAST) != 0)
3522                                 ifp->if_capenable ^= IFCAP_WOL_UCAST;
3523                         if ((mask & IFCAP_WOL_MCAST) != 0)
3524                                 ifp->if_capenable ^= IFCAP_WOL_MCAST;
3525                         if ((mask & IFCAP_WOL_MAGIC) != 0)
3526                                 ifp->if_capenable ^= IFCAP_WOL_MAGIC;
3527                 }
3528                 if (reinit && ifp->if_drv_flags & IFF_DRV_RUNNING) {
3529                         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3530                         re_init_locked(sc);
3531                 }
3532                 RL_UNLOCK(sc);
3533                 VLAN_CAPABILITIES(ifp);
3534             }
3535                 break;
3536         default:
3537                 error = ether_ioctl(ifp, command, data);
3538                 break;
3539         }
3540
3541         return (error);
3542 }
3543
3544 static void
3545 re_watchdog(struct rl_softc *sc)
3546 {
3547         struct ifnet            *ifp;
3548
3549         RL_LOCK_ASSERT(sc);
3550
3551         if (sc->rl_watchdog_timer == 0 || --sc->rl_watchdog_timer != 0)
3552                 return;
3553
3554         ifp = sc->rl_ifp;
3555         re_txeof(sc);
3556         if (sc->rl_ldata.rl_tx_free == sc->rl_ldata.rl_tx_desc_cnt) {
3557                 if_printf(ifp, "watchdog timeout (missed Tx interrupts) "
3558                     "-- recovering\n");
3559                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
3560                         re_start_locked(ifp);
3561                 return;
3562         }
3563
3564         if_printf(ifp, "watchdog timeout\n");
3565         if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
3566
3567         re_rxeof(sc, NULL);
3568         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3569         re_init_locked(sc);
3570         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
3571                 re_start_locked(ifp);
3572 }
3573
3574 /*
3575  * Stop the adapter and free any mbufs allocated to the
3576  * RX and TX lists.
3577  */
3578 static void
3579 re_stop(struct rl_softc *sc)
3580 {
3581         int                     i;
3582         struct ifnet            *ifp;
3583         struct rl_txdesc        *txd;
3584         struct rl_rxdesc        *rxd;
3585
3586         RL_LOCK_ASSERT(sc);
3587
3588         ifp = sc->rl_ifp;
3589
3590         sc->rl_watchdog_timer = 0;
3591         callout_stop(&sc->rl_stat_callout);
3592         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
3593
3594         /*
3595          * Disable accepting frames to put RX MAC into idle state.
3596          * Otherwise it's possible to get frames while stop command
3597          * execution is in progress and controller can DMA the frame
3598          * to already freed RX buffer during that period.
3599          */
3600         CSR_WRITE_4(sc, RL_RXCFG, CSR_READ_4(sc, RL_RXCFG) &
3601             ~(RL_RXCFG_RX_ALLPHYS | RL_RXCFG_RX_INDIV | RL_RXCFG_RX_MULTI |
3602             RL_RXCFG_RX_BROAD));
3603
3604         if ((sc->rl_flags & RL_FLAG_8168G_PLUS) != 0) {
3605                 /* Enable RXDV gate. */
3606                 CSR_WRITE_4(sc, RL_MISC, CSR_READ_4(sc, RL_MISC) |
3607                     0x00080000);
3608         }
3609
3610         if ((sc->rl_flags & RL_FLAG_WAIT_TXPOLL) != 0) {
3611                 for (i = RL_TIMEOUT; i > 0; i--) {
3612                         if ((CSR_READ_1(sc, sc->rl_txstart) &
3613                             RL_TXSTART_START) == 0)
3614                                 break;
3615                         DELAY(20);
3616                 }
3617                 if (i == 0)
3618                         device_printf(sc->rl_dev,
3619                             "stopping TX poll timed out!\n");
3620                 CSR_WRITE_1(sc, RL_COMMAND, 0x00);
3621         } else if ((sc->rl_flags & RL_FLAG_CMDSTOP) != 0) {
3622                 CSR_WRITE_1(sc, RL_COMMAND, RL_CMD_STOPREQ | RL_CMD_TX_ENB |
3623                     RL_CMD_RX_ENB);
3624                 if ((sc->rl_flags & RL_FLAG_CMDSTOP_WAIT_TXQ) != 0) {
3625                         for (i = RL_TIMEOUT; i > 0; i--) {
3626                                 if ((CSR_READ_4(sc, RL_TXCFG) &
3627                                     RL_TXCFG_QUEUE_EMPTY) != 0)
3628                                         break;
3629                                 DELAY(100);
3630                         }
3631                         if (i == 0)
3632                                 device_printf(sc->rl_dev,
3633                                    "stopping TXQ timed out!\n");
3634                 }
3635         } else
3636                 CSR_WRITE_1(sc, RL_COMMAND, 0x00);
3637         DELAY(1000);
3638         CSR_WRITE_2(sc, RL_IMR, 0x0000);
3639         CSR_WRITE_2(sc, RL_ISR, 0xFFFF);
3640
3641         if (sc->rl_head != NULL) {
3642                 m_freem(sc->rl_head);
3643                 sc->rl_head = sc->rl_tail = NULL;
3644         }
3645
3646         /* Free the TX list buffers. */
3647         for (i = 0; i < sc->rl_ldata.rl_tx_desc_cnt; i++) {
3648                 txd = &sc->rl_ldata.rl_tx_desc[i];
3649                 if (txd->tx_m != NULL) {
3650                         bus_dmamap_sync(sc->rl_ldata.rl_tx_mtag,
3651                             txd->tx_dmamap, BUS_DMASYNC_POSTWRITE);
3652                         bus_dmamap_unload(sc->rl_ldata.rl_tx_mtag,
3653                             txd->tx_dmamap);
3654                         m_freem(txd->tx_m);
3655                         txd->tx_m = NULL;
3656                 }
3657         }
3658
3659         /* Free the RX list buffers. */
3660         for (i = 0; i < sc->rl_ldata.rl_rx_desc_cnt; i++) {
3661                 rxd = &sc->rl_ldata.rl_rx_desc[i];
3662                 if (rxd->rx_m != NULL) {
3663                         bus_dmamap_sync(sc->rl_ldata.rl_rx_mtag,
3664                             rxd->rx_dmamap, BUS_DMASYNC_POSTREAD);
3665                         bus_dmamap_unload(sc->rl_ldata.rl_rx_mtag,
3666                             rxd->rx_dmamap);
3667                         m_freem(rxd->rx_m);
3668                         rxd->rx_m = NULL;
3669                 }
3670         }
3671
3672         if ((sc->rl_flags & RL_FLAG_JUMBOV2) != 0) {
3673                 for (i = 0; i < sc->rl_ldata.rl_rx_desc_cnt; i++) {
3674                         rxd = &sc->rl_ldata.rl_jrx_desc[i];
3675                         if (rxd->rx_m != NULL) {
3676                                 bus_dmamap_sync(sc->rl_ldata.rl_jrx_mtag,
3677                                     rxd->rx_dmamap, BUS_DMASYNC_POSTREAD);
3678                                 bus_dmamap_unload(sc->rl_ldata.rl_jrx_mtag,
3679                                     rxd->rx_dmamap);
3680                                 m_freem(rxd->rx_m);
3681                                 rxd->rx_m = NULL;
3682                         }
3683                 }
3684         }
3685 }
3686
3687 /*
3688  * Device suspend routine.  Stop the interface and save some PCI
3689  * settings in case the BIOS doesn't restore them properly on
3690  * resume.
3691  */
3692 static int
3693 re_suspend(device_t dev)
3694 {
3695         struct rl_softc         *sc;
3696
3697         sc = device_get_softc(dev);
3698
3699         RL_LOCK(sc);
3700         re_stop(sc);
3701         re_setwol(sc);
3702         sc->suspended = 1;
3703         RL_UNLOCK(sc);
3704
3705         return (0);
3706 }
3707
3708 /*
3709  * Device resume routine.  Restore some PCI settings in case the BIOS
3710  * doesn't, re-enable busmastering, and restart the interface if
3711  * appropriate.
3712  */
3713 static int
3714 re_resume(device_t dev)
3715 {
3716         struct rl_softc         *sc;
3717         struct ifnet            *ifp;
3718
3719         sc = device_get_softc(dev);
3720
3721         RL_LOCK(sc);
3722
3723         ifp = sc->rl_ifp;
3724         /* Take controller out of sleep mode. */
3725         if ((sc->rl_flags & RL_FLAG_MACSLEEP) != 0) {
3726                 if ((CSR_READ_1(sc, RL_MACDBG) & 0x80) == 0x80)
3727                         CSR_WRITE_1(sc, RL_GPIO,
3728                             CSR_READ_1(sc, RL_GPIO) | 0x01);
3729         }
3730
3731         /*
3732          * Clear WOL matching such that normal Rx filtering
3733          * wouldn't interfere with WOL patterns.
3734          */
3735         re_clrwol(sc);
3736
3737         /* reinitialize interface if necessary */
3738         if (ifp->if_flags & IFF_UP)
3739                 re_init_locked(sc);
3740
3741         sc->suspended = 0;
3742         RL_UNLOCK(sc);
3743
3744         return (0);
3745 }
3746
3747 /*
3748  * Stop all chip I/O so that the kernel's probe routines don't
3749  * get confused by errant DMAs when rebooting.
3750  */
3751 static int
3752 re_shutdown(device_t dev)
3753 {
3754         struct rl_softc         *sc;
3755
3756         sc = device_get_softc(dev);
3757
3758         RL_LOCK(sc);
3759         re_stop(sc);
3760         /*
3761          * Mark interface as down since otherwise we will panic if
3762          * interrupt comes in later on, which can happen in some
3763          * cases.
3764          */
3765         sc->rl_ifp->if_flags &= ~IFF_UP;
3766         re_setwol(sc);
3767         RL_UNLOCK(sc);
3768
3769         return (0);
3770 }
3771
3772 static void
3773 re_set_linkspeed(struct rl_softc *sc)
3774 {
3775         struct mii_softc *miisc;
3776         struct mii_data *mii;
3777         int aneg, i, phyno;
3778
3779         RL_LOCK_ASSERT(sc);
3780
3781         mii = device_get_softc(sc->rl_miibus);
3782         mii_pollstat(mii);
3783         aneg = 0;
3784         if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) ==
3785             (IFM_ACTIVE | IFM_AVALID)) {
3786                 switch IFM_SUBTYPE(mii->mii_media_active) {
3787                 case IFM_10_T:
3788                 case IFM_100_TX:
3789                         return;
3790                 case IFM_1000_T:
3791                         aneg++;
3792                         break;
3793                 default:
3794                         break;
3795                 }
3796         }
3797         miisc = LIST_FIRST(&mii->mii_phys);
3798         phyno = miisc->mii_phy;
3799         LIST_FOREACH(miisc, &mii->mii_phys, mii_list)
3800                 PHY_RESET(miisc);
3801         re_miibus_writereg(sc->rl_dev, phyno, MII_100T2CR, 0);
3802         re_miibus_writereg(sc->rl_dev, phyno,
3803             MII_ANAR, ANAR_TX_FD | ANAR_TX | ANAR_10_FD | ANAR_10 | ANAR_CSMA);
3804         re_miibus_writereg(sc->rl_dev, phyno,
3805             MII_BMCR, BMCR_AUTOEN | BMCR_STARTNEG);
3806         DELAY(1000);
3807         if (aneg != 0) {
3808                 /*
3809                  * Poll link state until re(4) get a 10/100Mbps link.
3810                  */
3811                 for (i = 0; i < MII_ANEGTICKS_GIGE; i++) {
3812                         mii_pollstat(mii);
3813                         if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID))
3814                             == (IFM_ACTIVE | IFM_AVALID)) {
3815                                 switch (IFM_SUBTYPE(mii->mii_media_active)) {
3816                                 case IFM_10_T:
3817                                 case IFM_100_TX:
3818                                         return;
3819                                 default:
3820                                         break;
3821                                 }
3822                         }
3823                         RL_UNLOCK(sc);
3824                         pause("relnk", hz);
3825                         RL_LOCK(sc);
3826                 }
3827                 if (i == MII_ANEGTICKS_GIGE)
3828                         device_printf(sc->rl_dev,
3829                             "establishing a link failed, WOL may not work!");
3830         }
3831         /*
3832          * No link, force MAC to have 100Mbps, full-duplex link.
3833          * MAC does not require reprogramming on resolved speed/duplex,
3834          * so this is just for completeness.
3835          */
3836         mii->mii_media_status = IFM_AVALID | IFM_ACTIVE;
3837         mii->mii_media_active = IFM_ETHER | IFM_100_TX | IFM_FDX;
3838 }
3839
3840 static void
3841 re_setwol(struct rl_softc *sc)
3842 {
3843         struct ifnet            *ifp;
3844         int                     pmc;
3845         uint16_t                pmstat;
3846         uint8_t                 v;
3847
3848         RL_LOCK_ASSERT(sc);
3849
3850         if (pci_find_cap(sc->rl_dev, PCIY_PMG, &pmc) != 0)
3851                 return;
3852
3853         ifp = sc->rl_ifp;
3854         /* Put controller into sleep mode. */
3855         if ((sc->rl_flags & RL_FLAG_MACSLEEP) != 0) {
3856                 if ((CSR_READ_1(sc, RL_MACDBG) & 0x80) == 0x80)
3857                         CSR_WRITE_1(sc, RL_GPIO,
3858                             CSR_READ_1(sc, RL_GPIO) & ~0x01);
3859         }
3860         if ((ifp->if_capenable & IFCAP_WOL) != 0) {
3861                 if ((sc->rl_flags & RL_FLAG_8168G_PLUS) != 0) {
3862                         /* Disable RXDV gate. */
3863                         CSR_WRITE_4(sc, RL_MISC, CSR_READ_4(sc, RL_MISC) &
3864                             ~0x00080000);
3865                 }
3866                 re_set_rxmode(sc);
3867                 if ((sc->rl_flags & RL_FLAG_WOL_MANLINK) != 0)
3868                         re_set_linkspeed(sc);
3869                 if ((sc->rl_flags & RL_FLAG_WOLRXENB) != 0)
3870                         CSR_WRITE_1(sc, RL_COMMAND, RL_CMD_RX_ENB);
3871         }
3872         /* Enable config register write. */
3873         CSR_WRITE_1(sc, RL_EECMD, RL_EE_MODE);
3874
3875         /* Enable PME. */
3876         v = CSR_READ_1(sc, sc->rl_cfg1);
3877         v &= ~RL_CFG1_PME;
3878         if ((ifp->if_capenable & IFCAP_WOL) != 0)
3879                 v |= RL_CFG1_PME;
3880         CSR_WRITE_1(sc, sc->rl_cfg1, v);
3881
3882         v = CSR_READ_1(sc, sc->rl_cfg3);
3883         v &= ~(RL_CFG3_WOL_LINK | RL_CFG3_WOL_MAGIC);
3884         if ((ifp->if_capenable & IFCAP_WOL_MAGIC) != 0)
3885                 v |= RL_CFG3_WOL_MAGIC;
3886         CSR_WRITE_1(sc, sc->rl_cfg3, v);
3887
3888         v = CSR_READ_1(sc, sc->rl_cfg5);
3889         v &= ~(RL_CFG5_WOL_BCAST | RL_CFG5_WOL_MCAST | RL_CFG5_WOL_UCAST |
3890             RL_CFG5_WOL_LANWAKE);
3891         if ((ifp->if_capenable & IFCAP_WOL_UCAST) != 0)
3892                 v |= RL_CFG5_WOL_UCAST;
3893         if ((ifp->if_capenable & IFCAP_WOL_MCAST) != 0)
3894                 v |= RL_CFG5_WOL_MCAST | RL_CFG5_WOL_BCAST;
3895         if ((ifp->if_capenable & IFCAP_WOL) != 0)
3896                 v |= RL_CFG5_WOL_LANWAKE;
3897         CSR_WRITE_1(sc, sc->rl_cfg5, v);
3898
3899         /* Config register write done. */
3900         CSR_WRITE_1(sc, RL_EECMD, RL_EEMODE_OFF);
3901
3902         if ((ifp->if_capenable & IFCAP_WOL) == 0 &&
3903             (sc->rl_flags & RL_FLAG_PHYWAKE_PM) != 0)
3904                 CSR_WRITE_1(sc, RL_PMCH, CSR_READ_1(sc, RL_PMCH) & ~0x80);
3905         /*
3906          * It seems that hardware resets its link speed to 100Mbps in
3907          * power down mode so switching to 100Mbps in driver is not
3908          * needed.
3909          */
3910
3911         /* Request PME if WOL is requested. */
3912         pmstat = pci_read_config(sc->rl_dev, pmc + PCIR_POWER_STATUS, 2);
3913         pmstat &= ~(PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE);
3914         if ((ifp->if_capenable & IFCAP_WOL) != 0)
3915                 pmstat |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
3916         pci_write_config(sc->rl_dev, pmc + PCIR_POWER_STATUS, pmstat, 2);
3917 }
3918
3919 static void
3920 re_clrwol(struct rl_softc *sc)
3921 {
3922         int                     pmc;
3923         uint8_t                 v;
3924
3925         RL_LOCK_ASSERT(sc);
3926
3927         if (pci_find_cap(sc->rl_dev, PCIY_PMG, &pmc) != 0)
3928                 return;
3929
3930         /* Enable config register write. */
3931         CSR_WRITE_1(sc, RL_EECMD, RL_EE_MODE);
3932
3933         v = CSR_READ_1(sc, sc->rl_cfg3);
3934         v &= ~(RL_CFG3_WOL_LINK | RL_CFG3_WOL_MAGIC);
3935         CSR_WRITE_1(sc, sc->rl_cfg3, v);
3936
3937         /* Config register write done. */
3938         CSR_WRITE_1(sc, RL_EECMD, RL_EEMODE_OFF);
3939
3940         v = CSR_READ_1(sc, sc->rl_cfg5);
3941         v &= ~(RL_CFG5_WOL_BCAST | RL_CFG5_WOL_MCAST | RL_CFG5_WOL_UCAST);
3942         v &= ~RL_CFG5_WOL_LANWAKE;
3943         CSR_WRITE_1(sc, sc->rl_cfg5, v);
3944 }
3945
3946 static void
3947 re_add_sysctls(struct rl_softc *sc)
3948 {
3949         struct sysctl_ctx_list  *ctx;
3950         struct sysctl_oid_list  *children;
3951         int                     error;
3952
3953         ctx = device_get_sysctl_ctx(sc->rl_dev);
3954         children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->rl_dev));
3955
3956         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "stats",
3957             CTLTYPE_INT | CTLFLAG_RW, sc, 0, re_sysctl_stats, "I",
3958             "Statistics Information");
3959         if ((sc->rl_flags & (RL_FLAG_MSI | RL_FLAG_MSIX)) == 0)
3960                 return;
3961
3962         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "int_rx_mod",
3963             CTLTYPE_INT | CTLFLAG_RW, &sc->rl_int_rx_mod, 0,
3964             sysctl_hw_re_int_mod, "I", "re RX interrupt moderation");
3965         /* Pull in device tunables. */
3966         sc->rl_int_rx_mod = RL_TIMER_DEFAULT;
3967         error = resource_int_value(device_get_name(sc->rl_dev),
3968             device_get_unit(sc->rl_dev), "int_rx_mod", &sc->rl_int_rx_mod);
3969         if (error == 0) {
3970                 if (sc->rl_int_rx_mod < RL_TIMER_MIN ||
3971                     sc->rl_int_rx_mod > RL_TIMER_MAX) {
3972                         device_printf(sc->rl_dev, "int_rx_mod value out of "
3973                             "range; using default: %d\n",
3974                             RL_TIMER_DEFAULT);
3975                         sc->rl_int_rx_mod = RL_TIMER_DEFAULT;
3976                 }
3977         }
3978 }
3979
3980 static int
3981 re_sysctl_stats(SYSCTL_HANDLER_ARGS)
3982 {
3983         struct rl_softc         *sc;
3984         struct rl_stats         *stats;
3985         int                     error, i, result;
3986
3987         result = -1;
3988         error = sysctl_handle_int(oidp, &result, 0, req);
3989         if (error || req->newptr == NULL)
3990                 return (error);
3991
3992         if (result == 1) {
3993                 sc = (struct rl_softc *)arg1;
3994                 RL_LOCK(sc);
3995                 if ((sc->rl_ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
3996                         RL_UNLOCK(sc);
3997                         goto done;
3998                 }
3999                 bus_dmamap_sync(sc->rl_ldata.rl_stag,
4000                     sc->rl_ldata.rl_smap, BUS_DMASYNC_PREREAD);
4001                 CSR_WRITE_4(sc, RL_DUMPSTATS_HI,
4002                     RL_ADDR_HI(sc->rl_ldata.rl_stats_addr));
4003                 CSR_WRITE_4(sc, RL_DUMPSTATS_LO,
4004                     RL_ADDR_LO(sc->rl_ldata.rl_stats_addr));
4005                 CSR_WRITE_4(sc, RL_DUMPSTATS_LO,
4006                     RL_ADDR_LO(sc->rl_ldata.rl_stats_addr |
4007                     RL_DUMPSTATS_START));
4008                 for (i = RL_TIMEOUT; i > 0; i--) {
4009                         if ((CSR_READ_4(sc, RL_DUMPSTATS_LO) &
4010                             RL_DUMPSTATS_START) == 0)
4011                                 break;
4012                         DELAY(1000);
4013                 }
4014                 bus_dmamap_sync(sc->rl_ldata.rl_stag,
4015                     sc->rl_ldata.rl_smap, BUS_DMASYNC_POSTREAD);
4016                 RL_UNLOCK(sc);
4017                 if (i == 0) {
4018                         device_printf(sc->rl_dev,
4019                             "DUMP statistics request timed out\n");
4020                         return (ETIMEDOUT);
4021                 }
4022 done:
4023                 stats = sc->rl_ldata.rl_stats;
4024                 printf("%s statistics:\n", device_get_nameunit(sc->rl_dev));
4025                 printf("Tx frames : %ju\n",
4026                     (uintmax_t)le64toh(stats->rl_tx_pkts));
4027                 printf("Rx frames : %ju\n",
4028                     (uintmax_t)le64toh(stats->rl_rx_pkts));
4029                 printf("Tx errors : %ju\n",
4030                     (uintmax_t)le64toh(stats->rl_tx_errs));
4031                 printf("Rx errors : %u\n",
4032                     le32toh(stats->rl_rx_errs));
4033                 printf("Rx missed frames : %u\n",
4034                     (uint32_t)le16toh(stats->rl_missed_pkts));
4035                 printf("Rx frame alignment errs : %u\n",
4036                     (uint32_t)le16toh(stats->rl_rx_framealign_errs));
4037                 printf("Tx single collisions : %u\n",
4038                     le32toh(stats->rl_tx_onecoll));
4039                 printf("Tx multiple collisions : %u\n",
4040                     le32toh(stats->rl_tx_multicolls));
4041                 printf("Rx unicast frames : %ju\n",
4042                     (uintmax_t)le64toh(stats->rl_rx_ucasts));
4043                 printf("Rx broadcast frames : %ju\n",
4044                     (uintmax_t)le64toh(stats->rl_rx_bcasts));
4045                 printf("Rx multicast frames : %u\n",
4046                     le32toh(stats->rl_rx_mcasts));
4047                 printf("Tx aborts : %u\n",
4048                     (uint32_t)le16toh(stats->rl_tx_aborts));
4049                 printf("Tx underruns : %u\n",
4050                     (uint32_t)le16toh(stats->rl_rx_underruns));
4051         }
4052
4053         return (error);
4054 }
4055
4056 static int
4057 sysctl_int_range(SYSCTL_HANDLER_ARGS, int low, int high)
4058 {
4059         int error, value;
4060
4061         if (arg1 == NULL)
4062                 return (EINVAL);
4063         value = *(int *)arg1;
4064         error = sysctl_handle_int(oidp, &value, 0, req);
4065         if (error || req->newptr == NULL)
4066                 return (error);
4067         if (value < low || value > high)
4068                 return (EINVAL);
4069         *(int *)arg1 = value;
4070
4071         return (0);
4072 }
4073
4074 static int
4075 sysctl_hw_re_int_mod(SYSCTL_HANDLER_ARGS)
4076 {
4077
4078         return (sysctl_int_range(oidp, arg1, arg2, req, RL_TIMER_MIN,
4079             RL_TIMER_MAX));
4080 }