sys/dev/cadence/if_cgem.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2012-2014 Thomas Skibo <thomasskibo@yahoo.com>
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28
  29 /*
  30  * A network interface driver for Cadence GEM Gigabit Ethernet
  31  * interface such as the one used in Xilinx Zynq-7000 SoC.
  32  *
  33  * Reference: Zynq-7000 All Programmable SoC Technical Reference Manual.
  34  * (v1.4) November 16, 2012.  Xilinx doc UG585.  GEM is covered in Ch. 16
  35  * and register definitions are in appendix B.18.
  36  */
  37
  38 #include <sys/cdefs.h>
  39 __FBSDID("$FreeBSD$");
  40
  41 #include <sys/param.h>
  42 #include <sys/systm.h>
  43 #include <sys/bus.h>
  44 #include <sys/kernel.h>
  45 #include <sys/malloc.h>
  46 #include <sys/mbuf.h>
  47 #include <sys/module.h>
  48 #include <sys/rman.h>
  49 #include <sys/socket.h>
  50 #include <sys/sockio.h>
  51 #include <sys/sysctl.h>
  52
  53 #include <machine/bus.h>
  54
  55 #include <net/ethernet.h>
  56 #include <net/if.h>
  57 #include <net/if_arp.h>
  58 #include <net/if_dl.h>
  59 #include <net/if_media.h>
  60 #include <net/if_mib.h>
  61 #include <net/if_types.h>
  62
  63 #ifdef INET
  64 #include <netinet/in.h>
  65 #include <netinet/in_systm.h>
  66 #include <netinet/in_var.h>
  67 #include <netinet/ip.h>
  68 #endif
  69
  70 #include <net/bpf.h>
  71 #include <net/bpfdesc.h>
  72
  73 #include <dev/fdt/fdt_common.h>
  74 #include <dev/ofw/ofw_bus.h>
  75 #include <dev/ofw/ofw_bus_subr.h>
  76
  77 #include <dev/mii/mii.h>
  78 #include <dev/mii/miivar.h>
  79
  80 #include <dev/cadence/if_cgem_hw.h>
  81
  82 #include "miibus_if.h"
  83
  84 #define IF_CGEM_NAME "cgem"
  85
  86 #define CGEM_NUM_RX_DESCS       512     /* size of receive descriptor ring */
  87 #define CGEM_NUM_TX_DESCS       512     /* size of transmit descriptor ring */
  88
  89 #define MAX_DESC_RING_SIZE (MAX(CGEM_NUM_RX_DESCS*sizeof(struct cgem_rx_desc),\
  90                                 CGEM_NUM_TX_DESCS*sizeof(struct cgem_tx_desc)))
  91
  92 /* Default for sysctl rxbufs.  Must be < CGEM_NUM_RX_DESCS of course. */
  93 #define DEFAULT_NUM_RX_BUFS     256     /* number of receive bufs to queue. */
  94
  95 #define TX_MAX_DMA_SEGS         8       /* maximum segs in a tx mbuf dma */
  96
  97 #define CGEM_CKSUM_ASSIST       (CSUM_IP | CSUM_TCP | CSUM_UDP | \
  98                                  CSUM_TCP_IPV6 | CSUM_UDP_IPV6)
  99
 100 static struct ofw_compat_data compat_data[] = {
 101         { "cadence,gem",                1 },
 102         { "cdns,macb",                  1 },
 103         { "sifive,fu540-c000-gem",      1 },
 104         { NULL,                         0 },
 105 };
 106
 107 struct cgem_softc {
 108         if_t                    ifp;
 109         struct mtx              sc_mtx;
 110         device_t                dev;
 111         device_t                miibus;
 112         u_int                   mii_media_active;       /* last active media */
 113         int                     if_old_flags;
 114         struct resource         *mem_res;
 115         struct resource         *irq_res;
 116         void                    *intrhand;
 117         struct callout          tick_ch;
 118         uint32_t                net_ctl_shadow;
 119         int                     ref_clk_num;
 120         u_char                  eaddr[6];
 121
 122         bus_dma_tag_t           desc_dma_tag;
 123         bus_dma_tag_t           mbuf_dma_tag;
 124
 125         /* receive descriptor ring */
 126         struct cgem_rx_desc     *rxring;
 127         bus_addr_t              rxring_physaddr;
 128         struct mbuf             *rxring_m[CGEM_NUM_RX_DESCS];
 129         bus_dmamap_t            rxring_m_dmamap[CGEM_NUM_RX_DESCS];
 130         int                     rxring_hd_ptr;  /* where to put rcv bufs */
 131         int                     rxring_tl_ptr;  /* where to get receives */
 132         int                     rxring_queued;  /* how many rcv bufs queued */
 133         bus_dmamap_t            rxring_dma_map;
 134         int                     rxbufs;         /* tunable number rcv bufs */
 135         int                     rxhangwar;      /* rx hang work-around */
 136         u_int                   rxoverruns;     /* rx overruns */
 137         u_int                   rxnobufs;       /* rx buf ring empty events */
 138         u_int                   rxdmamapfails;  /* rx dmamap failures */
 139         uint32_t                rx_frames_prev;
 140
 141         /* transmit descriptor ring */
 142         struct cgem_tx_desc     *txring;
 143         bus_addr_t              txring_physaddr;
 144         struct mbuf             *txring_m[CGEM_NUM_TX_DESCS];
 145         bus_dmamap_t            txring_m_dmamap[CGEM_NUM_TX_DESCS];
 146         int                     txring_hd_ptr;  /* where to put next xmits */
 147         int                     txring_tl_ptr;  /* next xmit mbuf to free */
 148         int                     txring_queued;  /* num xmits segs queued */
 149         bus_dmamap_t            txring_dma_map;
 150         u_int                   txfull;         /* tx ring full events */
 151         u_int                   txdefrags;      /* tx calls to m_defrag() */
 152         u_int                   txdefragfails;  /* tx m_defrag() failures */
 153         u_int                   txdmamapfails;  /* tx dmamap failures */
 154
 155         /* hardware provided statistics */
 156         struct cgem_hw_stats {
 157                 uint64_t                tx_bytes;
 158                 uint32_t                tx_frames;
 159                 uint32_t                tx_frames_bcast;
 160                 uint32_t                tx_frames_multi;
 161                 uint32_t                tx_frames_pause;
 162                 uint32_t                tx_frames_64b;
 163                 uint32_t                tx_frames_65to127b;
 164                 uint32_t                tx_frames_128to255b;
 165                 uint32_t                tx_frames_256to511b;
 166                 uint32_t                tx_frames_512to1023b;
 167                 uint32_t                tx_frames_1024to1536b;
 168                 uint32_t                tx_under_runs;
 169                 uint32_t                tx_single_collisn;
 170                 uint32_t                tx_multi_collisn;
 171                 uint32_t                tx_excsv_collisn;
 172                 uint32_t                tx_late_collisn;
 173                 uint32_t                tx_deferred_frames;
 174                 uint32_t                tx_carrier_sense_errs;
 175
 176                 uint64_t                rx_bytes;
 177                 uint32_t                rx_frames;
 178                 uint32_t                rx_frames_bcast;
 179                 uint32_t                rx_frames_multi;
 180                 uint32_t                rx_frames_pause;
 181                 uint32_t                rx_frames_64b;
 182                 uint32_t                rx_frames_65to127b;
 183                 uint32_t                rx_frames_128to255b;
 184                 uint32_t                rx_frames_256to511b;
 185                 uint32_t                rx_frames_512to1023b;
 186                 uint32_t                rx_frames_1024to1536b;
 187                 uint32_t                rx_frames_undersize;
 188                 uint32_t                rx_frames_oversize;
 189                 uint32_t                rx_frames_jabber;
 190                 uint32_t                rx_frames_fcs_errs;
 191                 uint32_t                rx_frames_length_errs;
 192                 uint32_t                rx_symbol_errs;
 193                 uint32_t                rx_align_errs;
 194                 uint32_t                rx_resource_errs;
 195                 uint32_t                rx_overrun_errs;
 196                 uint32_t                rx_ip_hdr_csum_errs;
 197                 uint32_t                rx_tcp_csum_errs;
 198                 uint32_t                rx_udp_csum_errs;
 199         } stats;
 200 };
 201
 202 #define RD4(sc, off)            (bus_read_4((sc)->mem_res, (off)))
 203 #define WR4(sc, off, val)       (bus_write_4((sc)->mem_res, (off), (val)))
 204 #define BARRIER(sc, off, len, flags) \
 205         (bus_barrier((sc)->mem_res, (off), (len), (flags))
 206
 207 #define CGEM_LOCK(sc)           mtx_lock(&(sc)->sc_mtx)
 208 #define CGEM_UNLOCK(sc)         mtx_unlock(&(sc)->sc_mtx)
 209 #define CGEM_LOCK_INIT(sc)      mtx_init(&(sc)->sc_mtx, \
 210             device_get_nameunit((sc)->dev), MTX_NETWORK_LOCK, MTX_DEF)
 211 #define CGEM_LOCK_DESTROY(sc)   mtx_destroy(&(sc)->sc_mtx)
 212 #define CGEM_ASSERT_LOCKED(sc)  mtx_assert(&(sc)->sc_mtx, MA_OWNED)
 213
 214 /* Allow platforms to optionally provide a way to set the reference clock. */
 215 int cgem_set_ref_clk(int unit, int frequency);
 216
 217 static devclass_t cgem_devclass;
 218
 219 static int cgem_probe(device_t dev);
 220 static int cgem_attach(device_t dev);
 221 static int cgem_detach(device_t dev);
 222 static void cgem_tick(void *);
 223 static void cgem_intr(void *);
 224
 225 static void cgem_mediachange(struct cgem_softc *, struct mii_data *);
 226
 227 static void
 228 cgem_get_mac(struct cgem_softc *sc, u_char eaddr[])
 229 {
 230         int i;
 231         uint32_t rnd;
 232
 233         /* See if boot loader gave us a MAC address already. */
 234         for (i = 0; i < 4; i++) {
 235                 uint32_t low = RD4(sc, CGEM_SPEC_ADDR_LOW(i));
 236                 uint32_t high = RD4(sc, CGEM_SPEC_ADDR_HI(i)) & 0xffff;
 237                 if (low != 0 || high != 0) {
 238                         eaddr[0] = low & 0xff;
 239                         eaddr[1] = (low >> 8) & 0xff;
 240                         eaddr[2] = (low >> 16) & 0xff;
 241                         eaddr[3] = (low >> 24) & 0xff;
 242                         eaddr[4] = high & 0xff;
 243                         eaddr[5] = (high >> 8) & 0xff;
 244                         break;
 245                 }
 246         }
 247
 248         /* No MAC from boot loader?  Assign a random one. */
 249         if (i == 4) {
 250                 rnd = arc4random();
 251
 252                 eaddr[0] = 'b';
 253                 eaddr[1] = 's';
 254                 eaddr[2] = 'd';
 255                 eaddr[3] = (rnd >> 16) & 0xff;
 256                 eaddr[4] = (rnd >> 8) & 0xff;
 257                 eaddr[5] = rnd & 0xff;
 258
 259                 device_printf(sc->dev, "no mac address found, assigning "
 260                     "random: %02x:%02x:%02x:%02x:%02x:%02x\n", eaddr[0],
 261                     eaddr[1], eaddr[2], eaddr[3], eaddr[4], eaddr[5]);
 262         }
 263
 264         /* Move address to first slot and zero out the rest. */
 265         WR4(sc, CGEM_SPEC_ADDR_LOW(0), (eaddr[3] << 24) |
 266             (eaddr[2] << 16) | (eaddr[1] << 8) | eaddr[0]);
 267         WR4(sc, CGEM_SPEC_ADDR_HI(0), (eaddr[5] << 8) | eaddr[4]);
 268
 269         for (i = 1; i < 4; i++) {
 270                 WR4(sc, CGEM_SPEC_ADDR_LOW(i), 0);
 271                 WR4(sc, CGEM_SPEC_ADDR_HI(i), 0);
 272         }
 273 }
 274
 275 /*
 276  * cgem_mac_hash():  map 48-bit address to a 6-bit hash. The 6-bit hash
 277  * corresponds to a bit in a 64-bit hash register.  Setting that bit in the hash
 278  * register enables reception of all frames with a destination address that
 279  * hashes to that 6-bit value.
 280  *
 281  * The hash function is described in sec. 16.2.3 in the Zynq-7000 Tech
 282  * Reference Manual.  Bits 0-5 in the hash are the exclusive-or of
 283  * every sixth bit in the destination address.
 284  */
 285 static int
 286 cgem_mac_hash(u_char eaddr[])
 287 {
 288         int hash;
 289         int i, j;
 290
 291         hash = 0;
 292         for (i = 0; i < 6; i++)
 293                 for (j = i; j < 48; j += 6)
 294                         if ((eaddr[j >> 3] & (1 << (j & 7))) != 0)
 295                                 hash ^= (1 << i);
 296
 297         return hash;
 298 }
 299
 300 static u_int
 301 cgem_hash_maddr(void *arg, struct sockaddr_dl *sdl, u_int cnt)
 302 {
 303         uint32_t *hashes = arg;
 304         int index;
 305
 306         index = cgem_mac_hash(LLADDR(sdl));
 307         if (index > 31)
 308                 hashes[0] |= (1U << (index - 32));
 309         else
 310                 hashes[1] |= (1U << index);
 311
 312         return (1);
 313 }
 314
 315 /*
 316  * After any change in rx flags or multi-cast addresses, set up hash registers
 317  * and net config register bits.
 318  */
 319 static void
 320 cgem_rx_filter(struct cgem_softc *sc)
 321 {
 322         if_t ifp = sc->ifp;
 323         uint32_t hashes[2] = { 0, 0 };
 324         uint32_t net_cfg;
 325
 326         net_cfg = RD4(sc, CGEM_NET_CFG);
 327
 328         net_cfg &= ~(CGEM_NET_CFG_MULTI_HASH_EN |
 329             CGEM_NET_CFG_NO_BCAST | CGEM_NET_CFG_COPY_ALL);
 330
 331         if ((if_getflags(ifp) & IFF_PROMISC) != 0)
 332                 net_cfg |= CGEM_NET_CFG_COPY_ALL;
 333         else {
 334                 if ((if_getflags(ifp) & IFF_BROADCAST) == 0)
 335                         net_cfg |= CGEM_NET_CFG_NO_BCAST;
 336                 if ((if_getflags(ifp) & IFF_ALLMULTI) != 0) {
 337                         hashes[0] = 0xffffffff;
 338                         hashes[1] = 0xffffffff;
 339                 } else
 340                         if_foreach_llmaddr(ifp, cgem_hash_maddr, hashes);
 341
 342                 if (hashes[0] != 0 || hashes[1] != 0)
 343                         net_cfg |= CGEM_NET_CFG_MULTI_HASH_EN;
 344         }
 345
 346         WR4(sc, CGEM_HASH_TOP, hashes[0]);
 347         WR4(sc, CGEM_HASH_BOT, hashes[1]);
 348         WR4(sc, CGEM_NET_CFG, net_cfg);
 349 }
 350
 351 /* For bus_dmamap_load() callback. */
 352 static void
 353 cgem_getaddr(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
 354 {
 355
 356         if (nsegs != 1 || error != 0)
 357                 return;
 358         *(bus_addr_t *)arg = segs[0].ds_addr;
 359 }
 360
 361 /* Create DMA'able descriptor rings. */
 362 static int
 363 cgem_setup_descs(struct cgem_softc *sc)
 364 {
 365         int i, err;
 366
 367         sc->txring = NULL;
 368         sc->rxring = NULL;
 369
 370         /* Allocate non-cached DMA space for RX and TX descriptors. */
 371         err = bus_dma_tag_create(bus_get_dma_tag(sc->dev), 1, 0,
 372             BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL,
 373             MAX_DESC_RING_SIZE, 1, MAX_DESC_RING_SIZE, 0,
 374             busdma_lock_mutex, &sc->sc_mtx, &sc->desc_dma_tag);
 375         if (err)
 376                 return (err);
 377
 378         /* Set up a bus_dma_tag for mbufs. */
 379         err = bus_dma_tag_create(bus_get_dma_tag(sc->dev), 1, 0,
 380             BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL,
 381             MCLBYTES, TX_MAX_DMA_SEGS, MCLBYTES, 0,
 382             busdma_lock_mutex, &sc->sc_mtx, &sc->mbuf_dma_tag);
 383         if (err)
 384                 return (err);
 385
 386         /* Allocate DMA memory in non-cacheable space. */
 387         err = bus_dmamem_alloc(sc->desc_dma_tag, (void **)&sc->rxring,
 388             BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &sc->rxring_dma_map);
 389         if (err)
 390                 return (err);
 391
 392         /* Load descriptor DMA memory. */
 393         err = bus_dmamap_load(sc->desc_dma_tag, sc->rxring_dma_map,
 394             (void *)sc->rxring, CGEM_NUM_RX_DESCS*sizeof(struct cgem_rx_desc),
 395             cgem_getaddr, &sc->rxring_physaddr, BUS_DMA_NOWAIT);
 396         if (err)
 397                 return (err);
 398
 399         /* Initialize RX descriptors. */
 400         for (i = 0; i < CGEM_NUM_RX_DESCS; i++) {
 401                 sc->rxring[i].addr = CGEM_RXDESC_OWN;
 402                 sc->rxring[i].ctl = 0;
 403                 sc->rxring_m[i] = NULL;
 404                 sc->rxring_m_dmamap[i] = NULL;
 405         }
 406         sc->rxring[CGEM_NUM_RX_DESCS - 1].addr |= CGEM_RXDESC_WRAP;
 407
 408         sc->rxring_hd_ptr = 0;
 409         sc->rxring_tl_ptr = 0;
 410         sc->rxring_queued = 0;
 411
 412         /* Allocate DMA memory for TX descriptors in non-cacheable space. */
 413         err = bus_dmamem_alloc(sc->desc_dma_tag, (void **)&sc->txring,
 414             BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &sc->txring_dma_map);
 415         if (err)
 416                 return (err);
 417
 418         /* Load TX descriptor DMA memory. */
 419         err = bus_dmamap_load(sc->desc_dma_tag, sc->txring_dma_map,
 420             (void *)sc->txring, CGEM_NUM_TX_DESCS*sizeof(struct cgem_tx_desc),
 421             cgem_getaddr, &sc->txring_physaddr, BUS_DMA_NOWAIT);
 422         if (err)
 423                 return (err);
 424
 425         /* Initialize TX descriptor ring. */
 426         for (i = 0; i < CGEM_NUM_TX_DESCS; i++) {
 427                 sc->txring[i].addr = 0;
 428                 sc->txring[i].ctl = CGEM_TXDESC_USED;
 429                 sc->txring_m[i] = NULL;
 430                 sc->txring_m_dmamap[i] = NULL;
 431         }
 432         sc->txring[CGEM_NUM_TX_DESCS - 1].ctl |= CGEM_TXDESC_WRAP;
 433
 434         sc->txring_hd_ptr = 0;
 435         sc->txring_tl_ptr = 0;
 436         sc->txring_queued = 0;
 437
 438         return (0);
 439 }
 440
 441 /* Fill receive descriptor ring with mbufs. */
 442 static void
 443 cgem_fill_rqueue(struct cgem_softc *sc)
 444 {
 445         struct mbuf *m = NULL;
 446         bus_dma_segment_t segs[TX_MAX_DMA_SEGS];
 447         int nsegs;
 448
 449         CGEM_ASSERT_LOCKED(sc);
 450
 451         while (sc->rxring_queued < sc->rxbufs) {
 452                 /* Get a cluster mbuf. */
 453                 m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
 454                 if (m == NULL)
 455                         break;
 456
 457                 m->m_len = MCLBYTES;
 458                 m->m_pkthdr.len = MCLBYTES;
 459                 m->m_pkthdr.rcvif = sc->ifp;
 460
 461                 /* Load map and plug in physical address. */
 462                 if (bus_dmamap_create(sc->mbuf_dma_tag, 0,
 463                     &sc->rxring_m_dmamap[sc->rxring_hd_ptr])) {
 464                         sc->rxdmamapfails++;
 465                         m_free(m);
 466                         break;
 467                 }
 468                 if (bus_dmamap_load_mbuf_sg(sc->mbuf_dma_tag,
 469                     sc->rxring_m_dmamap[sc->rxring_hd_ptr], m,
 470                     segs, &nsegs, BUS_DMA_NOWAIT)) {
 471                         sc->rxdmamapfails++;
 472                         bus_dmamap_destroy(sc->mbuf_dma_tag,
 473                                    sc->rxring_m_dmamap[sc->rxring_hd_ptr]);
 474                         sc->rxring_m_dmamap[sc->rxring_hd_ptr] = NULL;
 475                         m_free(m);
 476                         break;
 477                 }
 478                 sc->rxring_m[sc->rxring_hd_ptr] = m;
 479
 480                 /* Sync cache with receive buffer. */
 481                 bus_dmamap_sync(sc->mbuf_dma_tag,
 482                     sc->rxring_m_dmamap[sc->rxring_hd_ptr],
 483                     BUS_DMASYNC_PREREAD);
 484
 485                 /* Write rx descriptor and increment head pointer. */
 486                 sc->rxring[sc->rxring_hd_ptr].ctl = 0;
 487                 if (sc->rxring_hd_ptr == CGEM_NUM_RX_DESCS - 1) {
 488                         sc->rxring[sc->rxring_hd_ptr].addr = segs[0].ds_addr |
 489                             CGEM_RXDESC_WRAP;
 490                         sc->rxring_hd_ptr = 0;
 491                 } else
 492                         sc->rxring[sc->rxring_hd_ptr++].addr = segs[0].ds_addr;
 493
 494                 sc->rxring_queued++;
 495         }
 496 }
 497
 498 /* Pull received packets off of receive descriptor ring. */
 499 static void
 500 cgem_recv(struct cgem_softc *sc)
 501 {
 502         if_t ifp = sc->ifp;
 503         struct mbuf *m, *m_hd, **m_tl;
 504         uint32_t ctl;
 505
 506         CGEM_ASSERT_LOCKED(sc);
 507
 508         /* Pick up all packets in which the OWN bit is set. */
 509         m_hd = NULL;
 510         m_tl = &m_hd;
 511         while (sc->rxring_queued > 0 &&
 512            (sc->rxring[sc->rxring_tl_ptr].addr & CGEM_RXDESC_OWN) != 0) {
 513                 ctl = sc->rxring[sc->rxring_tl_ptr].ctl;
 514
 515                 /* Grab filled mbuf. */
 516                 m = sc->rxring_m[sc->rxring_tl_ptr];
 517                 sc->rxring_m[sc->rxring_tl_ptr] = NULL;
 518
 519                 /* Sync cache with receive buffer. */
 520                 bus_dmamap_sync(sc->mbuf_dma_tag,
 521                     sc->rxring_m_dmamap[sc->rxring_tl_ptr],
 522                     BUS_DMASYNC_POSTREAD);
 523
 524                 /* Unload and destroy dmamap. */
 525                 bus_dmamap_unload(sc->mbuf_dma_tag,
 526                     sc->rxring_m_dmamap[sc->rxring_tl_ptr]);
 527                 bus_dmamap_destroy(sc->mbuf_dma_tag,
 528                     sc->rxring_m_dmamap[sc->rxring_tl_ptr]);
 529                 sc->rxring_m_dmamap[sc->rxring_tl_ptr] = NULL;
 530
 531                 /* Increment tail pointer. */
 532                 if (++sc->rxring_tl_ptr == CGEM_NUM_RX_DESCS)
 533                         sc->rxring_tl_ptr = 0;
 534                 sc->rxring_queued--;
 535
 536                 /*
 537                  * Check FCS and make sure entire packet landed in one mbuf
 538                  * cluster (which is much bigger than the largest ethernet
 539                  * packet).
 540                  */
 541                 if ((ctl & CGEM_RXDESC_BAD_FCS) != 0 ||
 542                     (ctl & (CGEM_RXDESC_SOF | CGEM_RXDESC_EOF)) !=
 543                     (CGEM_RXDESC_SOF | CGEM_RXDESC_EOF)) {
 544                         /* discard. */
 545                         m_free(m);
 546                         if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
 547                         continue;
 548                 }
 549
 550                 /* Ready it to hand off to upper layers. */
 551                 m->m_data += ETHER_ALIGN;
 552                 m->m_len = (ctl & CGEM_RXDESC_LENGTH_MASK);
 553                 m->m_pkthdr.rcvif = ifp;
 554                 m->m_pkthdr.len = m->m_len;
 555
 556                 /*
 557                  * Are we using hardware checksumming?  Check the status in the
 558                  * receive descriptor.
 559                  */
 560                 if ((if_getcapenable(ifp) & IFCAP_RXCSUM) != 0) {
 561                         /* TCP or UDP checks out, IP checks out too. */
 562                         if ((ctl & CGEM_RXDESC_CKSUM_STAT_MASK) ==
 563                             CGEM_RXDESC_CKSUM_STAT_TCP_GOOD ||
 564                             (ctl & CGEM_RXDESC_CKSUM_STAT_MASK) ==
 565                             CGEM_RXDESC_CKSUM_STAT_UDP_GOOD) {
 566                                 m->m_pkthdr.csum_flags |=
 567                                     CSUM_IP_CHECKED | CSUM_IP_VALID |
 568                                     CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
 569                                 m->m_pkthdr.csum_data = 0xffff;
 570                         } else if ((ctl & CGEM_RXDESC_CKSUM_STAT_MASK) ==
 571                             CGEM_RXDESC_CKSUM_STAT_IP_GOOD) {
 572                                 /* Only IP checks out. */
 573                                 m->m_pkthdr.csum_flags |=
 574                                     CSUM_IP_CHECKED | CSUM_IP_VALID;
 575                                 m->m_pkthdr.csum_data = 0xffff;
 576                         }
 577                 }
 578
 579                 /* Queue it up for delivery below. */
 580                 *m_tl = m;
 581                 m_tl = &m->m_next;
 582         }
 583
 584         /* Replenish receive buffers. */
 585         cgem_fill_rqueue(sc);
 586
 587         /* Unlock and send up packets. */
 588         CGEM_UNLOCK(sc);
 589         while (m_hd != NULL) {
 590                 m = m_hd;
 591                 m_hd = m_hd->m_next;
 592                 m->m_next = NULL;
 593                 if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);
 594                 if_input(ifp, m);
 595         }
 596         CGEM_LOCK(sc);
 597 }
 598
 599 /* Find completed transmits and free their mbufs. */
 600 static void
 601 cgem_clean_tx(struct cgem_softc *sc)
 602 {
 603         struct mbuf *m;
 604         uint32_t ctl;
 605
 606         CGEM_ASSERT_LOCKED(sc);
 607
 608         /* free up finished transmits. */
 609         while (sc->txring_queued > 0 &&
 610             ((ctl = sc->txring[sc->txring_tl_ptr].ctl) &
 611             CGEM_TXDESC_USED) != 0) {
 612                 /* Sync cache. */
 613                 bus_dmamap_sync(sc->mbuf_dma_tag,
 614                     sc->txring_m_dmamap[sc->txring_tl_ptr],
 615                     BUS_DMASYNC_POSTWRITE);
 616
 617                 /* Unload and destroy DMA map. */
 618                 bus_dmamap_unload(sc->mbuf_dma_tag,
 619                     sc->txring_m_dmamap[sc->txring_tl_ptr]);
 620                 bus_dmamap_destroy(sc->mbuf_dma_tag,
 621                     sc->txring_m_dmamap[sc->txring_tl_ptr]);
 622                 sc->txring_m_dmamap[sc->txring_tl_ptr] = NULL;
 623
 624                 /* Free up the mbuf. */
 625                 m = sc->txring_m[sc->txring_tl_ptr];
 626                 sc->txring_m[sc->txring_tl_ptr] = NULL;
 627                 m_freem(m);
 628
 629                 /* Check the status. */
 630                 if ((ctl & CGEM_TXDESC_AHB_ERR) != 0) {
 631                         /* Serious bus error. log to console. */
 632                         device_printf(sc->dev,
 633                             "cgem_clean_tx: AHB error, addr=0x%x\n",
 634                             sc->txring[sc->txring_tl_ptr].addr);
 635                 } else if ((ctl & (CGEM_TXDESC_RETRY_ERR |
 636                     CGEM_TXDESC_LATE_COLL)) != 0) {
 637                         if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1);
 638                 } else
 639                         if_inc_counter(sc->ifp, IFCOUNTER_OPACKETS, 1);
 640
 641                 /*
 642                  * If the packet spanned more than one tx descriptor, skip
 643                  * descriptors until we find the end so that only start-of-frame
 644                  * descriptors are processed.
 645                  */
 646                 while ((ctl & CGEM_TXDESC_LAST_BUF) == 0) {
 647                         if ((ctl & CGEM_TXDESC_WRAP) != 0)
 648                                 sc->txring_tl_ptr = 0;
 649                         else
 650                                 sc->txring_tl_ptr++;
 651                         sc->txring_queued--;
 652
 653                         ctl = sc->txring[sc->txring_tl_ptr].ctl;
 654
 655                         sc->txring[sc->txring_tl_ptr].ctl =
 656                             ctl | CGEM_TXDESC_USED;
 657                 }
 658
 659                 /* Next descriptor. */
 660                 if ((ctl & CGEM_TXDESC_WRAP) != 0)
 661                         sc->txring_tl_ptr = 0;
 662                 else
 663                         sc->txring_tl_ptr++;
 664                 sc->txring_queued--;
 665
 666                 if_setdrvflagbits(sc->ifp, 0, IFF_DRV_OACTIVE);
 667         }
 668 }
 669
 670 /* Start transmits. */
 671 static void
 672 cgem_start_locked(if_t ifp)
 673 {
 674         struct cgem_softc *sc = (struct cgem_softc *) if_getsoftc(ifp);
 675         struct mbuf *m;
 676         bus_dma_segment_t segs[TX_MAX_DMA_SEGS];
 677         uint32_t ctl;
 678         int i, nsegs, wrap, err;
 679
 680         CGEM_ASSERT_LOCKED(sc);
 681
 682         if ((if_getdrvflags(ifp) & IFF_DRV_OACTIVE) != 0)
 683                 return;
 684
 685         for (;;) {
 686                 /* Check that there is room in the descriptor ring. */
 687                 if (sc->txring_queued >=
 688                     CGEM_NUM_TX_DESCS - TX_MAX_DMA_SEGS * 2) {
 689                         /* Try to make room. */
 690                         cgem_clean_tx(sc);
 691
 692                         /* Still no room? */
 693                         if (sc->txring_queued >=
 694                             CGEM_NUM_TX_DESCS - TX_MAX_DMA_SEGS * 2) {
 695                                 if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0);
 696                                 sc->txfull++;
 697                                 break;
 698                         }
 699                 }
 700
 701                 /* Grab next transmit packet. */
 702                 m = if_dequeue(ifp);
 703                 if (m == NULL)
 704                         break;
 705
 706                 /* Create and load DMA map. */
 707                 if (bus_dmamap_create(sc->mbuf_dma_tag, 0,
 708                         &sc->txring_m_dmamap[sc->txring_hd_ptr])) {
 709                         m_freem(m);
 710                         sc->txdmamapfails++;
 711                         continue;
 712                 }
 713                 err = bus_dmamap_load_mbuf_sg(sc->mbuf_dma_tag,
 714                     sc->txring_m_dmamap[sc->txring_hd_ptr], m, segs, &nsegs,
 715                     BUS_DMA_NOWAIT);
 716                 if (err == EFBIG) {
 717                         /* Too many segments!  defrag and try again. */
 718                         struct mbuf *m2 = m_defrag(m, M_NOWAIT);
 719
 720                         if (m2 == NULL) {
 721                                 sc->txdefragfails++;
 722                                 m_freem(m);
 723                                 bus_dmamap_destroy(sc->mbuf_dma_tag,
 724                                     sc->txring_m_dmamap[sc->txring_hd_ptr]);
 725                                 sc->txring_m_dmamap[sc->txring_hd_ptr] = NULL;
 726                                 continue;
 727                         }
 728                         m = m2;
 729                         err = bus_dmamap_load_mbuf_sg(sc->mbuf_dma_tag,
 730                             sc->txring_m_dmamap[sc->txring_hd_ptr], m, segs,
 731                             &nsegs, BUS_DMA_NOWAIT);
 732                         sc->txdefrags++;
 733                 }
 734                 if (err) {
 735                         /* Give up. */
 736                         m_freem(m);
 737                         bus_dmamap_destroy(sc->mbuf_dma_tag,
 738                             sc->txring_m_dmamap[sc->txring_hd_ptr]);
 739                         sc->txring_m_dmamap[sc->txring_hd_ptr] = NULL;
 740                         sc->txdmamapfails++;
 741                         continue;
 742                 }
 743                 sc->txring_m[sc->txring_hd_ptr] = m;
 744
 745                 /* Sync tx buffer with cache. */
 746                 bus_dmamap_sync(sc->mbuf_dma_tag,
 747                     sc->txring_m_dmamap[sc->txring_hd_ptr],
 748                     BUS_DMASYNC_PREWRITE);
 749
 750                 /* Set wrap flag if next packet might run off end of ring. */
 751                 wrap = sc->txring_hd_ptr + nsegs + TX_MAX_DMA_SEGS >=
 752                     CGEM_NUM_TX_DESCS;
 753
 754                 /*
 755                  * Fill in the TX descriptors back to front so that USED bit in
 756                  * first descriptor is cleared last.
 757                  */
 758                 for (i = nsegs - 1; i >= 0; i--) {
 759                         /* Descriptor address. */
 760                         sc->txring[sc->txring_hd_ptr + i].addr =
 761                             segs[i].ds_addr;
 762
 763                         /* Descriptor control word. */
 764                         ctl = segs[i].ds_len;
 765                         if (i == nsegs - 1) {
 766                                 ctl |= CGEM_TXDESC_LAST_BUF;
 767                                 if (wrap)
 768                                         ctl |= CGEM_TXDESC_WRAP;
 769                         }
 770                         sc->txring[sc->txring_hd_ptr + i].ctl = ctl;
 771
 772                         if (i != 0)
 773                                 sc->txring_m[sc->txring_hd_ptr + i] = NULL;
 774                 }
 775
 776                 if (wrap)
 777                         sc->txring_hd_ptr = 0;
 778                 else
 779                         sc->txring_hd_ptr += nsegs;
 780                 sc->txring_queued += nsegs;
 781
 782                 /* Kick the transmitter. */
 783                 WR4(sc, CGEM_NET_CTRL, sc->net_ctl_shadow |
 784                     CGEM_NET_CTRL_START_TX);
 785
 786                 /* If there is a BPF listener, bounce a copy to him. */
 787                 ETHER_BPF_MTAP(ifp, m);
 788         }
 789 }
 790
 791 static void
 792 cgem_start(if_t ifp)
 793 {
 794         struct cgem_softc *sc = (struct cgem_softc *) if_getsoftc(ifp);
 795
 796         CGEM_LOCK(sc);
 797         cgem_start_locked(ifp);
 798         CGEM_UNLOCK(sc);
 799 }
 800
 801 static void
 802 cgem_poll_hw_stats(struct cgem_softc *sc)
 803 {
 804         uint32_t n;
 805
 806         CGEM_ASSERT_LOCKED(sc);
 807
 808         sc->stats.tx_bytes += RD4(sc, CGEM_OCTETS_TX_BOT);
 809         sc->stats.tx_bytes += (uint64_t)RD4(sc, CGEM_OCTETS_TX_TOP) << 32;
 810
 811         sc->stats.tx_frames += RD4(sc, CGEM_FRAMES_TX);
 812         sc->stats.tx_frames_bcast += RD4(sc, CGEM_BCAST_FRAMES_TX);
 813         sc->stats.tx_frames_multi += RD4(sc, CGEM_MULTI_FRAMES_TX);
 814         sc->stats.tx_frames_pause += RD4(sc, CGEM_PAUSE_FRAMES_TX);
 815         sc->stats.tx_frames_64b += RD4(sc, CGEM_FRAMES_64B_TX);
 816         sc->stats.tx_frames_65to127b += RD4(sc, CGEM_FRAMES_65_127B_TX);
 817         sc->stats.tx_frames_128to255b += RD4(sc, CGEM_FRAMES_128_255B_TX);
 818         sc->stats.tx_frames_256to511b += RD4(sc, CGEM_FRAMES_256_511B_TX);
 819         sc->stats.tx_frames_512to1023b += RD4(sc, CGEM_FRAMES_512_1023B_TX);
 820         sc->stats.tx_frames_1024to1536b += RD4(sc, CGEM_FRAMES_1024_1518B_TX);
 821         sc->stats.tx_under_runs += RD4(sc, CGEM_TX_UNDERRUNS);
 822
 823         n = RD4(sc, CGEM_SINGLE_COLL_FRAMES);
 824         sc->stats.tx_single_collisn += n;
 825         if_inc_counter(sc->ifp, IFCOUNTER_COLLISIONS, n);
 826         n = RD4(sc, CGEM_MULTI_COLL_FRAMES);
 827         sc->stats.tx_multi_collisn += n;
 828         if_inc_counter(sc->ifp, IFCOUNTER_COLLISIONS, n);
 829         n = RD4(sc, CGEM_EXCESSIVE_COLL_FRAMES);
 830         sc->stats.tx_excsv_collisn += n;
 831         if_inc_counter(sc->ifp, IFCOUNTER_COLLISIONS, n);
 832         n = RD4(sc, CGEM_LATE_COLL);
 833         sc->stats.tx_late_collisn += n;
 834         if_inc_counter(sc->ifp, IFCOUNTER_COLLISIONS, n);
 835
 836         sc->stats.tx_deferred_frames += RD4(sc, CGEM_DEFERRED_TX_FRAMES);
 837         sc->stats.tx_carrier_sense_errs += RD4(sc, CGEM_CARRIER_SENSE_ERRS);
 838
 839         sc->stats.rx_bytes += RD4(sc, CGEM_OCTETS_RX_BOT);
 840         sc->stats.rx_bytes += (uint64_t)RD4(sc, CGEM_OCTETS_RX_TOP) << 32;
 841
 842         sc->stats.rx_frames += RD4(sc, CGEM_FRAMES_RX);
 843         sc->stats.rx_frames_bcast += RD4(sc, CGEM_BCAST_FRAMES_RX);
 844         sc->stats.rx_frames_multi += RD4(sc, CGEM_MULTI_FRAMES_RX);
 845         sc->stats.rx_frames_pause += RD4(sc, CGEM_PAUSE_FRAMES_RX);
 846         sc->stats.rx_frames_64b += RD4(sc, CGEM_FRAMES_64B_RX);
 847         sc->stats.rx_frames_65to127b += RD4(sc, CGEM_FRAMES_65_127B_RX);
 848         sc->stats.rx_frames_128to255b += RD4(sc, CGEM_FRAMES_128_255B_RX);
 849         sc->stats.rx_frames_256to511b += RD4(sc, CGEM_FRAMES_256_511B_RX);
 850         sc->stats.rx_frames_512to1023b += RD4(sc, CGEM_FRAMES_512_1023B_RX);
 851         sc->stats.rx_frames_1024to1536b += RD4(sc, CGEM_FRAMES_1024_1518B_RX);
 852         sc->stats.rx_frames_undersize += RD4(sc, CGEM_UNDERSZ_RX);
 853         sc->stats.rx_frames_oversize += RD4(sc, CGEM_OVERSZ_RX);
 854         sc->stats.rx_frames_jabber += RD4(sc, CGEM_JABBERS_RX);
 855         sc->stats.rx_frames_fcs_errs += RD4(sc, CGEM_FCS_ERRS);
 856         sc->stats.rx_frames_length_errs += RD4(sc, CGEM_LENGTH_FIELD_ERRS);
 857         sc->stats.rx_symbol_errs += RD4(sc, CGEM_RX_SYMBOL_ERRS);
 858         sc->stats.rx_align_errs += RD4(sc, CGEM_ALIGN_ERRS);
 859         sc->stats.rx_resource_errs += RD4(sc, CGEM_RX_RESOURCE_ERRS);
 860         sc->stats.rx_overrun_errs += RD4(sc, CGEM_RX_OVERRUN_ERRS);
 861         sc->stats.rx_ip_hdr_csum_errs += RD4(sc, CGEM_IP_HDR_CKSUM_ERRS);
 862         sc->stats.rx_tcp_csum_errs += RD4(sc, CGEM_TCP_CKSUM_ERRS);
 863         sc->stats.rx_udp_csum_errs += RD4(sc, CGEM_UDP_CKSUM_ERRS);
 864 }
 865
 866 static void
 867 cgem_tick(void *arg)
 868 {
 869         struct cgem_softc *sc = (struct cgem_softc *)arg;
 870         struct mii_data *mii;
 871
 872         CGEM_ASSERT_LOCKED(sc);
 873
 874         /* Poll the phy. */
 875         if (sc->miibus != NULL) {
 876                 mii = device_get_softc(sc->miibus);
 877                 mii_tick(mii);
 878         }
 879
 880         /* Poll statistics registers. */
 881         cgem_poll_hw_stats(sc);
 882
 883         /* Check for receiver hang. */
 884         if (sc->rxhangwar && sc->rx_frames_prev == sc->stats.rx_frames) {
 885                 /*
 886                  * Reset receiver logic by toggling RX_EN bit.  1usec
 887                  * delay is necessary especially when operating at 100mbps
 888                  * and 10mbps speeds.
 889                  */
 890                 WR4(sc, CGEM_NET_CTRL, sc->net_ctl_shadow &
 891                     ~CGEM_NET_CTRL_RX_EN);
 892                 DELAY(1);
 893                 WR4(sc, CGEM_NET_CTRL, sc->net_ctl_shadow);
 894         }
 895         sc->rx_frames_prev = sc->stats.rx_frames;
 896
 897         /* Next callout in one second. */
 898         callout_reset(&sc->tick_ch, hz, cgem_tick, sc);
 899 }
 900
 901 /* Interrupt handler. */
 902 static void
 903 cgem_intr(void *arg)
 904 {
 905         struct cgem_softc *sc = (struct cgem_softc *)arg;
 906         if_t ifp = sc->ifp;
 907         uint32_t istatus;
 908
 909         CGEM_LOCK(sc);
 910
 911         if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) {
 912                 CGEM_UNLOCK(sc);
 913                 return;
 914         }
 915
 916         /* Read interrupt status and immediately clear the bits. */
 917         istatus = RD4(sc, CGEM_INTR_STAT);
 918         WR4(sc, CGEM_INTR_STAT, istatus);
 919
 920         /* Packets received. */
 921         if ((istatus & CGEM_INTR_RX_COMPLETE) != 0)
 922                 cgem_recv(sc);
 923
 924         /* Free up any completed transmit buffers. */
 925         cgem_clean_tx(sc);
 926
 927         /* Hresp not ok.  Something is very bad with DMA.  Try to clear. */
 928         if ((istatus & CGEM_INTR_HRESP_NOT_OK) != 0) {
 929                 device_printf(sc->dev,
 930                     "cgem_intr: hresp not okay! rx_status=0x%x\n",
 931                     RD4(sc, CGEM_RX_STAT));
 932                 WR4(sc, CGEM_RX_STAT, CGEM_RX_STAT_HRESP_NOT_OK);
 933         }
 934
 935         /* Receiver overrun. */
 936         if ((istatus & CGEM_INTR_RX_OVERRUN) != 0) {
 937                 /* Clear status bit. */
 938                 WR4(sc, CGEM_RX_STAT, CGEM_RX_STAT_OVERRUN);
 939                 sc->rxoverruns++;
 940         }
 941
 942         /* Receiver ran out of bufs. */
 943         if ((istatus & CGEM_INTR_RX_USED_READ) != 0) {
 944                 WR4(sc, CGEM_NET_CTRL, sc->net_ctl_shadow |
 945                     CGEM_NET_CTRL_FLUSH_DPRAM_PKT);
 946                 cgem_fill_rqueue(sc);
 947                 sc->rxnobufs++;
 948         }
 949
 950         /* Restart transmitter if needed. */
 951         if (!if_sendq_empty(ifp))
 952                 cgem_start_locked(ifp);
 953
 954         CGEM_UNLOCK(sc);
 955 }
 956
 957 /* Reset hardware. */
 958 static void
 959 cgem_reset(struct cgem_softc *sc)
 960 {
 961
 962         CGEM_ASSERT_LOCKED(sc);
 963
 964         WR4(sc, CGEM_NET_CTRL, 0);
 965         WR4(sc, CGEM_NET_CFG, 0);
 966         WR4(sc, CGEM_NET_CTRL, CGEM_NET_CTRL_CLR_STAT_REGS);
 967         WR4(sc, CGEM_TX_STAT, CGEM_TX_STAT_ALL);
 968         WR4(sc, CGEM_RX_STAT, CGEM_RX_STAT_ALL);
 969         WR4(sc, CGEM_INTR_DIS, CGEM_INTR_ALL);
 970         WR4(sc, CGEM_HASH_BOT, 0);
 971         WR4(sc, CGEM_HASH_TOP, 0);
 972         WR4(sc, CGEM_TX_QBAR, 0);       /* manual says do this. */
 973         WR4(sc, CGEM_RX_QBAR, 0);
 974
 975         /* Get management port running even if interface is down. */
 976         WR4(sc, CGEM_NET_CFG, CGEM_NET_CFG_DBUS_WIDTH_32 |
 977             CGEM_NET_CFG_MDC_CLK_DIV_64);
 978
 979         sc->net_ctl_shadow = CGEM_NET_CTRL_MGMT_PORT_EN;
 980         WR4(sc, CGEM_NET_CTRL, sc->net_ctl_shadow);
 981 }
 982
 983 /* Bring up the hardware. */
 984 static void
 985 cgem_config(struct cgem_softc *sc)
 986 {
 987         if_t ifp = sc->ifp;
 988         uint32_t net_cfg;
 989         uint32_t dma_cfg;
 990         u_char *eaddr = if_getlladdr(ifp);
 991
 992         CGEM_ASSERT_LOCKED(sc);
 993
 994         /* Program Net Config Register. */
 995         net_cfg = CGEM_NET_CFG_DBUS_WIDTH_32 |
 996             CGEM_NET_CFG_MDC_CLK_DIV_64 |
 997             CGEM_NET_CFG_FCS_REMOVE |
 998             CGEM_NET_CFG_RX_BUF_OFFSET(ETHER_ALIGN) |
 999             CGEM_NET_CFG_GIGE_EN |
1000             CGEM_NET_CFG_1536RXEN |
1001             CGEM_NET_CFG_FULL_DUPLEX |
1002             CGEM_NET_CFG_SPEED100;
1003
1004         /* Enable receive checksum offloading? */
1005         if ((if_getcapenable(ifp) & IFCAP_RXCSUM) != 0)
1006                 net_cfg |=  CGEM_NET_CFG_RX_CHKSUM_OFFLD_EN;
1007
1008         WR4(sc, CGEM_NET_CFG, net_cfg);
1009
1010         /* Program DMA Config Register. */
1011         dma_cfg = CGEM_DMA_CFG_RX_BUF_SIZE(MCLBYTES) |
1012             CGEM_DMA_CFG_RX_PKTBUF_MEMSZ_SEL_8K |
1013             CGEM_DMA_CFG_TX_PKTBUF_MEMSZ_SEL |
1014             CGEM_DMA_CFG_AHB_FIXED_BURST_LEN_16 |
1015             CGEM_DMA_CFG_DISC_WHEN_NO_AHB;
1016
1017         /* Enable transmit checksum offloading? */
1018         if ((if_getcapenable(ifp) & IFCAP_TXCSUM) != 0)
1019                 dma_cfg |= CGEM_DMA_CFG_CHKSUM_GEN_OFFLOAD_EN;
1020
1021         WR4(sc, CGEM_DMA_CFG, dma_cfg);
1022
1023         /* Write the rx and tx descriptor ring addresses to the QBAR regs. */
1024         WR4(sc, CGEM_RX_QBAR, (uint32_t) sc->rxring_physaddr);
1025         WR4(sc, CGEM_TX_QBAR, (uint32_t) sc->txring_physaddr);
1026
1027         /* Enable rx and tx. */
1028         sc->net_ctl_shadow |= (CGEM_NET_CTRL_TX_EN | CGEM_NET_CTRL_RX_EN);
1029         WR4(sc, CGEM_NET_CTRL, sc->net_ctl_shadow);
1030
1031         /* Set receive address in case it changed. */
1032         WR4(sc, CGEM_SPEC_ADDR_LOW(0), (eaddr[3] << 24) |
1033             (eaddr[2] << 16) | (eaddr[1] << 8) | eaddr[0]);
1034         WR4(sc, CGEM_SPEC_ADDR_HI(0), (eaddr[5] << 8) | eaddr[4]);
1035
1036         /* Set up interrupts. */
1037         WR4(sc, CGEM_INTR_EN, CGEM_INTR_RX_COMPLETE | CGEM_INTR_RX_OVERRUN |
1038             CGEM_INTR_TX_USED_READ | CGEM_INTR_RX_USED_READ |
1039             CGEM_INTR_HRESP_NOT_OK);
1040 }
1041
1042 /* Turn on interface and load up receive ring with buffers. */
1043 static void
1044 cgem_init_locked(struct cgem_softc *sc)
1045 {
1046         struct mii_data *mii;
1047
1048         CGEM_ASSERT_LOCKED(sc);
1049
1050         if ((if_getdrvflags(sc->ifp) & IFF_DRV_RUNNING) != 0)
1051                 return;
1052
1053         cgem_config(sc);
1054         cgem_fill_rqueue(sc);
1055
1056         if_setdrvflagbits(sc->ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);
1057
1058         mii = device_get_softc(sc->miibus);
1059         mii_mediachg(mii);
1060
1061         callout_reset(&sc->tick_ch, hz, cgem_tick, sc);
1062 }
1063
1064 static void
1065 cgem_init(void *arg)
1066 {
1067         struct cgem_softc *sc = (struct cgem_softc *)arg;
1068
1069         CGEM_LOCK(sc);
1070         cgem_init_locked(sc);
1071         CGEM_UNLOCK(sc);
1072 }
1073
1074 /* Turn off interface.  Free up any buffers in transmit or receive queues. */
1075 static void
1076 cgem_stop(struct cgem_softc *sc)
1077 {
1078         int i;
1079
1080         CGEM_ASSERT_LOCKED(sc);
1081
1082         callout_stop(&sc->tick_ch);
1083
1084         /* Shut down hardware. */
1085         cgem_reset(sc);
1086
1087         /* Clear out transmit queue. */
1088         for (i = 0; i < CGEM_NUM_TX_DESCS; i++) {
1089                 sc->txring[i].ctl = CGEM_TXDESC_USED;
1090                 sc->txring[i].addr = 0;
1091                 if (sc->txring_m[i]) {
1092                         /* Unload and destroy dmamap. */
1093                         bus_dmamap_unload(sc->mbuf_dma_tag,
1094                             sc->txring_m_dmamap[i]);
1095                         bus_dmamap_destroy(sc->mbuf_dma_tag,
1096                             sc->txring_m_dmamap[i]);
1097                         sc->txring_m_dmamap[i] = NULL;
1098                         m_freem(sc->txring_m[i]);
1099                         sc->txring_m[i] = NULL;
1100                 }
1101         }
1102         sc->txring[CGEM_NUM_TX_DESCS - 1].ctl |= CGEM_TXDESC_WRAP;
1103
1104         sc->txring_hd_ptr = 0;
1105         sc->txring_tl_ptr = 0;
1106         sc->txring_queued = 0;
1107
1108         /* Clear out receive queue. */
1109         for (i = 0; i < CGEM_NUM_RX_DESCS; i++) {
1110                 sc->rxring[i].addr = CGEM_RXDESC_OWN;
1111                 sc->rxring[i].ctl = 0;
1112                 if (sc->rxring_m[i]) {
1113                         /* Unload and destroy dmamap. */
1114                         bus_dmamap_unload(sc->mbuf_dma_tag,
1115                             sc->rxring_m_dmamap[i]);
1116                         bus_dmamap_destroy(sc->mbuf_dma_tag,
1117                             sc->rxring_m_dmamap[i]);
1118                         sc->rxring_m_dmamap[i] = NULL;
1119
1120                         m_freem(sc->rxring_m[i]);
1121                         sc->rxring_m[i] = NULL;
1122                 }
1123         }
1124         sc->rxring[CGEM_NUM_RX_DESCS - 1].addr |= CGEM_RXDESC_WRAP;
1125
1126         sc->rxring_hd_ptr = 0;
1127         sc->rxring_tl_ptr = 0;
1128         sc->rxring_queued = 0;
1129
1130         /* Force next statchg or linkchg to program net config register. */
1131         sc->mii_media_active = 0;
1132 }
1133
1134 static int
1135 cgem_ioctl(if_t ifp, u_long cmd, caddr_t data)
1136 {
1137         struct cgem_softc *sc = if_getsoftc(ifp);
1138         struct ifreq *ifr = (struct ifreq *)data;
1139         struct mii_data *mii;
1140         int error = 0, mask;
1141
1142         switch (cmd) {
1143         case SIOCSIFFLAGS:
1144                 CGEM_LOCK(sc);
1145                 if ((if_getflags(ifp) & IFF_UP) != 0) {
1146                         if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) {
1147                                 if (((if_getflags(ifp) ^ sc->if_old_flags) &
1148                                     (IFF_PROMISC | IFF_ALLMULTI)) != 0) {
1149                                         cgem_rx_filter(sc);
1150                                 }
1151                         } else {
1152                                 cgem_init_locked(sc);
1153                         }
1154                 } else if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) {
1155                         if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING);
1156                         cgem_stop(sc);
1157                 }
1158                 sc->if_old_flags = if_getflags(ifp);
1159                 CGEM_UNLOCK(sc);
1160                 break;
1161
1162         case SIOCADDMULTI:
1163         case SIOCDELMULTI:
1164                 /* Set up multi-cast filters. */
1165                 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) {
1166                         CGEM_LOCK(sc);
1167                         cgem_rx_filter(sc);
1168                         CGEM_UNLOCK(sc);
1169                 }
1170                 break;
1171
1172         case SIOCSIFMEDIA:
1173         case SIOCGIFMEDIA:
1174                 mii = device_get_softc(sc->miibus);
1175                 error = ifmedia_ioctl(ifp, ifr, &mii->mii_media, cmd);
1176                 break;
1177
1178         case SIOCSIFCAP:
1179                 CGEM_LOCK(sc);
1180                 mask = if_getcapenable(ifp) ^ ifr->ifr_reqcap;
1181
1182                 if ((mask & IFCAP_TXCSUM) != 0) {
1183                         if ((ifr->ifr_reqcap & IFCAP_TXCSUM) != 0) {
1184                                 /* Turn on TX checksumming. */
1185                                 if_setcapenablebit(ifp, IFCAP_TXCSUM |
1186                                     IFCAP_TXCSUM_IPV6, 0);
1187                                 if_sethwassistbits(ifp, CGEM_CKSUM_ASSIST, 0);
1188
1189                                 WR4(sc, CGEM_DMA_CFG,
1190                                     RD4(sc, CGEM_DMA_CFG) |
1191                                     CGEM_DMA_CFG_CHKSUM_GEN_OFFLOAD_EN);
1192                         } else {
1193                                 /* Turn off TX checksumming. */
1194                                 if_setcapenablebit(ifp, 0, IFCAP_TXCSUM |
1195                                     IFCAP_TXCSUM_IPV6);
1196                                 if_sethwassistbits(ifp, 0, CGEM_CKSUM_ASSIST);
1197
1198                                 WR4(sc, CGEM_DMA_CFG,
1199                                     RD4(sc, CGEM_DMA_CFG) &
1200                                     ~CGEM_DMA_CFG_CHKSUM_GEN_OFFLOAD_EN);
1201                         }
1202                 }
1203                 if ((mask & IFCAP_RXCSUM) != 0) {
1204                         if ((ifr->ifr_reqcap & IFCAP_RXCSUM) != 0) {
1205                                 /* Turn on RX checksumming. */
1206                                 if_setcapenablebit(ifp, IFCAP_RXCSUM |
1207                                     IFCAP_RXCSUM_IPV6, 0);
1208                                 WR4(sc, CGEM_NET_CFG,
1209                                     RD4(sc, CGEM_NET_CFG) |
1210                                     CGEM_NET_CFG_RX_CHKSUM_OFFLD_EN);
1211                         } else {
1212                                 /* Turn off RX checksumming. */
1213                                 if_setcapenablebit(ifp, 0, IFCAP_RXCSUM |
1214                                     IFCAP_RXCSUM_IPV6);
1215                                 WR4(sc, CGEM_NET_CFG,
1216                                     RD4(sc, CGEM_NET_CFG) &
1217                                     ~CGEM_NET_CFG_RX_CHKSUM_OFFLD_EN);
1218                         }
1219                 }
1220                 if ((if_getcapenable(ifp) & (IFCAP_RXCSUM | IFCAP_TXCSUM)) ==
1221                     (IFCAP_RXCSUM | IFCAP_TXCSUM))
1222                         if_setcapenablebit(ifp, IFCAP_VLAN_HWCSUM, 0);
1223                 else
1224                         if_setcapenablebit(ifp, 0, IFCAP_VLAN_HWCSUM);
1225
1226                 CGEM_UNLOCK(sc);
1227                 break;
1228         default:
1229                 error = ether_ioctl(ifp, cmd, data);
1230                 break;
1231         }
1232
1233         return (error);
1234 }
1235
1236 /* MII bus support routines.
1237  */
1238 static void
1239 cgem_child_detached(device_t dev, device_t child)
1240 {
1241         struct cgem_softc *sc = device_get_softc(dev);
1242
1243         if (child == sc->miibus)
1244                 sc->miibus = NULL;
1245 }
1246
1247 static int
1248 cgem_ifmedia_upd(if_t ifp)
1249 {
1250         struct cgem_softc *sc = (struct cgem_softc *) if_getsoftc(ifp);
1251         struct mii_data *mii;
1252         struct mii_softc *miisc;
1253         int error = 0;
1254
1255         mii = device_get_softc(sc->miibus);
1256         CGEM_LOCK(sc);
1257         if ((if_getflags(ifp) & IFF_UP) != 0) {
1258                 LIST_FOREACH(miisc, &mii->mii_phys, mii_list)
1259                         PHY_RESET(miisc);
1260                 error = mii_mediachg(mii);
1261         }
1262         CGEM_UNLOCK(sc);
1263
1264         return (error);
1265 }
1266
1267 static void
1268 cgem_ifmedia_sts(if_t ifp, struct ifmediareq *ifmr)
1269 {
1270         struct cgem_softc *sc = (struct cgem_softc *) if_getsoftc(ifp);
1271         struct mii_data *mii;
1272
1273         mii = device_get_softc(sc->miibus);
1274         CGEM_LOCK(sc);
1275         mii_pollstat(mii);
1276         ifmr->ifm_active = mii->mii_media_active;
1277         ifmr->ifm_status = mii->mii_media_status;
1278         CGEM_UNLOCK(sc);
1279 }
1280
1281 static int
1282 cgem_miibus_readreg(device_t dev, int phy, int reg)
1283 {
1284         struct cgem_softc *sc = device_get_softc(dev);
1285         int tries, val;
1286
1287         WR4(sc, CGEM_PHY_MAINT, CGEM_PHY_MAINT_CLAUSE_22 |
1288             CGEM_PHY_MAINT_MUST_10 | CGEM_PHY_MAINT_OP_READ |
1289             (phy << CGEM_PHY_MAINT_PHY_ADDR_SHIFT) |
1290             (reg << CGEM_PHY_MAINT_REG_ADDR_SHIFT));
1291
1292         /* Wait for completion. */
1293         tries=0;
1294         while ((RD4(sc, CGEM_NET_STAT) & CGEM_NET_STAT_PHY_MGMT_IDLE) == 0) {
1295                 DELAY(5);
1296                 if (++tries > 200) {
1297                         device_printf(dev, "phy read timeout: %d\n", reg);
1298                         return (-1);
1299                 }
1300         }
1301
1302         val = RD4(sc, CGEM_PHY_MAINT) & CGEM_PHY_MAINT_DATA_MASK;
1303
1304         if (reg == MII_EXTSR)
1305                 /*
1306                  * MAC does not support half-duplex at gig speeds.
1307                  * Let mii(4) exclude the capability.
1308                  */
1309                 val &= ~(EXTSR_1000XHDX | EXTSR_1000THDX);
1310
1311         return (val);
1312 }
1313
1314 static int
1315 cgem_miibus_writereg(device_t dev, int phy, int reg, int data)
1316 {
1317         struct cgem_softc *sc = device_get_softc(dev);
1318         int tries;
1319
1320         WR4(sc, CGEM_PHY_MAINT, CGEM_PHY_MAINT_CLAUSE_22 |
1321             CGEM_PHY_MAINT_MUST_10 | CGEM_PHY_MAINT_OP_WRITE |
1322             (phy << CGEM_PHY_MAINT_PHY_ADDR_SHIFT) |
1323             (reg << CGEM_PHY_MAINT_REG_ADDR_SHIFT) |
1324             (data & CGEM_PHY_MAINT_DATA_MASK));
1325
1326         /* Wait for completion. */
1327         tries = 0;
1328         while ((RD4(sc, CGEM_NET_STAT) & CGEM_NET_STAT_PHY_MGMT_IDLE) == 0) {
1329                 DELAY(5);
1330                 if (++tries > 200) {
1331                         device_printf(dev, "phy write timeout: %d\n", reg);
1332                         return (-1);
1333                 }
1334         }
1335
1336         return (0);
1337 }
1338
1339 static void
1340 cgem_miibus_statchg(device_t dev)
1341 {
1342         struct cgem_softc *sc  = device_get_softc(dev);
1343         struct mii_data *mii = device_get_softc(sc->miibus);
1344
1345         CGEM_ASSERT_LOCKED(sc);
1346
1347         if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) ==
1348             (IFM_ACTIVE | IFM_AVALID) &&
1349             sc->mii_media_active != mii->mii_media_active)
1350                 cgem_mediachange(sc, mii);
1351 }
1352
1353 static void
1354 cgem_miibus_linkchg(device_t dev)
1355 {
1356         struct cgem_softc *sc  = device_get_softc(dev);
1357         struct mii_data *mii = device_get_softc(sc->miibus);
1358
1359         CGEM_ASSERT_LOCKED(sc);
1360
1361         if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) ==
1362             (IFM_ACTIVE | IFM_AVALID) &&
1363             sc->mii_media_active != mii->mii_media_active)
1364                 cgem_mediachange(sc, mii);
1365 }
1366
1367 /*
1368  * Overridable weak symbol cgem_set_ref_clk().  This allows platforms to
1369  * provide a function to set the cgem's reference clock.
1370  */
1371 static int __used
1372 cgem_default_set_ref_clk(int unit, int frequency)
1373 {
1374
1375         return 0;
1376 }
1377 __weak_reference(cgem_default_set_ref_clk, cgem_set_ref_clk);
1378
1379 /* Call to set reference clock and network config bits according to media. */
1380 static void
1381 cgem_mediachange(struct cgem_softc *sc, struct mii_data *mii)
1382 {
1383         uint32_t net_cfg;
1384         int ref_clk_freq;
1385
1386         CGEM_ASSERT_LOCKED(sc);
1387
1388         /* Update hardware to reflect media. */
1389         net_cfg = RD4(sc, CGEM_NET_CFG);
1390         net_cfg &= ~(CGEM_NET_CFG_SPEED100 | CGEM_NET_CFG_GIGE_EN |
1391             CGEM_NET_CFG_FULL_DUPLEX);
1392
1393         switch (IFM_SUBTYPE(mii->mii_media_active)) {
1394         case IFM_1000_T:
1395                 net_cfg |= (CGEM_NET_CFG_SPEED100 |
1396                     CGEM_NET_CFG_GIGE_EN);
1397                 ref_clk_freq = 125000000;
1398                 break;
1399         case IFM_100_TX:
1400                 net_cfg |= CGEM_NET_CFG_SPEED100;
1401                 ref_clk_freq = 25000000;
1402                 break;
1403         default:
1404                 ref_clk_freq = 2500000;
1405         }
1406
1407         if ((mii->mii_media_active & IFM_FDX) != 0)
1408                 net_cfg |= CGEM_NET_CFG_FULL_DUPLEX;
1409
1410         WR4(sc, CGEM_NET_CFG, net_cfg);
1411
1412         /* Set the reference clock if necessary. */
1413         if (cgem_set_ref_clk(sc->ref_clk_num, ref_clk_freq))
1414                 device_printf(sc->dev,
1415                     "cgem_mediachange: could not set ref clk%d to %d.\n",
1416                     sc->ref_clk_num, ref_clk_freq);
1417
1418         sc->mii_media_active = mii->mii_media_active;
1419 }
1420
1421 static void
1422 cgem_add_sysctls(device_t dev)
1423 {
1424         struct cgem_softc *sc = device_get_softc(dev);
1425         struct sysctl_ctx_list *ctx;
1426         struct sysctl_oid_list *child;
1427         struct sysctl_oid *tree;
1428
1429         ctx = device_get_sysctl_ctx(dev);
1430         child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev));
1431
1432         SYSCTL_ADD_INT(ctx, child, OID_AUTO, "rxbufs", CTLFLAG_RW,
1433             &sc->rxbufs, 0, "Number receive buffers to provide");
1434
1435         SYSCTL_ADD_INT(ctx, child, OID_AUTO, "rxhangwar", CTLFLAG_RW,
1436             &sc->rxhangwar, 0, "Enable receive hang work-around");
1437
1438         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "_rxoverruns", CTLFLAG_RD,
1439             &sc->rxoverruns, 0, "Receive overrun events");
1440
1441         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "_rxnobufs", CTLFLAG_RD,
1442             &sc->rxnobufs, 0, "Receive buf queue empty events");
1443
1444         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "_rxdmamapfails", CTLFLAG_RD,
1445             &sc->rxdmamapfails, 0, "Receive DMA map failures");
1446
1447         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "_txfull", CTLFLAG_RD,
1448             &sc->txfull, 0, "Transmit ring full events");
1449
1450         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "_txdmamapfails", CTLFLAG_RD,
1451             &sc->txdmamapfails, 0, "Transmit DMA map failures");
1452
1453         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "_txdefrags", CTLFLAG_RD,
1454             &sc->txdefrags, 0, "Transmit m_defrag() calls");
1455
1456         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "_txdefragfails", CTLFLAG_RD,
1457             &sc->txdefragfails, 0, "Transmit m_defrag() failures");
1458
1459         tree = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "stats",
1460             CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "GEM statistics");
1461         child = SYSCTL_CHILDREN(tree);
1462
1463         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_bytes", CTLFLAG_RD,
1464             &sc->stats.tx_bytes, "Total bytes transmitted");
1465
1466         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames", CTLFLAG_RD,
1467             &sc->stats.tx_frames, 0, "Total frames transmitted");
1468
1469         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_bcast", CTLFLAG_RD,
1470             &sc->stats.tx_frames_bcast, 0,
1471             "Number broadcast frames transmitted");
1472
1473         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_multi", CTLFLAG_RD,
1474             &sc->stats.tx_frames_multi, 0,
1475             "Number multicast frames transmitted");
1476
1477         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_pause",
1478             CTLFLAG_RD, &sc->stats.tx_frames_pause, 0,
1479             "Number pause frames transmitted");
1480
1481         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_64b", CTLFLAG_RD,
1482             &sc->stats.tx_frames_64b, 0,
1483             "Number frames transmitted of size 64 bytes or less");
1484
1485         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_65to127b", CTLFLAG_RD,
1486             &sc->stats.tx_frames_65to127b, 0,
1487             "Number frames transmitted of size 65-127 bytes");
1488
1489         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_128to255b",
1490             CTLFLAG_RD, &sc->stats.tx_frames_128to255b, 0,
1491             "Number frames transmitted of size 128-255 bytes");
1492
1493         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_256to511b",
1494             CTLFLAG_RD, &sc->stats.tx_frames_256to511b, 0,
1495             "Number frames transmitted of size 256-511 bytes");
1496
1497         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_512to1023b",
1498             CTLFLAG_RD, &sc->stats.tx_frames_512to1023b, 0,
1499             "Number frames transmitted of size 512-1023 bytes");
1500
1501         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_1024to1536b",
1502             CTLFLAG_RD, &sc->stats.tx_frames_1024to1536b, 0,
1503             "Number frames transmitted of size 1024-1536 bytes");
1504
1505         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_under_runs",
1506             CTLFLAG_RD, &sc->stats.tx_under_runs, 0,
1507             "Number transmit under-run events");
1508
1509         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_single_collisn",
1510             CTLFLAG_RD, &sc->stats.tx_single_collisn, 0,
1511             "Number single-collision transmit frames");
1512
1513         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_multi_collisn",
1514             CTLFLAG_RD, &sc->stats.tx_multi_collisn, 0,
1515             "Number multi-collision transmit frames");
1516
1517         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_excsv_collisn",
1518             CTLFLAG_RD, &sc->stats.tx_excsv_collisn, 0,
1519             "Number excessive collision transmit frames");
1520
1521         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_late_collisn",
1522             CTLFLAG_RD, &sc->stats.tx_late_collisn, 0,
1523             "Number late-collision transmit frames");
1524
1525         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_deferred_frames",
1526             CTLFLAG_RD, &sc->stats.tx_deferred_frames, 0,
1527             "Number deferred transmit frames");
1528
1529         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_carrier_sense_errs",
1530             CTLFLAG_RD, &sc->stats.tx_carrier_sense_errs, 0,
1531             "Number carrier sense errors on transmit");
1532
1533         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_bytes", CTLFLAG_RD,
1534             &sc->stats.rx_bytes, "Total bytes received");
1535
1536         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames", CTLFLAG_RD,
1537             &sc->stats.rx_frames, 0, "Total frames received");
1538
1539         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_bcast",
1540             CTLFLAG_RD, &sc->stats.rx_frames_bcast, 0,
1541             "Number broadcast frames received");
1542
1543         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_multi",
1544             CTLFLAG_RD, &sc->stats.rx_frames_multi, 0,
1545             "Number multicast frames received");
1546
1547         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_pause",
1548             CTLFLAG_RD, &sc->stats.rx_frames_pause, 0,
1549             "Number pause frames received");
1550
1551         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_64b",
1552             CTLFLAG_RD, &sc->stats.rx_frames_64b, 0,
1553             "Number frames received of size 64 bytes or less");
1554
1555         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_65to127b",
1556             CTLFLAG_RD, &sc->stats.rx_frames_65to127b, 0,
1557             "Number frames received of size 65-127 bytes");
1558
1559         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_128to255b",
1560             CTLFLAG_RD, &sc->stats.rx_frames_128to255b, 0,
1561             "Number frames received of size 128-255 bytes");
1562
1563         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_256to511b",
1564             CTLFLAG_RD, &sc->stats.rx_frames_256to511b, 0,
1565             "Number frames received of size 256-511 bytes");
1566
1567         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_512to1023b",
1568             CTLFLAG_RD, &sc->stats.rx_frames_512to1023b, 0,
1569             "Number frames received of size 512-1023 bytes");
1570
1571         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_1024to1536b",
1572             CTLFLAG_RD, &sc->stats.rx_frames_1024to1536b, 0,
1573             "Number frames received of size 1024-1536 bytes");
1574
1575         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_undersize",
1576             CTLFLAG_RD, &sc->stats.rx_frames_undersize, 0,
1577             "Number undersize frames received");
1578
1579         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_oversize",
1580             CTLFLAG_RD, &sc->stats.rx_frames_oversize, 0,
1581             "Number oversize frames received");
1582
1583         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_jabber",
1584             CTLFLAG_RD, &sc->stats.rx_frames_jabber, 0,
1585             "Number jabber frames received");
1586
1587         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_fcs_errs",
1588             CTLFLAG_RD, &sc->stats.rx_frames_fcs_errs, 0,
1589             "Number frames received with FCS errors");
1590
1591         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_length_errs",
1592             CTLFLAG_RD, &sc->stats.rx_frames_length_errs, 0,
1593             "Number frames received with length errors");
1594
1595         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_symbol_errs",
1596             CTLFLAG_RD, &sc->stats.rx_symbol_errs, 0,
1597             "Number receive symbol errors");
1598
1599         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_align_errs",
1600             CTLFLAG_RD, &sc->stats.rx_align_errs, 0,
1601             "Number receive alignment errors");
1602
1603         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_resource_errs",
1604             CTLFLAG_RD, &sc->stats.rx_resource_errs, 0,
1605             "Number frames received when no rx buffer available");
1606
1607         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_overrun_errs",
1608             CTLFLAG_RD, &sc->stats.rx_overrun_errs, 0,
1609             "Number frames received but not copied due to receive overrun");
1610
1611         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_ip_hdr_csum_errs",
1612             CTLFLAG_RD, &sc->stats.rx_ip_hdr_csum_errs, 0,
1613             "Number frames received with IP header checksum errors");
1614
1615         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_tcp_csum_errs",
1616             CTLFLAG_RD, &sc->stats.rx_tcp_csum_errs, 0,
1617             "Number frames received with TCP checksum errors");
1618
1619         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_udp_csum_errs",
1620             CTLFLAG_RD, &sc->stats.rx_udp_csum_errs, 0,
1621             "Number frames received with UDP checksum errors");
1622 }
1623
1624 static int
1625 cgem_probe(device_t dev)
1626 {
1627
1628         if (!ofw_bus_status_okay(dev))
1629                 return (ENXIO);
1630
1631         if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0)
1632                 return (ENXIO);
1633
1634         device_set_desc(dev, "Cadence CGEM Gigabit Ethernet Interface");
1635         return (0);
1636 }
1637
1638 static int
1639 cgem_attach(device_t dev)
1640 {
1641         struct cgem_softc *sc = device_get_softc(dev);
1642         if_t ifp = NULL;
1643         phandle_t node;
1644         pcell_t cell;
1645         int rid, err;
1646         u_char eaddr[ETHER_ADDR_LEN];
1647
1648         sc->dev = dev;
1649         CGEM_LOCK_INIT(sc);
1650
1651         /* Get reference clock number and base divider from fdt. */
1652         node = ofw_bus_get_node(dev);
1653         sc->ref_clk_num = 0;
1654         if (OF_getprop(node, "ref-clock-num", &cell, sizeof(cell)) > 0)
1655                 sc->ref_clk_num = fdt32_to_cpu(cell);
1656
1657         /* Get memory resource. */
1658         rid = 0;
1659         sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
1660             RF_ACTIVE);
1661         if (sc->mem_res == NULL) {
1662                 device_printf(dev, "could not allocate memory resources.\n");
1663                 return (ENOMEM);
1664         }
1665
1666         /* Get IRQ resource. */
1667         rid = 0;
1668         sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE);
1669         if (sc->irq_res == NULL) {
1670                 device_printf(dev, "could not allocate interrupt resource.\n");
1671                 cgem_detach(dev);
1672                 return (ENOMEM);
1673         }
1674
1675         /* Set up ifnet structure. */
1676         ifp = sc->ifp = if_alloc(IFT_ETHER);
1677         if (ifp == NULL) {
1678                 device_printf(dev, "could not allocate ifnet structure\n");
1679                 cgem_detach(dev);
1680                 return (ENOMEM);
1681         }
1682         if_setsoftc(ifp, sc);
1683         if_initname(ifp, IF_CGEM_NAME, device_get_unit(dev));
1684         if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
1685         if_setinitfn(ifp, cgem_init);
1686         if_setioctlfn(ifp, cgem_ioctl);
1687         if_setstartfn(ifp, cgem_start);
1688         if_setcapabilitiesbit(ifp, IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6 |
1689             IFCAP_VLAN_MTU | IFCAP_VLAN_HWCSUM, 0);
1690         if_setsendqlen(ifp, CGEM_NUM_TX_DESCS);
1691         if_setsendqready(ifp);
1692
1693         /* Disable hardware checksumming by default. */
1694         if_sethwassist(ifp, 0);
1695         if_setcapenable(ifp, if_getcapabilities(ifp) &
1696             ~(IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6 | IFCAP_VLAN_HWCSUM));
1697
1698         sc->if_old_flags = if_getflags(ifp);
1699         sc->rxbufs = DEFAULT_NUM_RX_BUFS;
1700         sc->rxhangwar = 1;
1701
1702         /* Reset hardware. */
1703         CGEM_LOCK(sc);
1704         cgem_reset(sc);
1705         CGEM_UNLOCK(sc);
1706
1707         /* Attach phy to mii bus. */
1708         err = mii_attach(dev, &sc->miibus, ifp,
1709             cgem_ifmedia_upd, cgem_ifmedia_sts, BMSR_DEFCAPMASK,
1710             MII_PHY_ANY, MII_OFFSET_ANY, 0);
1711         if (err) {
1712                 device_printf(dev, "attaching PHYs failed\n");
1713                 cgem_detach(dev);
1714                 return (err);
1715         }
1716
1717         /* Set up TX and RX descriptor area. */
1718         err = cgem_setup_descs(sc);
1719         if (err) {
1720                 device_printf(dev, "could not set up dma mem for descs.\n");
1721                 cgem_detach(dev);
1722                 return (ENOMEM);
1723         }
1724
1725         /* Get a MAC address. */
1726         cgem_get_mac(sc, eaddr);
1727
1728         /* Start ticks. */
1729         callout_init_mtx(&sc->tick_ch, &sc->sc_mtx, 0);
1730
1731         ether_ifattach(ifp, eaddr);
1732
1733         err = bus_setup_intr(dev, sc->irq_res, INTR_TYPE_NET | INTR_MPSAFE |
1734             INTR_EXCL, NULL, cgem_intr, sc, &sc->intrhand);
1735         if (err) {
1736                 device_printf(dev, "could not set interrupt handler.\n");
1737                 ether_ifdetach(ifp);
1738                 cgem_detach(dev);
1739                 return (err);
1740         }
1741
1742         cgem_add_sysctls(dev);
1743
1744         return (0);
1745 }
1746
1747 static int
1748 cgem_detach(device_t dev)
1749 {
1750         struct cgem_softc *sc = device_get_softc(dev);
1751         int i;
1752
1753         if (sc == NULL)
1754                 return (ENODEV);
1755
1756         if (device_is_attached(dev)) {
1757                 CGEM_LOCK(sc);
1758                 cgem_stop(sc);
1759                 CGEM_UNLOCK(sc);
1760                 callout_drain(&sc->tick_ch);
1761                 if_setflagbits(sc->ifp, 0, IFF_UP);
1762                 ether_ifdetach(sc->ifp);
1763         }
1764
1765         if (sc->miibus != NULL) {
1766                 device_delete_child(dev, sc->miibus);
1767                 sc->miibus = NULL;
1768         }
1769
1770         /* Release resources. */
1771         if (sc->mem_res != NULL) {
1772                 bus_release_resource(dev, SYS_RES_MEMORY,
1773                     rman_get_rid(sc->mem_res), sc->mem_res);
1774                 sc->mem_res = NULL;
1775         }
1776         if (sc->irq_res != NULL) {
1777                 if (sc->intrhand)
1778                         bus_teardown_intr(dev, sc->irq_res, sc->intrhand);
1779                 bus_release_resource(dev, SYS_RES_IRQ,
1780                     rman_get_rid(sc->irq_res), sc->irq_res);
1781                 sc->irq_res = NULL;
1782         }
1783
1784         /* Release DMA resources. */
1785         if (sc->rxring != NULL) {
1786                 if (sc->rxring_physaddr != 0) {
1787                         bus_dmamap_unload(sc->desc_dma_tag,
1788                             sc->rxring_dma_map);
1789                         sc->rxring_physaddr = 0;
1790                 }
1791                 bus_dmamem_free(sc->desc_dma_tag, sc->rxring,
1792                                 sc->rxring_dma_map);
1793                 sc->rxring = NULL;
1794                 for (i = 0; i < CGEM_NUM_RX_DESCS; i++)
1795                         if (sc->rxring_m_dmamap[i] != NULL) {
1796                                 bus_dmamap_destroy(sc->mbuf_dma_tag,
1797                                     sc->rxring_m_dmamap[i]);
1798                                 sc->rxring_m_dmamap[i] = NULL;
1799                         }
1800         }
1801         if (sc->txring != NULL) {
1802                 if (sc->txring_physaddr != 0) {
1803                         bus_dmamap_unload(sc->desc_dma_tag,
1804                             sc->txring_dma_map);
1805                         sc->txring_physaddr = 0;
1806                 }
1807                 bus_dmamem_free(sc->desc_dma_tag, sc->txring,
1808                                 sc->txring_dma_map);
1809                 sc->txring = NULL;
1810                 for (i = 0; i < CGEM_NUM_TX_DESCS; i++)
1811                         if (sc->txring_m_dmamap[i] != NULL) {
1812                                 bus_dmamap_destroy(sc->mbuf_dma_tag,
1813                                     sc->txring_m_dmamap[i]);
1814                                 sc->txring_m_dmamap[i] = NULL;
1815                         }
1816         }
1817         if (sc->desc_dma_tag != NULL) {
1818                 bus_dma_tag_destroy(sc->desc_dma_tag);
1819                 sc->desc_dma_tag = NULL;
1820         }
1821         if (sc->mbuf_dma_tag != NULL) {
1822                 bus_dma_tag_destroy(sc->mbuf_dma_tag);
1823                 sc->mbuf_dma_tag = NULL;
1824         }
1825
1826         bus_generic_detach(dev);
1827
1828         CGEM_LOCK_DESTROY(sc);
1829
1830         return (0);
1831 }
1832
1833 static device_method_t cgem_methods[] = {
1834         /* Device interface */
1835         DEVMETHOD(device_probe,         cgem_probe),
1836         DEVMETHOD(device_attach,        cgem_attach),
1837         DEVMETHOD(device_detach,        cgem_detach),
1838
1839         /* Bus interface */
1840         DEVMETHOD(bus_child_detached,   cgem_child_detached),
1841
1842         /* MII interface */
1843         DEVMETHOD(miibus_readreg,       cgem_miibus_readreg),
1844         DEVMETHOD(miibus_writereg,      cgem_miibus_writereg),
1845         DEVMETHOD(miibus_statchg,       cgem_miibus_statchg),
1846         DEVMETHOD(miibus_linkchg,       cgem_miibus_linkchg),
1847
1848         DEVMETHOD_END
1849 };
1850
1851 static driver_t cgem_driver = {
1852         "cgem",
1853         cgem_methods,
1854         sizeof(struct cgem_softc),
1855 };
1856
1857 DRIVER_MODULE(cgem, simplebus, cgem_driver, cgem_devclass, NULL, NULL);
1858 DRIVER_MODULE(miibus, cgem, miibus_driver, miibus_devclass, NULL, NULL);
1859 MODULE_DEPEND(cgem, miibus, 1, 1, 1);
1860 MODULE_DEPEND(cgem, ether, 1, 1, 1);