sys/dev/cadence/if_cgem.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2012-2014 Thomas Skibo <thomasskibo@yahoo.com>
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28
  29 /*
  30  * A network interface driver for Cadence GEM Gigabit Ethernet
  31  * interface such as the one used in Xilinx Zynq-7000 SoC.
  32  *
  33  * Reference: Zynq-7000 All Programmable SoC Technical Reference Manual.
  34  * (v1.4) November 16, 2012.  Xilinx doc UG585.  GEM is covered in Ch. 16
  35  * and register definitions are in appendix B.18.
  36  */
  37
  38 #include <sys/cdefs.h>
  39 __FBSDID("$FreeBSD$");
  40
  41 #include <sys/param.h>
  42 #include <sys/systm.h>
  43 #include <sys/bus.h>
  44 #include <sys/kernel.h>
  45 #include <sys/malloc.h>
  46 #include <sys/mbuf.h>
  47 #include <sys/module.h>
  48 #include <sys/rman.h>
  49 #include <sys/socket.h>
  50 #include <sys/sockio.h>
  51 #include <sys/sysctl.h>
  52
  53 #include <machine/bus.h>
  54
  55 #include <net/ethernet.h>
  56 #include <net/if.h>
  57 #include <net/if_arp.h>
  58 #include <net/if_dl.h>
  59 #include <net/if_media.h>
  60 #include <net/if_mib.h>
  61 #include <net/if_types.h>
  62
  63 #ifdef INET
  64 #include <netinet/in.h>
  65 #include <netinet/in_systm.h>
  66 #include <netinet/in_var.h>
  67 #include <netinet/ip.h>
  68 #endif
  69
  70 #include <net/bpf.h>
  71 #include <net/bpfdesc.h>
  72
  73 #include <dev/fdt/fdt_common.h>
  74 #include <dev/ofw/ofw_bus.h>
  75 #include <dev/ofw/ofw_bus_subr.h>
  76
  77 #include <dev/mii/mii.h>
  78 #include <dev/mii/miivar.h>
  79
  80 #include <dev/cadence/if_cgem_hw.h>
  81
  82 #include "miibus_if.h"
  83
  84 #define IF_CGEM_NAME "cgem"
  85
  86 #define CGEM_NUM_RX_DESCS       512     /* size of receive descriptor ring */
  87 #define CGEM_NUM_TX_DESCS       512     /* size of transmit descriptor ring */
  88
  89 #define MAX_DESC_RING_SIZE (MAX(CGEM_NUM_RX_DESCS*sizeof(struct cgem_rx_desc),\
  90                                 CGEM_NUM_TX_DESCS*sizeof(struct cgem_tx_desc)))
  91
  92
  93 /* Default for sysctl rxbufs.  Must be < CGEM_NUM_RX_DESCS of course. */
  94 #define DEFAULT_NUM_RX_BUFS     256     /* number of receive bufs to queue. */
  95
  96 #define TX_MAX_DMA_SEGS         8       /* maximum segs in a tx mbuf dma */
  97
  98 #define CGEM_CKSUM_ASSIST       (CSUM_IP | CSUM_TCP | CSUM_UDP | \
  99                                  CSUM_TCP_IPV6 | CSUM_UDP_IPV6)
 100
 101 static struct ofw_compat_data compat_data[] = {
 102         { "cadence,gem",        1 },
 103         { "cdns,macb",          1 },
 104         { NULL,                 0 },
 105 };
 106
 107 struct cgem_softc {
 108         if_t                    ifp;
 109         struct mtx              sc_mtx;
 110         device_t                dev;
 111         device_t                miibus;
 112         u_int                   mii_media_active;       /* last active media */
 113         int                     if_old_flags;
 114         struct resource         *mem_res;
 115         struct resource         *irq_res;
 116         void                    *intrhand;
 117         struct callout          tick_ch;
 118         uint32_t                net_ctl_shadow;
 119         int                     ref_clk_num;
 120         u_char                  eaddr[6];
 121
 122         bus_dma_tag_t           desc_dma_tag;
 123         bus_dma_tag_t           mbuf_dma_tag;
 124
 125         /* receive descriptor ring */
 126         struct cgem_rx_desc     *rxring;
 127         bus_addr_t              rxring_physaddr;
 128         struct mbuf             *rxring_m[CGEM_NUM_RX_DESCS];
 129         bus_dmamap_t            rxring_m_dmamap[CGEM_NUM_RX_DESCS];
 130         int                     rxring_hd_ptr;  /* where to put rcv bufs */
 131         int                     rxring_tl_ptr;  /* where to get receives */
 132         int                     rxring_queued;  /* how many rcv bufs queued */
 133         bus_dmamap_t            rxring_dma_map;
 134         int                     rxbufs;         /* tunable number rcv bufs */
 135         int                     rxhangwar;      /* rx hang work-around */
 136         u_int                   rxoverruns;     /* rx overruns */
 137         u_int                   rxnobufs;       /* rx buf ring empty events */
 138         u_int                   rxdmamapfails;  /* rx dmamap failures */
 139         uint32_t                rx_frames_prev;
 140
 141         /* transmit descriptor ring */
 142         struct cgem_tx_desc     *txring;
 143         bus_addr_t              txring_physaddr;
 144         struct mbuf             *txring_m[CGEM_NUM_TX_DESCS];
 145         bus_dmamap_t            txring_m_dmamap[CGEM_NUM_TX_DESCS];
 146         int                     txring_hd_ptr;  /* where to put next xmits */
 147         int                     txring_tl_ptr;  /* next xmit mbuf to free */
 148         int                     txring_queued;  /* num xmits segs queued */
 149         bus_dmamap_t            txring_dma_map;
 150         u_int                   txfull;         /* tx ring full events */
 151         u_int                   txdefrags;      /* tx calls to m_defrag() */
 152         u_int                   txdefragfails;  /* tx m_defrag() failures */
 153         u_int                   txdmamapfails;  /* tx dmamap failures */
 154
 155         /* hardware provided statistics */
 156         struct cgem_hw_stats {
 157                 uint64_t                tx_bytes;
 158                 uint32_t                tx_frames;
 159                 uint32_t                tx_frames_bcast;
 160                 uint32_t                tx_frames_multi;
 161                 uint32_t                tx_frames_pause;
 162                 uint32_t                tx_frames_64b;
 163                 uint32_t                tx_frames_65to127b;
 164                 uint32_t                tx_frames_128to255b;
 165                 uint32_t                tx_frames_256to511b;
 166                 uint32_t                tx_frames_512to1023b;
 167                 uint32_t                tx_frames_1024to1536b;
 168                 uint32_t                tx_under_runs;
 169                 uint32_t                tx_single_collisn;
 170                 uint32_t                tx_multi_collisn;
 171                 uint32_t                tx_excsv_collisn;
 172                 uint32_t                tx_late_collisn;
 173                 uint32_t                tx_deferred_frames;
 174                 uint32_t                tx_carrier_sense_errs;
 175
 176                 uint64_t                rx_bytes;
 177                 uint32_t                rx_frames;
 178                 uint32_t                rx_frames_bcast;
 179                 uint32_t                rx_frames_multi;
 180                 uint32_t                rx_frames_pause;
 181                 uint32_t                rx_frames_64b;
 182                 uint32_t                rx_frames_65to127b;
 183                 uint32_t                rx_frames_128to255b;
 184                 uint32_t                rx_frames_256to511b;
 185                 uint32_t                rx_frames_512to1023b;
 186                 uint32_t                rx_frames_1024to1536b;
 187                 uint32_t                rx_frames_undersize;
 188                 uint32_t                rx_frames_oversize;
 189                 uint32_t                rx_frames_jabber;
 190                 uint32_t                rx_frames_fcs_errs;
 191                 uint32_t                rx_frames_length_errs;
 192                 uint32_t                rx_symbol_errs;
 193                 uint32_t                rx_align_errs;
 194                 uint32_t                rx_resource_errs;
 195                 uint32_t                rx_overrun_errs;
 196                 uint32_t                rx_ip_hdr_csum_errs;
 197                 uint32_t                rx_tcp_csum_errs;
 198                 uint32_t                rx_udp_csum_errs;
 199         } stats;
 200 };
 201
 202 #define RD4(sc, off)            (bus_read_4((sc)->mem_res, (off)))
 203 #define WR4(sc, off, val)       (bus_write_4((sc)->mem_res, (off), (val)))
 204 #define BARRIER(sc, off, len, flags) \
 205         (bus_barrier((sc)->mem_res, (off), (len), (flags))
 206
 207 #define CGEM_LOCK(sc)           mtx_lock(&(sc)->sc_mtx)
 208 #define CGEM_UNLOCK(sc) mtx_unlock(&(sc)->sc_mtx)
 209 #define CGEM_LOCK_INIT(sc)      \
 210         mtx_init(&(sc)->sc_mtx, device_get_nameunit((sc)->dev), \
 211                  MTX_NETWORK_LOCK, MTX_DEF)
 212 #define CGEM_LOCK_DESTROY(sc)   mtx_destroy(&(sc)->sc_mtx)
 213 #define CGEM_ASSERT_LOCKED(sc)  mtx_assert(&(sc)->sc_mtx, MA_OWNED)
 214
 215 /* Allow platforms to optionally provide a way to set the reference clock. */
 216 int cgem_set_ref_clk(int unit, int frequency);
 217
 218 static devclass_t cgem_devclass;
 219
 220 static int cgem_probe(device_t dev);
 221 static int cgem_attach(device_t dev);
 222 static int cgem_detach(device_t dev);
 223 static void cgem_tick(void *);
 224 static void cgem_intr(void *);
 225
 226 static void cgem_mediachange(struct cgem_softc *, struct mii_data *);
 227
 228 static void
 229 cgem_get_mac(struct cgem_softc *sc, u_char eaddr[])
 230 {
 231         int i;
 232         uint32_t rnd;
 233
 234         /* See if boot loader gave us a MAC address already. */
 235         for (i = 0; i < 4; i++) {
 236                 uint32_t low = RD4(sc, CGEM_SPEC_ADDR_LOW(i));
 237                 uint32_t high = RD4(sc, CGEM_SPEC_ADDR_HI(i)) & 0xffff;
 238                 if (low != 0 || high != 0) {
 239                         eaddr[0] = low & 0xff;
 240                         eaddr[1] = (low >> 8) & 0xff;
 241                         eaddr[2] = (low >> 16) & 0xff;
 242                         eaddr[3] = (low >> 24) & 0xff;
 243                         eaddr[4] = high & 0xff;
 244                         eaddr[5] = (high >> 8) & 0xff;
 245                         break;
 246                 }
 247         }
 248
 249         /* No MAC from boot loader?  Assign a random one. */
 250         if (i == 4) {
 251                 rnd = arc4random();
 252
 253                 eaddr[0] = 'b';
 254                 eaddr[1] = 's';
 255                 eaddr[2] = 'd';
 256                 eaddr[3] = (rnd >> 16) & 0xff;
 257                 eaddr[4] = (rnd >> 8) & 0xff;
 258                 eaddr[5] = rnd & 0xff;
 259
 260                 device_printf(sc->dev, "no mac address found, assigning "
 261                               "random: %02x:%02x:%02x:%02x:%02x:%02x\n",
 262                               eaddr[0], eaddr[1], eaddr[2],
 263                               eaddr[3], eaddr[4], eaddr[5]);
 264         }
 265
 266         /* Move address to first slot and zero out the rest. */
 267         WR4(sc, CGEM_SPEC_ADDR_LOW(0), (eaddr[3] << 24) |
 268             (eaddr[2] << 16) | (eaddr[1] << 8) | eaddr[0]);
 269         WR4(sc, CGEM_SPEC_ADDR_HI(0), (eaddr[5] << 8) | eaddr[4]);
 270
 271         for (i = 1; i < 4; i++) {
 272                 WR4(sc, CGEM_SPEC_ADDR_LOW(i), 0);
 273                 WR4(sc, CGEM_SPEC_ADDR_HI(i), 0);
 274         }
 275 }
 276
 277 /* cgem_mac_hash():  map 48-bit address to a 6-bit hash.
 278  * The 6-bit hash corresponds to a bit in a 64-bit hash
 279  * register.  Setting that bit in the hash register enables
 280  * reception of all frames with a destination address that hashes
 281  * to that 6-bit value.
 282  *
 283  * The hash function is described in sec. 16.2.3 in the Zynq-7000 Tech
 284  * Reference Manual.  Bits 0-5 in the hash are the exclusive-or of
 285  * every sixth bit in the destination address.
 286  */
 287 static int
 288 cgem_mac_hash(u_char eaddr[])
 289 {
 290         int hash;
 291         int i, j;
 292
 293         hash = 0;
 294         for (i = 0; i < 6; i++)
 295                 for (j = i; j < 48; j += 6)
 296                         if ((eaddr[j >> 3] & (1 << (j & 7))) != 0)
 297                                 hash ^= (1 << i);
 298
 299         return hash;
 300 }
 301
 302 static u_int
 303 cgem_hash_maddr(void *arg, struct sockaddr_dl *sdl, u_int cnt)
 304 {
 305         uint32_t *hashes = arg;
 306         int index;
 307
 308         index = cgem_mac_hash(LLADDR(sdl));
 309         if (index > 31)
 310                 hashes[0] |= (1 << (index - 32));
 311         else
 312                 hashes[1] |= (1 << index);
 313
 314         return (1);
 315 }
 316
 317 /* After any change in rx flags or multi-cast addresses, set up
 318  * hash registers and net config register bits.
 319  */
 320 static void
 321 cgem_rx_filter(struct cgem_softc *sc)
 322 {
 323         if_t ifp = sc->ifp;
 324         uint32_t hashes[2] = { 0, 0 };
 325         uint32_t net_cfg;
 326
 327         net_cfg = RD4(sc, CGEM_NET_CFG);
 328
 329         net_cfg &= ~(CGEM_NET_CFG_MULTI_HASH_EN |
 330                      CGEM_NET_CFG_NO_BCAST |
 331                      CGEM_NET_CFG_COPY_ALL);
 332
 333         if ((if_getflags(ifp) & IFF_PROMISC) != 0)
 334                 net_cfg |= CGEM_NET_CFG_COPY_ALL;
 335         else {
 336                 if ((if_getflags(ifp) & IFF_BROADCAST) == 0)
 337                         net_cfg |= CGEM_NET_CFG_NO_BCAST;
 338                 if ((if_getflags(ifp) & IFF_ALLMULTI) != 0) {
 339                         hashes[0] = 0xffffffff;
 340                         hashes[1] = 0xffffffff;
 341                 } else
 342                         if_foreach_llmaddr(ifp, cgem_hash_maddr, hashes);
 343
 344                 if (hashes[0] != 0 || hashes[1] != 0)
 345                         net_cfg |= CGEM_NET_CFG_MULTI_HASH_EN;
 346         }
 347
 348         WR4(sc, CGEM_HASH_TOP, hashes[0]);
 349         WR4(sc, CGEM_HASH_BOT, hashes[1]);
 350         WR4(sc, CGEM_NET_CFG, net_cfg);
 351 }
 352
 353 /* For bus_dmamap_load() callback. */
 354 static void
 355 cgem_getaddr(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
 356 {
 357
 358         if (nsegs != 1 || error != 0)
 359                 return;
 360         *(bus_addr_t *)arg = segs[0].ds_addr;
 361 }
 362
 363 /* Create DMA'able descriptor rings. */
 364 static int
 365 cgem_setup_descs(struct cgem_softc *sc)
 366 {
 367         int i, err;
 368
 369         sc->txring = NULL;
 370         sc->rxring = NULL;
 371
 372         /* Allocate non-cached DMA space for RX and TX descriptors.
 373          */
 374         err = bus_dma_tag_create(bus_get_dma_tag(sc->dev), 1, 0,
 375                                  BUS_SPACE_MAXADDR_32BIT,
 376                                  BUS_SPACE_MAXADDR,
 377                                  NULL, NULL,
 378                                  MAX_DESC_RING_SIZE,
 379                                  1,
 380                                  MAX_DESC_RING_SIZE,
 381                                  0,
 382                                  busdma_lock_mutex,
 383                                  &sc->sc_mtx,
 384                                  &sc->desc_dma_tag);
 385         if (err)
 386                 return (err);
 387
 388         /* Set up a bus_dma_tag for mbufs. */
 389         err = bus_dma_tag_create(bus_get_dma_tag(sc->dev), 1, 0,
 390                                  BUS_SPACE_MAXADDR_32BIT,
 391                                  BUS_SPACE_MAXADDR,
 392                                  NULL, NULL,
 393                                  MCLBYTES,
 394                                  TX_MAX_DMA_SEGS,
 395                                  MCLBYTES,
 396                                  0,
 397                                  busdma_lock_mutex,
 398                                  &sc->sc_mtx,
 399                                  &sc->mbuf_dma_tag);
 400         if (err)
 401                 return (err);
 402
 403         /* Allocate DMA memory in non-cacheable space. */
 404         err = bus_dmamem_alloc(sc->desc_dma_tag,
 405                                (void **)&sc->rxring,
 406                                BUS_DMA_NOWAIT | BUS_DMA_COHERENT,
 407                                &sc->rxring_dma_map);
 408         if (err)
 409                 return (err);
 410
 411         /* Load descriptor DMA memory. */
 412         err = bus_dmamap_load(sc->desc_dma_tag, sc->rxring_dma_map,
 413                               (void *)sc->rxring,
 414                               CGEM_NUM_RX_DESCS*sizeof(struct cgem_rx_desc),
 415                               cgem_getaddr, &sc->rxring_physaddr,
 416                               BUS_DMA_NOWAIT);
 417         if (err)
 418                 return (err);
 419
 420         /* Initialize RX descriptors. */
 421         for (i = 0; i < CGEM_NUM_RX_DESCS; i++) {
 422                 sc->rxring[i].addr = CGEM_RXDESC_OWN;
 423                 sc->rxring[i].ctl = 0;
 424                 sc->rxring_m[i] = NULL;
 425                 sc->rxring_m_dmamap[i] = NULL;
 426         }
 427         sc->rxring[CGEM_NUM_RX_DESCS - 1].addr |= CGEM_RXDESC_WRAP;
 428
 429         sc->rxring_hd_ptr = 0;
 430         sc->rxring_tl_ptr = 0;
 431         sc->rxring_queued = 0;
 432
 433         /* Allocate DMA memory for TX descriptors in non-cacheable space. */
 434         err = bus_dmamem_alloc(sc->desc_dma_tag,
 435                                (void **)&sc->txring,
 436                                BUS_DMA_NOWAIT | BUS_DMA_COHERENT,
 437                                &sc->txring_dma_map);
 438         if (err)
 439                 return (err);
 440
 441         /* Load TX descriptor DMA memory. */
 442         err = bus_dmamap_load(sc->desc_dma_tag, sc->txring_dma_map,
 443                               (void *)sc->txring,
 444                               CGEM_NUM_TX_DESCS*sizeof(struct cgem_tx_desc),
 445                               cgem_getaddr, &sc->txring_physaddr,
 446                               BUS_DMA_NOWAIT);
 447         if (err)
 448                 return (err);
 449
 450         /* Initialize TX descriptor ring. */
 451         for (i = 0; i < CGEM_NUM_TX_DESCS; i++) {
 452                 sc->txring[i].addr = 0;
 453                 sc->txring[i].ctl = CGEM_TXDESC_USED;
 454                 sc->txring_m[i] = NULL;
 455                 sc->txring_m_dmamap[i] = NULL;
 456         }
 457         sc->txring[CGEM_NUM_TX_DESCS - 1].ctl |= CGEM_TXDESC_WRAP;
 458
 459         sc->txring_hd_ptr = 0;
 460         sc->txring_tl_ptr = 0;
 461         sc->txring_queued = 0;
 462
 463         return (0);
 464 }
 465
 466 /* Fill receive descriptor ring with mbufs. */
 467 static void
 468 cgem_fill_rqueue(struct cgem_softc *sc)
 469 {
 470         struct mbuf *m = NULL;
 471         bus_dma_segment_t segs[TX_MAX_DMA_SEGS];
 472         int nsegs;
 473
 474         CGEM_ASSERT_LOCKED(sc);
 475
 476         while (sc->rxring_queued < sc->rxbufs) {
 477                 /* Get a cluster mbuf. */
 478                 m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
 479                 if (m == NULL)
 480                         break;
 481
 482                 m->m_len = MCLBYTES;
 483                 m->m_pkthdr.len = MCLBYTES;
 484                 m->m_pkthdr.rcvif = sc->ifp;
 485
 486                 /* Load map and plug in physical address. */
 487                 if (bus_dmamap_create(sc->mbuf_dma_tag, 0,
 488                               &sc->rxring_m_dmamap[sc->rxring_hd_ptr])) {
 489                         sc->rxdmamapfails++;
 490                         m_free(m);
 491                         break;
 492                 }
 493                 if (bus_dmamap_load_mbuf_sg(sc->mbuf_dma_tag,
 494                               sc->rxring_m_dmamap[sc->rxring_hd_ptr], m,
 495                               segs, &nsegs, BUS_DMA_NOWAIT)) {
 496                         sc->rxdmamapfails++;
 497                         bus_dmamap_destroy(sc->mbuf_dma_tag,
 498                                    sc->rxring_m_dmamap[sc->rxring_hd_ptr]);
 499                         sc->rxring_m_dmamap[sc->rxring_hd_ptr] = NULL;
 500                         m_free(m);
 501                         break;
 502                 }
 503                 sc->rxring_m[sc->rxring_hd_ptr] = m;
 504
 505                 /* Sync cache with receive buffer. */
 506                 bus_dmamap_sync(sc->mbuf_dma_tag,
 507                                 sc->rxring_m_dmamap[sc->rxring_hd_ptr],
 508                                 BUS_DMASYNC_PREREAD);
 509
 510                 /* Write rx descriptor and increment head pointer. */
 511                 sc->rxring[sc->rxring_hd_ptr].ctl = 0;
 512                 if (sc->rxring_hd_ptr == CGEM_NUM_RX_DESCS - 1) {
 513                         sc->rxring[sc->rxring_hd_ptr].addr = segs[0].ds_addr |
 514                                 CGEM_RXDESC_WRAP;
 515                         sc->rxring_hd_ptr = 0;
 516                 } else
 517                         sc->rxring[sc->rxring_hd_ptr++].addr = segs[0].ds_addr;
 518
 519                 sc->rxring_queued++;
 520         }
 521 }
 522
 523 /* Pull received packets off of receive descriptor ring. */
 524 static void
 525 cgem_recv(struct cgem_softc *sc)
 526 {
 527         if_t ifp = sc->ifp;
 528         struct mbuf *m, *m_hd, **m_tl;
 529         uint32_t ctl;
 530
 531         CGEM_ASSERT_LOCKED(sc);
 532
 533         /* Pick up all packets in which the OWN bit is set. */
 534         m_hd = NULL;
 535         m_tl = &m_hd;
 536         while (sc->rxring_queued > 0 &&
 537                (sc->rxring[sc->rxring_tl_ptr].addr & CGEM_RXDESC_OWN) != 0) {
 538
 539                 ctl = sc->rxring[sc->rxring_tl_ptr].ctl;
 540
 541                 /* Grab filled mbuf. */
 542                 m = sc->rxring_m[sc->rxring_tl_ptr];
 543                 sc->rxring_m[sc->rxring_tl_ptr] = NULL;
 544
 545                 /* Sync cache with receive buffer. */
 546                 bus_dmamap_sync(sc->mbuf_dma_tag,
 547                                 sc->rxring_m_dmamap[sc->rxring_tl_ptr],
 548                                 BUS_DMASYNC_POSTREAD);
 549
 550                 /* Unload and destroy dmamap. */
 551                 bus_dmamap_unload(sc->mbuf_dma_tag,
 552                         sc->rxring_m_dmamap[sc->rxring_tl_ptr]);
 553                 bus_dmamap_destroy(sc->mbuf_dma_tag,
 554                                    sc->rxring_m_dmamap[sc->rxring_tl_ptr]);
 555                 sc->rxring_m_dmamap[sc->rxring_tl_ptr] = NULL;
 556
 557                 /* Increment tail pointer. */
 558                 if (++sc->rxring_tl_ptr == CGEM_NUM_RX_DESCS)
 559                         sc->rxring_tl_ptr = 0;
 560                 sc->rxring_queued--;
 561
 562                 /* Check FCS and make sure entire packet landed in one mbuf
 563                  * cluster (which is much bigger than the largest ethernet
 564                  * packet).
 565                  */
 566                 if ((ctl & CGEM_RXDESC_BAD_FCS) != 0 ||
 567                     (ctl & (CGEM_RXDESC_SOF | CGEM_RXDESC_EOF)) !=
 568                            (CGEM_RXDESC_SOF | CGEM_RXDESC_EOF)) {
 569                         /* discard. */
 570                         m_free(m);
 571                         if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
 572                         continue;
 573                 }
 574
 575                 /* Ready it to hand off to upper layers. */
 576                 m->m_data += ETHER_ALIGN;
 577                 m->m_len = (ctl & CGEM_RXDESC_LENGTH_MASK);
 578                 m->m_pkthdr.rcvif = ifp;
 579                 m->m_pkthdr.len = m->m_len;
 580
 581                 /* Are we using hardware checksumming?  Check the
 582                  * status in the receive descriptor.
 583                  */
 584                 if ((if_getcapenable(ifp) & IFCAP_RXCSUM) != 0) {
 585                         /* TCP or UDP checks out, IP checks out too. */
 586                         if ((ctl & CGEM_RXDESC_CKSUM_STAT_MASK) ==
 587                             CGEM_RXDESC_CKSUM_STAT_TCP_GOOD ||
 588                             (ctl & CGEM_RXDESC_CKSUM_STAT_MASK) ==
 589                             CGEM_RXDESC_CKSUM_STAT_UDP_GOOD) {
 590                                 m->m_pkthdr.csum_flags |=
 591                                         CSUM_IP_CHECKED | CSUM_IP_VALID |
 592                                         CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
 593                                 m->m_pkthdr.csum_data = 0xffff;
 594                         } else if ((ctl & CGEM_RXDESC_CKSUM_STAT_MASK) ==
 595                                    CGEM_RXDESC_CKSUM_STAT_IP_GOOD) {
 596                                 /* Only IP checks out. */
 597                                 m->m_pkthdr.csum_flags |=
 598                                         CSUM_IP_CHECKED | CSUM_IP_VALID;
 599                                 m->m_pkthdr.csum_data = 0xffff;
 600                         }
 601                 }
 602
 603                 /* Queue it up for delivery below. */
 604                 *m_tl = m;
 605                 m_tl = &m->m_next;
 606         }
 607
 608         /* Replenish receive buffers. */
 609         cgem_fill_rqueue(sc);
 610
 611         /* Unlock and send up packets. */
 612         CGEM_UNLOCK(sc);
 613         while (m_hd != NULL) {
 614                 m = m_hd;
 615                 m_hd = m_hd->m_next;
 616                 m->m_next = NULL;
 617                 if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);
 618                 if_input(ifp, m);
 619         }
 620         CGEM_LOCK(sc);
 621 }
 622
 623 /* Find completed transmits and free their mbufs. */
 624 static void
 625 cgem_clean_tx(struct cgem_softc *sc)
 626 {
 627         struct mbuf *m;
 628         uint32_t ctl;
 629
 630         CGEM_ASSERT_LOCKED(sc);
 631
 632         /* free up finished transmits. */
 633         while (sc->txring_queued > 0 &&
 634                ((ctl = sc->txring[sc->txring_tl_ptr].ctl) &
 635                 CGEM_TXDESC_USED) != 0) {
 636
 637                 /* Sync cache. */
 638                 bus_dmamap_sync(sc->mbuf_dma_tag,
 639                                 sc->txring_m_dmamap[sc->txring_tl_ptr],
 640                                 BUS_DMASYNC_POSTWRITE);
 641
 642                 /* Unload and destroy DMA map. */
 643                 bus_dmamap_unload(sc->mbuf_dma_tag,
 644                                   sc->txring_m_dmamap[sc->txring_tl_ptr]);
 645                 bus_dmamap_destroy(sc->mbuf_dma_tag,
 646                                    sc->txring_m_dmamap[sc->txring_tl_ptr]);
 647                 sc->txring_m_dmamap[sc->txring_tl_ptr] = NULL;
 648
 649                 /* Free up the mbuf. */
 650                 m = sc->txring_m[sc->txring_tl_ptr];
 651                 sc->txring_m[sc->txring_tl_ptr] = NULL;
 652                 m_freem(m);
 653
 654                 /* Check the status. */
 655                 if ((ctl & CGEM_TXDESC_AHB_ERR) != 0) {
 656                         /* Serious bus error. log to console. */
 657                         device_printf(sc->dev, "cgem_clean_tx: Whoa! "
 658                                    "AHB error, addr=0x%x\n",
 659                                    sc->txring[sc->txring_tl_ptr].addr);
 660                 } else if ((ctl & (CGEM_TXDESC_RETRY_ERR |
 661                                    CGEM_TXDESC_LATE_COLL)) != 0) {
 662                         if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1);
 663                 } else
 664                         if_inc_counter(sc->ifp, IFCOUNTER_OPACKETS, 1);
 665
 666                 /* If the packet spanned more than one tx descriptor,
 667                  * skip descriptors until we find the end so that only
 668                  * start-of-frame descriptors are processed.
 669                  */
 670                 while ((ctl & CGEM_TXDESC_LAST_BUF) == 0) {
 671                         if ((ctl & CGEM_TXDESC_WRAP) != 0)
 672                                 sc->txring_tl_ptr = 0;
 673                         else
 674                                 sc->txring_tl_ptr++;
 675                         sc->txring_queued--;
 676
 677                         ctl = sc->txring[sc->txring_tl_ptr].ctl;
 678
 679                         sc->txring[sc->txring_tl_ptr].ctl =
 680                                 ctl | CGEM_TXDESC_USED;
 681                 }
 682
 683                 /* Next descriptor. */
 684                 if ((ctl & CGEM_TXDESC_WRAP) != 0)
 685                         sc->txring_tl_ptr = 0;
 686                 else
 687                         sc->txring_tl_ptr++;
 688                 sc->txring_queued--;
 689
 690                 if_setdrvflagbits(sc->ifp, 0, IFF_DRV_OACTIVE);
 691         }
 692 }
 693
 694 /* Start transmits. */
 695 static void
 696 cgem_start_locked(if_t ifp)
 697 {
 698         struct cgem_softc *sc = (struct cgem_softc *) if_getsoftc(ifp);
 699         struct mbuf *m;
 700         bus_dma_segment_t segs[TX_MAX_DMA_SEGS];
 701         uint32_t ctl;
 702         int i, nsegs, wrap, err;
 703
 704         CGEM_ASSERT_LOCKED(sc);
 705
 706         if ((if_getdrvflags(ifp) & IFF_DRV_OACTIVE) != 0)
 707                 return;
 708
 709         for (;;) {
 710                 /* Check that there is room in the descriptor ring. */
 711                 if (sc->txring_queued >=
 712                     CGEM_NUM_TX_DESCS - TX_MAX_DMA_SEGS * 2) {
 713
 714                         /* Try to make room. */
 715                         cgem_clean_tx(sc);
 716
 717                         /* Still no room? */
 718                         if (sc->txring_queued >=
 719                             CGEM_NUM_TX_DESCS - TX_MAX_DMA_SEGS * 2) {
 720                                 if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0);
 721                                 sc->txfull++;
 722                                 break;
 723                         }
 724                 }
 725
 726                 /* Grab next transmit packet. */
 727                 m = if_dequeue(ifp);
 728                 if (m == NULL)
 729                         break;
 730
 731                 /* Create and load DMA map. */
 732                 if (bus_dmamap_create(sc->mbuf_dma_tag, 0,
 733                               &sc->txring_m_dmamap[sc->txring_hd_ptr])) {
 734                         m_freem(m);
 735                         sc->txdmamapfails++;
 736                         continue;
 737                 }
 738                 err = bus_dmamap_load_mbuf_sg(sc->mbuf_dma_tag,
 739                                       sc->txring_m_dmamap[sc->txring_hd_ptr],
 740                                       m, segs, &nsegs, BUS_DMA_NOWAIT);
 741                 if (err == EFBIG) {
 742                         /* Too many segments!  defrag and try again. */
 743                         struct mbuf *m2 = m_defrag(m, M_NOWAIT);
 744
 745                         if (m2 == NULL) {
 746                                 sc->txdefragfails++;
 747                                 m_freem(m);
 748                                 bus_dmamap_destroy(sc->mbuf_dma_tag,
 749                                    sc->txring_m_dmamap[sc->txring_hd_ptr]);
 750                                 sc->txring_m_dmamap[sc->txring_hd_ptr] = NULL;
 751                                 continue;
 752                         }
 753                         m = m2;
 754                         err = bus_dmamap_load_mbuf_sg(sc->mbuf_dma_tag,
 755                                       sc->txring_m_dmamap[sc->txring_hd_ptr],
 756                                       m, segs, &nsegs, BUS_DMA_NOWAIT);
 757                         sc->txdefrags++;
 758                 }
 759                 if (err) {
 760                         /* Give up. */
 761                         m_freem(m);
 762                         bus_dmamap_destroy(sc->mbuf_dma_tag,
 763                                    sc->txring_m_dmamap[sc->txring_hd_ptr]);
 764                         sc->txring_m_dmamap[sc->txring_hd_ptr] = NULL;
 765                         sc->txdmamapfails++;
 766                         continue;
 767                 }
 768                 sc->txring_m[sc->txring_hd_ptr] = m;
 769
 770                 /* Sync tx buffer with cache. */
 771                 bus_dmamap_sync(sc->mbuf_dma_tag,
 772                                 sc->txring_m_dmamap[sc->txring_hd_ptr],
 773                                 BUS_DMASYNC_PREWRITE);
 774
 775                 /* Set wrap flag if next packet might run off end of ring. */
 776                 wrap = sc->txring_hd_ptr + nsegs + TX_MAX_DMA_SEGS >=
 777                         CGEM_NUM_TX_DESCS;
 778
 779                 /* Fill in the TX descriptors back to front so that USED
 780                  * bit in first descriptor is cleared last.
 781                  */
 782                 for (i = nsegs - 1; i >= 0; i--) {
 783                         /* Descriptor address. */
 784                         sc->txring[sc->txring_hd_ptr + i].addr =
 785                                 segs[i].ds_addr;
 786
 787                         /* Descriptor control word. */
 788                         ctl = segs[i].ds_len;
 789                         if (i == nsegs - 1) {
 790                                 ctl |= CGEM_TXDESC_LAST_BUF;
 791                                 if (wrap)
 792                                         ctl |= CGEM_TXDESC_WRAP;
 793                         }
 794                         sc->txring[sc->txring_hd_ptr + i].ctl = ctl;
 795
 796                         if (i != 0)
 797                                 sc->txring_m[sc->txring_hd_ptr + i] = NULL;
 798                 }
 799
 800                 if (wrap)
 801                         sc->txring_hd_ptr = 0;
 802                 else
 803                         sc->txring_hd_ptr += nsegs;
 804                 sc->txring_queued += nsegs;
 805
 806                 /* Kick the transmitter. */
 807                 WR4(sc, CGEM_NET_CTRL, sc->net_ctl_shadow |
 808                     CGEM_NET_CTRL_START_TX);
 809
 810                 /* If there is a BPF listener, bounce a copy to him. */
 811                 ETHER_BPF_MTAP(ifp, m);
 812         }
 813 }
 814
 815 static void
 816 cgem_start(if_t ifp)
 817 {
 818         struct cgem_softc *sc = (struct cgem_softc *) if_getsoftc(ifp);
 819
 820         CGEM_LOCK(sc);
 821         cgem_start_locked(ifp);
 822         CGEM_UNLOCK(sc);
 823 }
 824
 825 static void
 826 cgem_poll_hw_stats(struct cgem_softc *sc)
 827 {
 828         uint32_t n;
 829
 830         CGEM_ASSERT_LOCKED(sc);
 831
 832         sc->stats.tx_bytes += RD4(sc, CGEM_OCTETS_TX_BOT);
 833         sc->stats.tx_bytes += (uint64_t)RD4(sc, CGEM_OCTETS_TX_TOP) << 32;
 834
 835         sc->stats.tx_frames += RD4(sc, CGEM_FRAMES_TX);
 836         sc->stats.tx_frames_bcast += RD4(sc, CGEM_BCAST_FRAMES_TX);
 837         sc->stats.tx_frames_multi += RD4(sc, CGEM_MULTI_FRAMES_TX);
 838         sc->stats.tx_frames_pause += RD4(sc, CGEM_PAUSE_FRAMES_TX);
 839         sc->stats.tx_frames_64b += RD4(sc, CGEM_FRAMES_64B_TX);
 840         sc->stats.tx_frames_65to127b += RD4(sc, CGEM_FRAMES_65_127B_TX);
 841         sc->stats.tx_frames_128to255b += RD4(sc, CGEM_FRAMES_128_255B_TX);
 842         sc->stats.tx_frames_256to511b += RD4(sc, CGEM_FRAMES_256_511B_TX);
 843         sc->stats.tx_frames_512to1023b += RD4(sc, CGEM_FRAMES_512_1023B_TX);
 844         sc->stats.tx_frames_1024to1536b += RD4(sc, CGEM_FRAMES_1024_1518B_TX);
 845         sc->stats.tx_under_runs += RD4(sc, CGEM_TX_UNDERRUNS);
 846
 847         n = RD4(sc, CGEM_SINGLE_COLL_FRAMES);
 848         sc->stats.tx_single_collisn += n;
 849         if_inc_counter(sc->ifp, IFCOUNTER_COLLISIONS, n);
 850         n = RD4(sc, CGEM_MULTI_COLL_FRAMES);
 851         sc->stats.tx_multi_collisn += n;
 852         if_inc_counter(sc->ifp, IFCOUNTER_COLLISIONS, n);
 853         n = RD4(sc, CGEM_EXCESSIVE_COLL_FRAMES);
 854         sc->stats.tx_excsv_collisn += n;
 855         if_inc_counter(sc->ifp, IFCOUNTER_COLLISIONS, n);
 856         n = RD4(sc, CGEM_LATE_COLL);
 857         sc->stats.tx_late_collisn += n;
 858         if_inc_counter(sc->ifp, IFCOUNTER_COLLISIONS, n);
 859
 860         sc->stats.tx_deferred_frames += RD4(sc, CGEM_DEFERRED_TX_FRAMES);
 861         sc->stats.tx_carrier_sense_errs += RD4(sc, CGEM_CARRIER_SENSE_ERRS);
 862
 863         sc->stats.rx_bytes += RD4(sc, CGEM_OCTETS_RX_BOT);
 864         sc->stats.rx_bytes += (uint64_t)RD4(sc, CGEM_OCTETS_RX_TOP) << 32;
 865
 866         sc->stats.rx_frames += RD4(sc, CGEM_FRAMES_RX);
 867         sc->stats.rx_frames_bcast += RD4(sc, CGEM_BCAST_FRAMES_RX);
 868         sc->stats.rx_frames_multi += RD4(sc, CGEM_MULTI_FRAMES_RX);
 869         sc->stats.rx_frames_pause += RD4(sc, CGEM_PAUSE_FRAMES_RX);
 870         sc->stats.rx_frames_64b += RD4(sc, CGEM_FRAMES_64B_RX);
 871         sc->stats.rx_frames_65to127b += RD4(sc, CGEM_FRAMES_65_127B_RX);
 872         sc->stats.rx_frames_128to255b += RD4(sc, CGEM_FRAMES_128_255B_RX);
 873         sc->stats.rx_frames_256to511b += RD4(sc, CGEM_FRAMES_256_511B_RX);
 874         sc->stats.rx_frames_512to1023b += RD4(sc, CGEM_FRAMES_512_1023B_RX);
 875         sc->stats.rx_frames_1024to1536b += RD4(sc, CGEM_FRAMES_1024_1518B_RX);
 876         sc->stats.rx_frames_undersize += RD4(sc, CGEM_UNDERSZ_RX);
 877         sc->stats.rx_frames_oversize += RD4(sc, CGEM_OVERSZ_RX);
 878         sc->stats.rx_frames_jabber += RD4(sc, CGEM_JABBERS_RX);
 879         sc->stats.rx_frames_fcs_errs += RD4(sc, CGEM_FCS_ERRS);
 880         sc->stats.rx_frames_length_errs += RD4(sc, CGEM_LENGTH_FIELD_ERRS);
 881         sc->stats.rx_symbol_errs += RD4(sc, CGEM_RX_SYMBOL_ERRS);
 882         sc->stats.rx_align_errs += RD4(sc, CGEM_ALIGN_ERRS);
 883         sc->stats.rx_resource_errs += RD4(sc, CGEM_RX_RESOURCE_ERRS);
 884         sc->stats.rx_overrun_errs += RD4(sc, CGEM_RX_OVERRUN_ERRS);
 885         sc->stats.rx_ip_hdr_csum_errs += RD4(sc, CGEM_IP_HDR_CKSUM_ERRS);
 886         sc->stats.rx_tcp_csum_errs += RD4(sc, CGEM_TCP_CKSUM_ERRS);
 887         sc->stats.rx_udp_csum_errs += RD4(sc, CGEM_UDP_CKSUM_ERRS);
 888 }
 889
 890 static void
 891 cgem_tick(void *arg)
 892 {
 893         struct cgem_softc *sc = (struct cgem_softc *)arg;
 894         struct mii_data *mii;
 895
 896         CGEM_ASSERT_LOCKED(sc);
 897
 898         /* Poll the phy. */
 899         if (sc->miibus != NULL) {
 900                 mii = device_get_softc(sc->miibus);
 901                 mii_tick(mii);
 902         }
 903
 904         /* Poll statistics registers. */
 905         cgem_poll_hw_stats(sc);
 906
 907         /* Check for receiver hang. */
 908         if (sc->rxhangwar && sc->rx_frames_prev == sc->stats.rx_frames) {
 909                 /*
 910                  * Reset receiver logic by toggling RX_EN bit.  1usec
 911                  * delay is necessary especially when operating at 100mbps
 912                  * and 10mbps speeds.
 913                  */
 914                 WR4(sc, CGEM_NET_CTRL, sc->net_ctl_shadow &
 915                     ~CGEM_NET_CTRL_RX_EN);
 916                 DELAY(1);
 917                 WR4(sc, CGEM_NET_CTRL, sc->net_ctl_shadow);
 918         }
 919         sc->rx_frames_prev = sc->stats.rx_frames;
 920
 921         /* Next callout in one second. */
 922         callout_reset(&sc->tick_ch, hz, cgem_tick, sc);
 923 }
 924
 925 /* Interrupt handler. */
 926 static void
 927 cgem_intr(void *arg)
 928 {
 929         struct cgem_softc *sc = (struct cgem_softc *)arg;
 930         if_t ifp = sc->ifp;
 931         uint32_t istatus;
 932
 933         CGEM_LOCK(sc);
 934
 935         if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) {
 936                 CGEM_UNLOCK(sc);
 937                 return;
 938         }
 939
 940         /* Read interrupt status and immediately clear the bits. */
 941         istatus = RD4(sc, CGEM_INTR_STAT);
 942         WR4(sc, CGEM_INTR_STAT, istatus);
 943
 944         /* Packets received. */
 945         if ((istatus & CGEM_INTR_RX_COMPLETE) != 0)
 946                 cgem_recv(sc);
 947
 948         /* Free up any completed transmit buffers. */
 949         cgem_clean_tx(sc);
 950
 951         /* Hresp not ok.  Something is very bad with DMA.  Try to clear. */
 952         if ((istatus & CGEM_INTR_HRESP_NOT_OK) != 0) {
 953                 device_printf(sc->dev, "cgem_intr: hresp not okay! "
 954                               "rx_status=0x%x\n", RD4(sc, CGEM_RX_STAT));
 955                 WR4(sc, CGEM_RX_STAT, CGEM_RX_STAT_HRESP_NOT_OK);
 956         }
 957
 958         /* Receiver overrun. */
 959         if ((istatus & CGEM_INTR_RX_OVERRUN) != 0) {
 960                 /* Clear status bit. */
 961                 WR4(sc, CGEM_RX_STAT, CGEM_RX_STAT_OVERRUN);
 962                 sc->rxoverruns++;
 963         }
 964
 965         /* Receiver ran out of bufs. */
 966         if ((istatus & CGEM_INTR_RX_USED_READ) != 0) {
 967                 WR4(sc, CGEM_NET_CTRL, sc->net_ctl_shadow |
 968                     CGEM_NET_CTRL_FLUSH_DPRAM_PKT);
 969                 cgem_fill_rqueue(sc);
 970                 sc->rxnobufs++;
 971         }
 972
 973         /* Restart transmitter if needed. */
 974         if (!if_sendq_empty(ifp))
 975                 cgem_start_locked(ifp);
 976
 977         CGEM_UNLOCK(sc);
 978 }
 979
 980 /* Reset hardware. */
 981 static void
 982 cgem_reset(struct cgem_softc *sc)
 983 {
 984
 985         CGEM_ASSERT_LOCKED(sc);
 986
 987         WR4(sc, CGEM_NET_CTRL, 0);
 988         WR4(sc, CGEM_NET_CFG, 0);
 989         WR4(sc, CGEM_NET_CTRL, CGEM_NET_CTRL_CLR_STAT_REGS);
 990         WR4(sc, CGEM_TX_STAT, CGEM_TX_STAT_ALL);
 991         WR4(sc, CGEM_RX_STAT, CGEM_RX_STAT_ALL);
 992         WR4(sc, CGEM_INTR_DIS, CGEM_INTR_ALL);
 993         WR4(sc, CGEM_HASH_BOT, 0);
 994         WR4(sc, CGEM_HASH_TOP, 0);
 995         WR4(sc, CGEM_TX_QBAR, 0);       /* manual says do this. */
 996         WR4(sc, CGEM_RX_QBAR, 0);
 997
 998         /* Get management port running even if interface is down. */
 999         WR4(sc, CGEM_NET_CFG,
1000             CGEM_NET_CFG_DBUS_WIDTH_32 |
1001             CGEM_NET_CFG_MDC_CLK_DIV_64);
1002
1003         sc->net_ctl_shadow = CGEM_NET_CTRL_MGMT_PORT_EN;
1004         WR4(sc, CGEM_NET_CTRL, sc->net_ctl_shadow);
1005 }
1006
1007 /* Bring up the hardware. */
1008 static void
1009 cgem_config(struct cgem_softc *sc)
1010 {
1011         if_t ifp = sc->ifp;
1012         uint32_t net_cfg;
1013         uint32_t dma_cfg;
1014         u_char *eaddr = if_getlladdr(ifp);
1015
1016         CGEM_ASSERT_LOCKED(sc);
1017
1018         /* Program Net Config Register. */
1019         net_cfg = CGEM_NET_CFG_DBUS_WIDTH_32 |
1020                 CGEM_NET_CFG_MDC_CLK_DIV_64 |
1021                 CGEM_NET_CFG_FCS_REMOVE |
1022                 CGEM_NET_CFG_RX_BUF_OFFSET(ETHER_ALIGN) |
1023                 CGEM_NET_CFG_GIGE_EN |
1024                 CGEM_NET_CFG_1536RXEN |
1025                 CGEM_NET_CFG_FULL_DUPLEX |
1026                 CGEM_NET_CFG_SPEED100;
1027
1028         /* Enable receive checksum offloading? */
1029         if ((if_getcapenable(ifp) & IFCAP_RXCSUM) != 0)
1030                 net_cfg |=  CGEM_NET_CFG_RX_CHKSUM_OFFLD_EN;
1031
1032         WR4(sc, CGEM_NET_CFG, net_cfg);
1033
1034         /* Program DMA Config Register. */
1035         dma_cfg = CGEM_DMA_CFG_RX_BUF_SIZE(MCLBYTES) |
1036                 CGEM_DMA_CFG_RX_PKTBUF_MEMSZ_SEL_8K |
1037                 CGEM_DMA_CFG_TX_PKTBUF_MEMSZ_SEL |
1038                 CGEM_DMA_CFG_AHB_FIXED_BURST_LEN_16 |
1039                 CGEM_DMA_CFG_DISC_WHEN_NO_AHB;
1040
1041         /* Enable transmit checksum offloading? */
1042         if ((if_getcapenable(ifp) & IFCAP_TXCSUM) != 0)
1043                 dma_cfg |= CGEM_DMA_CFG_CHKSUM_GEN_OFFLOAD_EN;
1044
1045         WR4(sc, CGEM_DMA_CFG, dma_cfg);
1046
1047         /* Write the rx and tx descriptor ring addresses to the QBAR regs. */
1048         WR4(sc, CGEM_RX_QBAR, (uint32_t) sc->rxring_physaddr);
1049         WR4(sc, CGEM_TX_QBAR, (uint32_t) sc->txring_physaddr);
1050
1051         /* Enable rx and tx. */
1052         sc->net_ctl_shadow |= (CGEM_NET_CTRL_TX_EN | CGEM_NET_CTRL_RX_EN);
1053         WR4(sc, CGEM_NET_CTRL, sc->net_ctl_shadow);
1054
1055         /* Set receive address in case it changed. */
1056         WR4(sc, CGEM_SPEC_ADDR_LOW(0), (eaddr[3] << 24) |
1057             (eaddr[2] << 16) | (eaddr[1] << 8) | eaddr[0]);
1058         WR4(sc, CGEM_SPEC_ADDR_HI(0), (eaddr[5] << 8) | eaddr[4]);
1059
1060         /* Set up interrupts. */
1061         WR4(sc, CGEM_INTR_EN,
1062             CGEM_INTR_RX_COMPLETE | CGEM_INTR_RX_OVERRUN |
1063             CGEM_INTR_TX_USED_READ | CGEM_INTR_RX_USED_READ |
1064             CGEM_INTR_HRESP_NOT_OK);
1065 }
1066
1067 /* Turn on interface and load up receive ring with buffers. */
1068 static void
1069 cgem_init_locked(struct cgem_softc *sc)
1070 {
1071         struct mii_data *mii;
1072
1073         CGEM_ASSERT_LOCKED(sc);
1074
1075         if ((if_getdrvflags(sc->ifp) & IFF_DRV_RUNNING) != 0)
1076                 return;
1077
1078         cgem_config(sc);
1079         cgem_fill_rqueue(sc);
1080
1081         if_setdrvflagbits(sc->ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);
1082
1083         mii = device_get_softc(sc->miibus);
1084         mii_mediachg(mii);
1085
1086         callout_reset(&sc->tick_ch, hz, cgem_tick, sc);
1087 }
1088
1089 static void
1090 cgem_init(void *arg)
1091 {
1092         struct cgem_softc *sc = (struct cgem_softc *)arg;
1093
1094         CGEM_LOCK(sc);
1095         cgem_init_locked(sc);
1096         CGEM_UNLOCK(sc);
1097 }
1098
1099 /* Turn off interface.  Free up any buffers in transmit or receive queues. */
1100 static void
1101 cgem_stop(struct cgem_softc *sc)
1102 {
1103         int i;
1104
1105         CGEM_ASSERT_LOCKED(sc);
1106
1107         callout_stop(&sc->tick_ch);
1108
1109         /* Shut down hardware. */
1110         cgem_reset(sc);
1111
1112         /* Clear out transmit queue. */
1113         for (i = 0; i < CGEM_NUM_TX_DESCS; i++) {
1114                 sc->txring[i].ctl = CGEM_TXDESC_USED;
1115                 sc->txring[i].addr = 0;
1116                 if (sc->txring_m[i]) {
1117                         /* Unload and destroy dmamap. */
1118                         bus_dmamap_unload(sc->mbuf_dma_tag,
1119                                           sc->txring_m_dmamap[i]);
1120                         bus_dmamap_destroy(sc->mbuf_dma_tag,
1121                                            sc->txring_m_dmamap[i]);
1122                         sc->txring_m_dmamap[i] = NULL;
1123                         m_freem(sc->txring_m[i]);
1124                         sc->txring_m[i] = NULL;
1125                 }
1126         }
1127         sc->txring[CGEM_NUM_TX_DESCS - 1].ctl |= CGEM_TXDESC_WRAP;
1128
1129         sc->txring_hd_ptr = 0;
1130         sc->txring_tl_ptr = 0;
1131         sc->txring_queued = 0;
1132
1133         /* Clear out receive queue. */
1134         for (i = 0; i < CGEM_NUM_RX_DESCS; i++) {
1135                 sc->rxring[i].addr = CGEM_RXDESC_OWN;
1136                 sc->rxring[i].ctl = 0;
1137                 if (sc->rxring_m[i]) {
1138                         /* Unload and destroy dmamap. */
1139                         bus_dmamap_unload(sc->mbuf_dma_tag,
1140                                   sc->rxring_m_dmamap[i]);
1141                         bus_dmamap_destroy(sc->mbuf_dma_tag,
1142                                    sc->rxring_m_dmamap[i]);
1143                         sc->rxring_m_dmamap[i] = NULL;
1144
1145                         m_freem(sc->rxring_m[i]);
1146                         sc->rxring_m[i] = NULL;
1147                 }
1148         }
1149         sc->rxring[CGEM_NUM_RX_DESCS - 1].addr |= CGEM_RXDESC_WRAP;
1150
1151         sc->rxring_hd_ptr = 0;
1152         sc->rxring_tl_ptr = 0;
1153         sc->rxring_queued = 0;
1154
1155         /* Force next statchg or linkchg to program net config register. */
1156         sc->mii_media_active = 0;
1157 }
1158
1159
1160 static int
1161 cgem_ioctl(if_t ifp, u_long cmd, caddr_t data)
1162 {
1163         struct cgem_softc *sc = if_getsoftc(ifp);
1164         struct ifreq *ifr = (struct ifreq *)data;
1165         struct mii_data *mii;
1166         int error = 0, mask;
1167
1168         switch (cmd) {
1169         case SIOCSIFFLAGS:
1170                 CGEM_LOCK(sc);
1171                 if ((if_getflags(ifp) & IFF_UP) != 0) {
1172                         if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) {
1173                                 if (((if_getflags(ifp) ^ sc->if_old_flags) &
1174                                      (IFF_PROMISC | IFF_ALLMULTI)) != 0) {
1175                                         cgem_rx_filter(sc);
1176                                 }
1177                         } else {
1178                                 cgem_init_locked(sc);
1179                         }
1180                 } else if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) {
1181                         if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING);
1182                         cgem_stop(sc);
1183                 }
1184                 sc->if_old_flags = if_getflags(ifp);
1185                 CGEM_UNLOCK(sc);
1186                 break;
1187
1188         case SIOCADDMULTI:
1189         case SIOCDELMULTI:
1190                 /* Set up multi-cast filters. */
1191                 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) {
1192                         CGEM_LOCK(sc);
1193                         cgem_rx_filter(sc);
1194                         CGEM_UNLOCK(sc);
1195                 }
1196                 break;
1197
1198         case SIOCSIFMEDIA:
1199         case SIOCGIFMEDIA:
1200                 mii = device_get_softc(sc->miibus);
1201                 error = ifmedia_ioctl(ifp, ifr, &mii->mii_media, cmd);
1202                 break;
1203
1204         case SIOCSIFCAP:
1205                 CGEM_LOCK(sc);
1206                 mask = if_getcapenable(ifp) ^ ifr->ifr_reqcap;
1207
1208                 if ((mask & IFCAP_TXCSUM) != 0) {
1209                         if ((ifr->ifr_reqcap & IFCAP_TXCSUM) != 0) {
1210                                 /* Turn on TX checksumming. */
1211                                 if_setcapenablebit(ifp, IFCAP_TXCSUM |
1212                                                    IFCAP_TXCSUM_IPV6, 0);
1213                                 if_sethwassistbits(ifp, CGEM_CKSUM_ASSIST, 0);
1214
1215                                 WR4(sc, CGEM_DMA_CFG,
1216                                     RD4(sc, CGEM_DMA_CFG) |
1217                                      CGEM_DMA_CFG_CHKSUM_GEN_OFFLOAD_EN);
1218                         } else {
1219                                 /* Turn off TX checksumming. */
1220                                 if_setcapenablebit(ifp, 0, IFCAP_TXCSUM |
1221                                                    IFCAP_TXCSUM_IPV6);
1222                                 if_sethwassistbits(ifp, 0, CGEM_CKSUM_ASSIST);
1223
1224                                 WR4(sc, CGEM_DMA_CFG,
1225                                     RD4(sc, CGEM_DMA_CFG) &
1226                                      ~CGEM_DMA_CFG_CHKSUM_GEN_OFFLOAD_EN);
1227                         }
1228                 }
1229                 if ((mask & IFCAP_RXCSUM) != 0) {
1230                         if ((ifr->ifr_reqcap & IFCAP_RXCSUM) != 0) {
1231                                 /* Turn on RX checksumming. */
1232                                 if_setcapenablebit(ifp, IFCAP_RXCSUM |
1233                                                    IFCAP_RXCSUM_IPV6, 0);
1234                                 WR4(sc, CGEM_NET_CFG,
1235                                     RD4(sc, CGEM_NET_CFG) |
1236                                      CGEM_NET_CFG_RX_CHKSUM_OFFLD_EN);
1237                         } else {
1238                                 /* Turn off RX checksumming. */
1239                                 if_setcapenablebit(ifp, 0, IFCAP_RXCSUM |
1240                                                    IFCAP_RXCSUM_IPV6);
1241                                 WR4(sc, CGEM_NET_CFG,
1242                                     RD4(sc, CGEM_NET_CFG) &
1243                                      ~CGEM_NET_CFG_RX_CHKSUM_OFFLD_EN);
1244                         }
1245                 }
1246                 if ((if_getcapenable(ifp) & (IFCAP_RXCSUM | IFCAP_TXCSUM)) ==
1247                     (IFCAP_RXCSUM | IFCAP_TXCSUM))
1248                         if_setcapenablebit(ifp, IFCAP_VLAN_HWCSUM, 0);
1249                 else
1250                         if_setcapenablebit(ifp, 0, IFCAP_VLAN_HWCSUM);
1251
1252                 CGEM_UNLOCK(sc);
1253                 break;
1254         default:
1255                 error = ether_ioctl(ifp, cmd, data);
1256                 break;
1257         }
1258
1259         return (error);
1260 }
1261
1262 /* MII bus support routines.
1263  */
1264 static void
1265 cgem_child_detached(device_t dev, device_t child)
1266 {
1267         struct cgem_softc *sc = device_get_softc(dev);
1268
1269         if (child == sc->miibus)
1270                 sc->miibus = NULL;
1271 }
1272
1273 static int
1274 cgem_ifmedia_upd(if_t ifp)
1275 {
1276         struct cgem_softc *sc = (struct cgem_softc *) if_getsoftc(ifp);
1277         struct mii_data *mii;
1278         struct mii_softc *miisc;
1279         int error = 0;
1280
1281         mii = device_get_softc(sc->miibus);
1282         CGEM_LOCK(sc);
1283         if ((if_getflags(ifp) & IFF_UP) != 0) {
1284                 LIST_FOREACH(miisc, &mii->mii_phys, mii_list)
1285                         PHY_RESET(miisc);
1286                 error = mii_mediachg(mii);
1287         }
1288         CGEM_UNLOCK(sc);
1289
1290         return (error);
1291 }
1292
1293 static void
1294 cgem_ifmedia_sts(if_t ifp, struct ifmediareq *ifmr)
1295 {
1296         struct cgem_softc *sc = (struct cgem_softc *) if_getsoftc(ifp);
1297         struct mii_data *mii;
1298
1299         mii = device_get_softc(sc->miibus);
1300         CGEM_LOCK(sc);
1301         mii_pollstat(mii);
1302         ifmr->ifm_active = mii->mii_media_active;
1303         ifmr->ifm_status = mii->mii_media_status;
1304         CGEM_UNLOCK(sc);
1305 }
1306
1307 static int
1308 cgem_miibus_readreg(device_t dev, int phy, int reg)
1309 {
1310         struct cgem_softc *sc = device_get_softc(dev);
1311         int tries, val;
1312
1313         WR4(sc, CGEM_PHY_MAINT,
1314             CGEM_PHY_MAINT_CLAUSE_22 | CGEM_PHY_MAINT_MUST_10 |
1315             CGEM_PHY_MAINT_OP_READ |
1316             (phy << CGEM_PHY_MAINT_PHY_ADDR_SHIFT) |
1317             (reg << CGEM_PHY_MAINT_REG_ADDR_SHIFT));
1318
1319         /* Wait for completion. */
1320         tries=0;
1321         while ((RD4(sc, CGEM_NET_STAT) & CGEM_NET_STAT_PHY_MGMT_IDLE) == 0) {
1322                 DELAY(5);
1323                 if (++tries > 200) {
1324                         device_printf(dev, "phy read timeout: %d\n", reg);
1325                         return (-1);
1326                 }
1327         }
1328
1329         val = RD4(sc, CGEM_PHY_MAINT) & CGEM_PHY_MAINT_DATA_MASK;
1330
1331         if (reg == MII_EXTSR)
1332                 /*
1333                  * MAC does not support half-duplex at gig speeds.
1334                  * Let mii(4) exclude the capability.
1335                  */
1336                 val &= ~(EXTSR_1000XHDX | EXTSR_1000THDX);
1337
1338         return (val);
1339 }
1340
1341 static int
1342 cgem_miibus_writereg(device_t dev, int phy, int reg, int data)
1343 {
1344         struct cgem_softc *sc = device_get_softc(dev);
1345         int tries;
1346
1347         WR4(sc, CGEM_PHY_MAINT,
1348             CGEM_PHY_MAINT_CLAUSE_22 | CGEM_PHY_MAINT_MUST_10 |
1349             CGEM_PHY_MAINT_OP_WRITE |
1350             (phy << CGEM_PHY_MAINT_PHY_ADDR_SHIFT) |
1351             (reg << CGEM_PHY_MAINT_REG_ADDR_SHIFT) |
1352             (data & CGEM_PHY_MAINT_DATA_MASK));
1353
1354         /* Wait for completion. */
1355         tries = 0;
1356         while ((RD4(sc, CGEM_NET_STAT) & CGEM_NET_STAT_PHY_MGMT_IDLE) == 0) {
1357                 DELAY(5);
1358                 if (++tries > 200) {
1359                         device_printf(dev, "phy write timeout: %d\n", reg);
1360                         return (-1);
1361                 }
1362         }
1363
1364         return (0);
1365 }
1366
1367 static void
1368 cgem_miibus_statchg(device_t dev)
1369 {
1370         struct cgem_softc *sc  = device_get_softc(dev);
1371         struct mii_data *mii = device_get_softc(sc->miibus);
1372
1373         CGEM_ASSERT_LOCKED(sc);
1374
1375         if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) ==
1376             (IFM_ACTIVE | IFM_AVALID) &&
1377             sc->mii_media_active != mii->mii_media_active)
1378                 cgem_mediachange(sc, mii);
1379 }
1380
1381 static void
1382 cgem_miibus_linkchg(device_t dev)
1383 {
1384         struct cgem_softc *sc  = device_get_softc(dev);
1385         struct mii_data *mii = device_get_softc(sc->miibus);
1386
1387         CGEM_ASSERT_LOCKED(sc);
1388
1389         if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) ==
1390             (IFM_ACTIVE | IFM_AVALID) &&
1391             sc->mii_media_active != mii->mii_media_active)
1392                 cgem_mediachange(sc, mii);
1393 }
1394
1395 /*
1396  * Overridable weak symbol cgem_set_ref_clk().  This allows platforms to
1397  * provide a function to set the cgem's reference clock.
1398  */
1399 static int __used
1400 cgem_default_set_ref_clk(int unit, int frequency)
1401 {
1402
1403         return 0;
1404 }
1405 __weak_reference(cgem_default_set_ref_clk, cgem_set_ref_clk);
1406
1407 /* Call to set reference clock and network config bits according to media. */
1408 static void
1409 cgem_mediachange(struct cgem_softc *sc, struct mii_data *mii)
1410 {
1411         uint32_t net_cfg;
1412         int ref_clk_freq;
1413
1414         CGEM_ASSERT_LOCKED(sc);
1415
1416         /* Update hardware to reflect media. */
1417         net_cfg = RD4(sc, CGEM_NET_CFG);
1418         net_cfg &= ~(CGEM_NET_CFG_SPEED100 | CGEM_NET_CFG_GIGE_EN |
1419                      CGEM_NET_CFG_FULL_DUPLEX);
1420
1421         switch (IFM_SUBTYPE(mii->mii_media_active)) {
1422         case IFM_1000_T:
1423                 net_cfg |= (CGEM_NET_CFG_SPEED100 |
1424                             CGEM_NET_CFG_GIGE_EN);
1425                 ref_clk_freq = 125000000;
1426                 break;
1427         case IFM_100_TX:
1428                 net_cfg |= CGEM_NET_CFG_SPEED100;
1429                 ref_clk_freq = 25000000;
1430                 break;
1431         default:
1432                 ref_clk_freq = 2500000;
1433         }
1434
1435         if ((mii->mii_media_active & IFM_FDX) != 0)
1436                 net_cfg |= CGEM_NET_CFG_FULL_DUPLEX;
1437
1438         WR4(sc, CGEM_NET_CFG, net_cfg);
1439
1440         /* Set the reference clock if necessary. */
1441         if (cgem_set_ref_clk(sc->ref_clk_num, ref_clk_freq))
1442                 device_printf(sc->dev, "cgem_mediachange: "
1443                               "could not set ref clk%d to %d.\n",
1444                               sc->ref_clk_num, ref_clk_freq);
1445
1446         sc->mii_media_active = mii->mii_media_active;
1447 }
1448
1449 static void
1450 cgem_add_sysctls(device_t dev)
1451 {
1452         struct cgem_softc *sc = device_get_softc(dev);
1453         struct sysctl_ctx_list *ctx;
1454         struct sysctl_oid_list *child;
1455         struct sysctl_oid *tree;
1456
1457         ctx = device_get_sysctl_ctx(dev);
1458         child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev));
1459
1460         SYSCTL_ADD_INT(ctx, child, OID_AUTO, "rxbufs", CTLFLAG_RW,
1461                        &sc->rxbufs, 0,
1462                        "Number receive buffers to provide");
1463
1464         SYSCTL_ADD_INT(ctx, child, OID_AUTO, "rxhangwar", CTLFLAG_RW,
1465                        &sc->rxhangwar, 0,
1466                        "Enable receive hang work-around");
1467
1468         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "_rxoverruns", CTLFLAG_RD,
1469                         &sc->rxoverruns, 0,
1470                         "Receive overrun events");
1471
1472         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "_rxnobufs", CTLFLAG_RD,
1473                         &sc->rxnobufs, 0,
1474                         "Receive buf queue empty events");
1475
1476         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "_rxdmamapfails", CTLFLAG_RD,
1477                         &sc->rxdmamapfails, 0,
1478                         "Receive DMA map failures");
1479
1480         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "_txfull", CTLFLAG_RD,
1481                         &sc->txfull, 0,
1482                         "Transmit ring full events");
1483
1484         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "_txdmamapfails", CTLFLAG_RD,
1485                         &sc->txdmamapfails, 0,
1486                         "Transmit DMA map failures");
1487
1488         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "_txdefrags", CTLFLAG_RD,
1489                         &sc->txdefrags, 0,
1490                         "Transmit m_defrag() calls");
1491
1492         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "_txdefragfails", CTLFLAG_RD,
1493                         &sc->txdefragfails, 0,
1494                         "Transmit m_defrag() failures");
1495
1496         tree = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "stats", CTLFLAG_RD,
1497                                NULL, "GEM statistics");
1498         child = SYSCTL_CHILDREN(tree);
1499
1500         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_bytes", CTLFLAG_RD,
1501                          &sc->stats.tx_bytes, "Total bytes transmitted");
1502
1503         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames", CTLFLAG_RD,
1504                         &sc->stats.tx_frames, 0, "Total frames transmitted");
1505         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_bcast", CTLFLAG_RD,
1506                         &sc->stats.tx_frames_bcast, 0,
1507                         "Number broadcast frames transmitted");
1508         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_multi", CTLFLAG_RD,
1509                         &sc->stats.tx_frames_multi, 0,
1510                         "Number multicast frames transmitted");
1511         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_pause",
1512                         CTLFLAG_RD, &sc->stats.tx_frames_pause, 0,
1513                         "Number pause frames transmitted");
1514         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_64b", CTLFLAG_RD,
1515                         &sc->stats.tx_frames_64b, 0,
1516                         "Number frames transmitted of size 64 bytes or less");
1517         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_65to127b", CTLFLAG_RD,
1518                         &sc->stats.tx_frames_65to127b, 0,
1519                         "Number frames transmitted of size 65-127 bytes");
1520         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_128to255b",
1521                         CTLFLAG_RD, &sc->stats.tx_frames_128to255b, 0,
1522                         "Number frames transmitted of size 128-255 bytes");
1523         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_256to511b",
1524                         CTLFLAG_RD, &sc->stats.tx_frames_256to511b, 0,
1525                         "Number frames transmitted of size 256-511 bytes");
1526         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_512to1023b",
1527                         CTLFLAG_RD, &sc->stats.tx_frames_512to1023b, 0,
1528                         "Number frames transmitted of size 512-1023 bytes");
1529         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_frames_1024to1536b",
1530                         CTLFLAG_RD, &sc->stats.tx_frames_1024to1536b, 0,
1531                         "Number frames transmitted of size 1024-1536 bytes");
1532         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_under_runs",
1533                         CTLFLAG_RD, &sc->stats.tx_under_runs, 0,
1534                         "Number transmit under-run events");
1535         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_single_collisn",
1536                         CTLFLAG_RD, &sc->stats.tx_single_collisn, 0,
1537                         "Number single-collision transmit frames");
1538         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_multi_collisn",
1539                         CTLFLAG_RD, &sc->stats.tx_multi_collisn, 0,
1540                         "Number multi-collision transmit frames");
1541         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_excsv_collisn",
1542                         CTLFLAG_RD, &sc->stats.tx_excsv_collisn, 0,
1543                         "Number excessive collision transmit frames");
1544         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_late_collisn",
1545                         CTLFLAG_RD, &sc->stats.tx_late_collisn, 0,
1546                         "Number late-collision transmit frames");
1547         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_deferred_frames",
1548                         CTLFLAG_RD, &sc->stats.tx_deferred_frames, 0,
1549                         "Number deferred transmit frames");
1550         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "tx_carrier_sense_errs",
1551                         CTLFLAG_RD, &sc->stats.tx_carrier_sense_errs, 0,
1552                         "Number carrier sense errors on transmit");
1553
1554         SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_bytes", CTLFLAG_RD,
1555                          &sc->stats.rx_bytes, "Total bytes received");
1556
1557         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames", CTLFLAG_RD,
1558                         &sc->stats.rx_frames, 0, "Total frames received");
1559         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_bcast",
1560                         CTLFLAG_RD, &sc->stats.rx_frames_bcast, 0,
1561                         "Number broadcast frames received");
1562         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_multi",
1563                         CTLFLAG_RD, &sc->stats.rx_frames_multi, 0,
1564                         "Number multicast frames received");
1565         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_pause",
1566                         CTLFLAG_RD, &sc->stats.rx_frames_pause, 0,
1567                         "Number pause frames received");
1568         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_64b",
1569                         CTLFLAG_RD, &sc->stats.rx_frames_64b, 0,
1570                         "Number frames received of size 64 bytes or less");
1571         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_65to127b",
1572                         CTLFLAG_RD, &sc->stats.rx_frames_65to127b, 0,
1573                         "Number frames received of size 65-127 bytes");
1574         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_128to255b",
1575                         CTLFLAG_RD, &sc->stats.rx_frames_128to255b, 0,
1576                         "Number frames received of size 128-255 bytes");
1577         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_256to511b",
1578                         CTLFLAG_RD, &sc->stats.rx_frames_256to511b, 0,
1579                         "Number frames received of size 256-511 bytes");
1580         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_512to1023b",
1581                         CTLFLAG_RD, &sc->stats.rx_frames_512to1023b, 0,
1582                         "Number frames received of size 512-1023 bytes");
1583         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_1024to1536b",
1584                         CTLFLAG_RD, &sc->stats.rx_frames_1024to1536b, 0,
1585                         "Number frames received of size 1024-1536 bytes");
1586         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_undersize",
1587                         CTLFLAG_RD, &sc->stats.rx_frames_undersize, 0,
1588                         "Number undersize frames received");
1589         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_oversize",
1590                         CTLFLAG_RD, &sc->stats.rx_frames_oversize, 0,
1591                         "Number oversize frames received");
1592         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_jabber",
1593                         CTLFLAG_RD, &sc->stats.rx_frames_jabber, 0,
1594                         "Number jabber frames received");
1595         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_fcs_errs",
1596                         CTLFLAG_RD, &sc->stats.rx_frames_fcs_errs, 0,
1597                         "Number frames received with FCS errors");
1598         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_length_errs",
1599                         CTLFLAG_RD, &sc->stats.rx_frames_length_errs, 0,
1600                         "Number frames received with length errors");
1601         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_symbol_errs",
1602                         CTLFLAG_RD, &sc->stats.rx_symbol_errs, 0,
1603                         "Number receive symbol errors");
1604         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_align_errs",
1605                         CTLFLAG_RD, &sc->stats.rx_align_errs, 0,
1606                         "Number receive alignment errors");
1607         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_resource_errs",
1608                         CTLFLAG_RD, &sc->stats.rx_resource_errs, 0,
1609                         "Number frames received when no rx buffer available");
1610         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_overrun_errs",
1611                         CTLFLAG_RD, &sc->stats.rx_overrun_errs, 0,
1612                         "Number frames received but not copied due to "
1613                         "receive overrun");
1614         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_ip_hdr_csum_errs",
1615                         CTLFLAG_RD, &sc->stats.rx_ip_hdr_csum_errs, 0,
1616                         "Number frames received with IP header checksum "
1617                         "errors");
1618         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_tcp_csum_errs",
1619                         CTLFLAG_RD, &sc->stats.rx_tcp_csum_errs, 0,
1620                         "Number frames received with TCP checksum errors");
1621         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "rx_frames_udp_csum_errs",
1622                         CTLFLAG_RD, &sc->stats.rx_udp_csum_errs, 0,
1623                         "Number frames received with UDP checksum errors");
1624 }
1625
1626
1627 static int
1628 cgem_probe(device_t dev)
1629 {
1630
1631         if (!ofw_bus_status_okay(dev))
1632                 return (ENXIO);
1633
1634         if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0)
1635                 return (ENXIO);
1636
1637         device_set_desc(dev, "Cadence CGEM Gigabit Ethernet Interface");
1638         return (0);
1639 }
1640
1641 static int
1642 cgem_attach(device_t dev)
1643 {
1644         struct cgem_softc *sc = device_get_softc(dev);
1645         if_t ifp = NULL;
1646         phandle_t node;
1647         pcell_t cell;
1648         int rid, err;
1649         u_char eaddr[ETHER_ADDR_LEN];
1650
1651         sc->dev = dev;
1652         CGEM_LOCK_INIT(sc);
1653
1654         /* Get reference clock number and base divider from fdt. */
1655         node = ofw_bus_get_node(dev);
1656         sc->ref_clk_num = 0;
1657         if (OF_getprop(node, "ref-clock-num", &cell, sizeof(cell)) > 0)
1658                 sc->ref_clk_num = fdt32_to_cpu(cell);
1659
1660         /* Get memory resource. */
1661         rid = 0;
1662         sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
1663                                              RF_ACTIVE);
1664         if (sc->mem_res == NULL) {
1665                 device_printf(dev, "could not allocate memory resources.\n");
1666                 return (ENOMEM);
1667         }
1668
1669         /* Get IRQ resource. */
1670         rid = 0;
1671         sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
1672                                              RF_ACTIVE);
1673         if (sc->irq_res == NULL) {
1674                 device_printf(dev, "could not allocate interrupt resource.\n");
1675                 cgem_detach(dev);
1676                 return (ENOMEM);
1677         }
1678
1679         /* Set up ifnet structure. */
1680         ifp = sc->ifp = if_alloc(IFT_ETHER);
1681         if (ifp == NULL) {
1682                 device_printf(dev, "could not allocate ifnet structure\n");
1683                 cgem_detach(dev);
1684                 return (ENOMEM);
1685         }
1686         if_setsoftc(ifp, sc);
1687         if_initname(ifp, IF_CGEM_NAME, device_get_unit(dev));
1688         if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
1689         if_setinitfn(ifp, cgem_init);
1690         if_setioctlfn(ifp, cgem_ioctl);
1691         if_setstartfn(ifp, cgem_start);
1692         if_setcapabilitiesbit(ifp, IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6 |
1693                               IFCAP_VLAN_MTU | IFCAP_VLAN_HWCSUM, 0);
1694         if_setsendqlen(ifp, CGEM_NUM_TX_DESCS);
1695         if_setsendqready(ifp);
1696
1697         /* Disable hardware checksumming by default. */
1698         if_sethwassist(ifp, 0);
1699         if_setcapenable(ifp, if_getcapabilities(ifp) &
1700                 ~(IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6 | IFCAP_VLAN_HWCSUM));
1701
1702         sc->if_old_flags = if_getflags(ifp);
1703         sc->rxbufs = DEFAULT_NUM_RX_BUFS;
1704         sc->rxhangwar = 1;
1705
1706         /* Reset hardware. */
1707         CGEM_LOCK(sc);
1708         cgem_reset(sc);
1709         CGEM_UNLOCK(sc);
1710
1711         /* Attach phy to mii bus. */
1712         err = mii_attach(dev, &sc->miibus, ifp,
1713                          cgem_ifmedia_upd, cgem_ifmedia_sts,
1714                          BMSR_DEFCAPMASK, MII_PHY_ANY, MII_OFFSET_ANY, 0);
1715         if (err) {
1716                 device_printf(dev, "attaching PHYs failed\n");
1717                 cgem_detach(dev);
1718                 return (err);
1719         }
1720
1721         /* Set up TX and RX descriptor area. */
1722         err = cgem_setup_descs(sc);
1723         if (err) {
1724                 device_printf(dev, "could not set up dma mem for descs.\n");
1725                 cgem_detach(dev);
1726                 return (ENOMEM);
1727         }
1728
1729         /* Get a MAC address. */
1730         cgem_get_mac(sc, eaddr);
1731
1732         /* Start ticks. */
1733         callout_init_mtx(&sc->tick_ch, &sc->sc_mtx, 0);
1734
1735         ether_ifattach(ifp, eaddr);
1736
1737         err = bus_setup_intr(dev, sc->irq_res, INTR_TYPE_NET | INTR_MPSAFE |
1738                              INTR_EXCL, NULL, cgem_intr, sc, &sc->intrhand);
1739         if (err) {
1740                 device_printf(dev, "could not set interrupt handler.\n");
1741                 ether_ifdetach(ifp);
1742                 cgem_detach(dev);
1743                 return (err);
1744         }
1745
1746         cgem_add_sysctls(dev);
1747
1748         return (0);
1749 }
1750
1751 static int
1752 cgem_detach(device_t dev)
1753 {
1754         struct cgem_softc *sc = device_get_softc(dev);
1755         int i;
1756
1757         if (sc == NULL)
1758                 return (ENODEV);
1759
1760         if (device_is_attached(dev)) {
1761                 CGEM_LOCK(sc);
1762                 cgem_stop(sc);
1763                 CGEM_UNLOCK(sc);
1764                 callout_drain(&sc->tick_ch);
1765                 if_setflagbits(sc->ifp, 0, IFF_UP);
1766                 ether_ifdetach(sc->ifp);
1767         }
1768
1769         if (sc->miibus != NULL) {
1770                 device_delete_child(dev, sc->miibus);
1771                 sc->miibus = NULL;
1772         }
1773
1774         /* Release resources. */
1775         if (sc->mem_res != NULL) {
1776                 bus_release_resource(dev, SYS_RES_MEMORY,
1777                                      rman_get_rid(sc->mem_res), sc->mem_res);
1778                 sc->mem_res = NULL;
1779         }
1780         if (sc->irq_res != NULL) {
1781                 if (sc->intrhand)
1782                         bus_teardown_intr(dev, sc->irq_res, sc->intrhand);
1783                 bus_release_resource(dev, SYS_RES_IRQ,
1784                                      rman_get_rid(sc->irq_res), sc->irq_res);
1785                 sc->irq_res = NULL;
1786         }
1787
1788         /* Release DMA resources. */
1789         if (sc->rxring != NULL) {
1790                 if (sc->rxring_physaddr != 0) {
1791                         bus_dmamap_unload(sc->desc_dma_tag,
1792                                           sc->rxring_dma_map);
1793                         sc->rxring_physaddr = 0;
1794                 }
1795                 bus_dmamem_free(sc->desc_dma_tag, sc->rxring,
1796                                 sc->rxring_dma_map);
1797                 sc->rxring = NULL;
1798                 for (i = 0; i < CGEM_NUM_RX_DESCS; i++)
1799                         if (sc->rxring_m_dmamap[i] != NULL) {
1800                                 bus_dmamap_destroy(sc->mbuf_dma_tag,
1801                                                    sc->rxring_m_dmamap[i]);
1802                                 sc->rxring_m_dmamap[i] = NULL;
1803                         }
1804         }
1805         if (sc->txring != NULL) {
1806                 if (sc->txring_physaddr != 0) {
1807                         bus_dmamap_unload(sc->desc_dma_tag,
1808                                           sc->txring_dma_map);
1809                         sc->txring_physaddr = 0;
1810                 }
1811                 bus_dmamem_free(sc->desc_dma_tag, sc->txring,
1812                                 sc->txring_dma_map);
1813                 sc->txring = NULL;
1814                 for (i = 0; i < CGEM_NUM_TX_DESCS; i++)
1815                         if (sc->txring_m_dmamap[i] != NULL) {
1816                                 bus_dmamap_destroy(sc->mbuf_dma_tag,
1817                                                    sc->txring_m_dmamap[i]);
1818                                 sc->txring_m_dmamap[i] = NULL;
1819                         }
1820         }
1821         if (sc->desc_dma_tag != NULL) {
1822                 bus_dma_tag_destroy(sc->desc_dma_tag);
1823                 sc->desc_dma_tag = NULL;
1824         }
1825         if (sc->mbuf_dma_tag != NULL) {
1826                 bus_dma_tag_destroy(sc->mbuf_dma_tag);
1827                 sc->mbuf_dma_tag = NULL;
1828         }
1829
1830         bus_generic_detach(dev);
1831
1832         CGEM_LOCK_DESTROY(sc);
1833
1834         return (0);
1835 }
1836
1837 static device_method_t cgem_methods[] = {
1838         /* Device interface */
1839         DEVMETHOD(device_probe,         cgem_probe),
1840         DEVMETHOD(device_attach,        cgem_attach),
1841         DEVMETHOD(device_detach,        cgem_detach),
1842
1843         /* Bus interface */
1844         DEVMETHOD(bus_child_detached,   cgem_child_detached),
1845
1846         /* MII interface */
1847         DEVMETHOD(miibus_readreg,       cgem_miibus_readreg),
1848         DEVMETHOD(miibus_writereg,      cgem_miibus_writereg),
1849         DEVMETHOD(miibus_statchg,       cgem_miibus_statchg),
1850         DEVMETHOD(miibus_linkchg,       cgem_miibus_linkchg),
1851
1852         DEVMETHOD_END
1853 };
1854
1855 static driver_t cgem_driver = {
1856         "cgem",
1857         cgem_methods,
1858         sizeof(struct cgem_softc),
1859 };
1860
1861 DRIVER_MODULE(cgem, simplebus, cgem_driver, cgem_devclass, NULL, NULL);
1862 DRIVER_MODULE(miibus, cgem, miibus_driver, miibus_devclass, NULL, NULL);
1863 MODULE_DEPEND(cgem, miibus, 1, 1, 1);
1864 MODULE_DEPEND(cgem, ether, 1, 1, 1);