sys/dev/ixgb/if_ixgb.c

   1 /*******************************************************************************
   2
   3 Copyright (c) 2001-2004, Intel Corporation
   4 All rights reserved.
   5
   6 Redistribution and use in source and binary forms, with or without
   7 modification, are permitted provided that the following conditions are met:
   8
   9  1. Redistributions of source code must retain the above copyright notice,
  10     this list of conditions and the following disclaimer.
  11
  12  2. Redistributions in binary form must reproduce the above copyright
  13     notice, this list of conditions and the following disclaimer in the
  14     documentation and/or other materials provided with the distribution.
  15
  16  3. Neither the name of the Intel Corporation nor the names of its
  17     contributors may be used to endorse or promote products derived from
  18     this software without specific prior written permission.
  19
  20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30 POSSIBILITY OF SUCH DAMAGE.
  31
  32 ***************************************************************************/
  33
  34 /*$FreeBSD$*/
  35
  36 #ifdef HAVE_KERNEL_OPTION_HEADERS
  37 #include "opt_device_polling.h"
  38 #endif
  39
  40 #include <dev/ixgb/if_ixgb.h>
  41
  42 /*********************************************************************
  43  *  Set this to one to display debug statistics
  44  *********************************************************************/
  45 int             ixgb_display_debug_stats = 0;
  46
  47 /*********************************************************************
  48  *  Linked list of board private structures for all NICs found
  49  *********************************************************************/
  50
  51 struct adapter *ixgb_adapter_list = NULL;
  52
  53
  54
  55 /*********************************************************************
  56  *  Driver version
  57  *********************************************************************/
  58
  59 char            ixgb_driver_version[] = "1.0.6";
  60 char            ixgb_copyright[] = "Copyright (c) 2001-2004 Intel Corporation.";
  61
  62 /*********************************************************************
  63  *  PCI Device ID Table
  64  *
  65  *  Used by probe to select devices to load on
  66  *  Last field stores an index into ixgb_strings
  67  *  Last entry must be all 0s
  68  *
  69  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  70  *********************************************************************/
  71
  72 static ixgb_vendor_info_t ixgb_vendor_info_array[] =
  73 {
  74         /* Intel(R) PRO/10000 Network Connection */
  75         {INTEL_VENDOR_ID, IXGB_DEVICE_ID_82597EX, PCI_ANY_ID, PCI_ANY_ID, 0},
  76         {INTEL_VENDOR_ID, IXGB_DEVICE_ID_82597EX_SR, PCI_ANY_ID, PCI_ANY_ID, 0},
  77         /* required last entry */
  78         {0, 0, 0, 0, 0}
  79 };
  80
  81 /*********************************************************************
  82  *  Table of branding strings for all supported NICs.
  83  *********************************************************************/
  84
  85 static char    *ixgb_strings[] = {
  86         "Intel(R) PRO/10GbE Network Driver"
  87 };
  88
  89 /*********************************************************************
  90  *  Function prototypes
  91  *********************************************************************/
  92 static int      ixgb_probe(device_t);
  93 static int      ixgb_attach(device_t);
  94 static int      ixgb_detach(device_t);
  95 static int      ixgb_shutdown(device_t);
  96 static void     ixgb_intr(void *);
  97 static void     ixgb_start(struct ifnet *);
  98 static void     ixgb_start_locked(struct ifnet *);
  99 static int      ixgb_ioctl(struct ifnet *, IOCTL_CMD_TYPE, caddr_t);
 100 static void     ixgb_watchdog(struct ifnet *);
 101 static void     ixgb_init(void *);
 102 static void     ixgb_init_locked(struct adapter *);
 103 static void     ixgb_stop(void *);
 104 static void     ixgb_media_status(struct ifnet *, struct ifmediareq *);
 105 static int      ixgb_media_change(struct ifnet *);
 106 static void     ixgb_identify_hardware(struct adapter *);
 107 static int      ixgb_allocate_pci_resources(struct adapter *);
 108 static void     ixgb_free_pci_resources(struct adapter *);
 109 static void     ixgb_local_timer(void *);
 110 static int      ixgb_hardware_init(struct adapter *);
 111 static int      ixgb_setup_interface(device_t, struct adapter *);
 112 static int      ixgb_setup_transmit_structures(struct adapter *);
 113 static void     ixgb_initialize_transmit_unit(struct adapter *);
 114 static int      ixgb_setup_receive_structures(struct adapter *);
 115 static void     ixgb_initialize_receive_unit(struct adapter *);
 116 static void     ixgb_enable_intr(struct adapter *);
 117 static void     ixgb_disable_intr(struct adapter *);
 118 static void     ixgb_free_transmit_structures(struct adapter *);
 119 static void     ixgb_free_receive_structures(struct adapter *);
 120 static void     ixgb_update_stats_counters(struct adapter *);
 121 static void     ixgb_clean_transmit_interrupts(struct adapter *);
 122 static int      ixgb_allocate_receive_structures(struct adapter *);
 123 static int      ixgb_allocate_transmit_structures(struct adapter *);
 124 static int      ixgb_process_receive_interrupts(struct adapter *, int);
 125 static void
 126 ixgb_receive_checksum(struct adapter *,
 127                       struct ixgb_rx_desc * rx_desc,
 128                       struct mbuf *);
 129 static void
 130 ixgb_transmit_checksum_setup(struct adapter *,
 131                              struct mbuf *,
 132                              u_int8_t *);
 133 static void     ixgb_set_promisc(struct adapter *);
 134 static void     ixgb_disable_promisc(struct adapter *);
 135 static void     ixgb_set_multi(struct adapter *);
 136 static void     ixgb_print_hw_stats(struct adapter *);
 137 static void     ixgb_print_link_status(struct adapter *);
 138 static int
 139 ixgb_get_buf(int i, struct adapter *,
 140              struct mbuf *);
 141 static void     ixgb_enable_vlans(struct adapter * adapter);
 142 static int      ixgb_encap(struct adapter * adapter, struct mbuf * m_head);
 143 static int      ixgb_sysctl_stats(SYSCTL_HANDLER_ARGS);
 144 static int
 145 ixgb_dma_malloc(struct adapter *, bus_size_t,
 146                 struct ixgb_dma_alloc *, int);
 147 static void     ixgb_dma_free(struct adapter *, struct ixgb_dma_alloc *);
 148 #ifdef DEVICE_POLLING
 149 static poll_handler_t ixgb_poll;
 150 #endif
 151
 152 /*********************************************************************
 153  *  FreeBSD Device Interface Entry Points
 154  *********************************************************************/
 155
 156 static device_method_t ixgb_methods[] = {
 157         /* Device interface */
 158         DEVMETHOD(device_probe, ixgb_probe),
 159         DEVMETHOD(device_attach, ixgb_attach),
 160         DEVMETHOD(device_detach, ixgb_detach),
 161         DEVMETHOD(device_shutdown, ixgb_shutdown),
 162         {0, 0}
 163 };
 164
 165 static driver_t ixgb_driver = {
 166         "ixgb", ixgb_methods, sizeof(struct adapter),
 167 };
 168
 169 static devclass_t ixgb_devclass;
 170 DRIVER_MODULE(ixgb, pci, ixgb_driver, ixgb_devclass, 0, 0);
 171
 172 MODULE_DEPEND(ixgb, pci, 1, 1, 1);
 173 MODULE_DEPEND(ixgb, ether, 1, 1, 1);
 174
 175 /* some defines for controlling descriptor fetches in h/w */
 176 #define RXDCTL_PTHRESH_DEFAULT 128      /* chip considers prefech below this */
 177 #define RXDCTL_HTHRESH_DEFAULT 16       /* chip will only prefetch if tail is
 178                                          * pushed this many descriptors from
 179                                          * head */
 180 #define RXDCTL_WTHRESH_DEFAULT 0        /* chip writes back at this many or RXT0 */
 181
 182
 183 /*********************************************************************
 184  *  Device identification routine
 185  *
 186  *  ixgb_probe determines if the driver should be loaded on
 187  *  adapter based on PCI vendor/device id of the adapter.
 188  *
 189  *  return 0 on success, positive on failure
 190  *********************************************************************/
 191
 192 static int
 193 ixgb_probe(device_t dev)
 194 {
 195         ixgb_vendor_info_t *ent;
 196
 197         u_int16_t       pci_vendor_id = 0;
 198         u_int16_t       pci_device_id = 0;
 199         u_int16_t       pci_subvendor_id = 0;
 200         u_int16_t       pci_subdevice_id = 0;
 201         char            adapter_name[60];
 202
 203         INIT_DEBUGOUT("ixgb_probe: begin");
 204
 205         pci_vendor_id = pci_get_vendor(dev);
 206         if (pci_vendor_id != IXGB_VENDOR_ID)
 207                 return (ENXIO);
 208
 209         pci_device_id = pci_get_device(dev);
 210         pci_subvendor_id = pci_get_subvendor(dev);
 211         pci_subdevice_id = pci_get_subdevice(dev);
 212
 213         ent = ixgb_vendor_info_array;
 214         while (ent->vendor_id != 0) {
 215                 if ((pci_vendor_id == ent->vendor_id) &&
 216                     (pci_device_id == ent->device_id) &&
 217
 218                     ((pci_subvendor_id == ent->subvendor_id) ||
 219                      (ent->subvendor_id == PCI_ANY_ID)) &&
 220
 221                     ((pci_subdevice_id == ent->subdevice_id) ||
 222                      (ent->subdevice_id == PCI_ANY_ID))) {
 223                         sprintf(adapter_name, "%s, Version - %s",
 224                                 ixgb_strings[ent->index],
 225                                 ixgb_driver_version);
 226                         device_set_desc_copy(dev, adapter_name);
 227                         return (BUS_PROBE_DEFAULT);
 228                 }
 229                 ent++;
 230         }
 231
 232         return (ENXIO);
 233 }
 234
 235 /*********************************************************************
 236  *  Device initialization routine
 237  *
 238  *  The attach entry point is called when the driver is being loaded.
 239  *  This routine identifies the type of hardware, allocates all resources
 240  *  and initializes the hardware.
 241  *
 242  *  return 0 on success, positive on failure
 243  *********************************************************************/
 244
 245 static int
 246 ixgb_attach(device_t dev)
 247 {
 248         struct adapter *adapter;
 249         int             tsize, rsize;
 250         int             error = 0;
 251
 252         printf("ixgb%d: %s\n", device_get_unit(dev), ixgb_copyright);
 253         INIT_DEBUGOUT("ixgb_attach: begin");
 254
 255         /* Allocate, clear, and link in our adapter structure */
 256         if (!(adapter = device_get_softc(dev))) {
 257                 printf("ixgb: adapter structure allocation failed\n");
 258                 return (ENOMEM);
 259         }
 260         bzero(adapter, sizeof(struct adapter));
 261         adapter->dev = dev;
 262         adapter->osdep.dev = dev;
 263         adapter->unit = device_get_unit(dev);
 264         IXGB_LOCK_INIT(adapter, device_get_nameunit(dev));
 265
 266         if (ixgb_adapter_list != NULL)
 267                 ixgb_adapter_list->prev = adapter;
 268         adapter->next = ixgb_adapter_list;
 269         ixgb_adapter_list = adapter;
 270
 271         /* SYSCTL APIs */
 272         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 273                         SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 274                         OID_AUTO, "stats", CTLTYPE_INT | CTLFLAG_RW,
 275                         (void *)adapter, 0,
 276                         ixgb_sysctl_stats, "I", "Statistics");
 277
 278         callout_init(&adapter->timer, CALLOUT_MPSAFE);
 279
 280         /* Determine hardware revision */
 281         ixgb_identify_hardware(adapter);
 282
 283         /* Parameters (to be read from user) */
 284         adapter->num_tx_desc = IXGB_MAX_TXD;
 285         adapter->num_rx_desc = IXGB_MAX_RXD;
 286         adapter->tx_int_delay = TIDV;
 287         adapter->rx_int_delay = RDTR;
 288         adapter->rx_buffer_len = IXGB_RXBUFFER_2048;
 289
 290         adapter->hw.fc.high_water = FCRTH;
 291         adapter->hw.fc.low_water = FCRTL;
 292         adapter->hw.fc.pause_time = FCPAUSE;
 293         adapter->hw.fc.send_xon = TRUE;
 294         adapter->hw.fc.type = FLOW_CONTROL;
 295
 296
 297         /* Set the max frame size assuming standard ethernet sized frames */
 298         adapter->hw.max_frame_size =
 299                 ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN;
 300
 301         if (ixgb_allocate_pci_resources(adapter)) {
 302                 printf("ixgb%d: Allocation of PCI resources failed\n",
 303                        adapter->unit);
 304                 error = ENXIO;
 305                 goto err_pci;
 306         }
 307         tsize = IXGB_ROUNDUP(adapter->num_tx_desc *
 308                              sizeof(struct ixgb_tx_desc), 4096);
 309
 310         /* Allocate Transmit Descriptor ring */
 311         if (ixgb_dma_malloc(adapter, tsize, &adapter->txdma, BUS_DMA_NOWAIT)) {
 312                 printf("ixgb%d: Unable to allocate TxDescriptor memory\n",
 313                        adapter->unit);
 314                 error = ENOMEM;
 315                 goto err_tx_desc;
 316         }
 317         adapter->tx_desc_base = (struct ixgb_tx_desc *) adapter->txdma.dma_vaddr;
 318
 319         rsize = IXGB_ROUNDUP(adapter->num_rx_desc *
 320                              sizeof(struct ixgb_rx_desc), 4096);
 321
 322         /* Allocate Receive Descriptor ring */
 323         if (ixgb_dma_malloc(adapter, rsize, &adapter->rxdma, BUS_DMA_NOWAIT)) {
 324                 printf("ixgb%d: Unable to allocate rx_desc memory\n",
 325                        adapter->unit);
 326                 error = ENOMEM;
 327                 goto err_rx_desc;
 328         }
 329         adapter->rx_desc_base = (struct ixgb_rx_desc *) adapter->rxdma.dma_vaddr;
 330
 331         /* Allocate multicast array memory. */
 332         adapter->mta = malloc(sizeof(u_int8_t) * IXGB_ETH_LENGTH_OF_ADDRESS *
 333             MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
 334         if (adapter->mta == NULL) {
 335                 device_printf(dev, "Can not allocate multicast setup array\n");
 336                 error = ENOMEM;
 337                 goto err_hw_init;
 338         }
 339
 340         /* Initialize the hardware */
 341         if (ixgb_hardware_init(adapter)) {
 342                 printf("ixgb%d: Unable to initialize the hardware\n",
 343                        adapter->unit);
 344                 error = EIO;
 345                 goto err_hw_init;
 346         }
 347         /* Setup OS specific network interface */
 348         if (ixgb_setup_interface(dev, adapter) != 0)
 349                 goto err_hw_init;
 350
 351         /* Initialize statistics */
 352         ixgb_clear_hw_cntrs(&adapter->hw);
 353         ixgb_update_stats_counters(adapter);
 354
 355         INIT_DEBUGOUT("ixgb_attach: end");
 356         return (0);
 357
 358 err_hw_init:
 359         ixgb_dma_free(adapter, &adapter->rxdma);
 360 err_rx_desc:
 361         ixgb_dma_free(adapter, &adapter->txdma);
 362 err_tx_desc:
 363 err_pci:
 364         if (adapter->ifp != NULL)
 365                 if_free(adapter->ifp);
 366         ixgb_free_pci_resources(adapter);
 367         sysctl_ctx_free(&adapter->sysctl_ctx);
 368         free(adapter->mta, M_DEVBUF);
 369         return (error);
 370
 371 }
 372
 373 /*********************************************************************
 374  *  Device removal routine
 375  *
 376  *  The detach entry point is called when the driver is being removed.
 377  *  This routine stops the adapter and deallocates all the resources
 378  *  that were allocated for driver operation.
 379  *
 380  *  return 0 on success, positive on failure
 381  *********************************************************************/
 382
 383 static int
 384 ixgb_detach(device_t dev)
 385 {
 386         struct adapter *adapter = device_get_softc(dev);
 387         struct ifnet   *ifp = adapter->ifp;
 388
 389         INIT_DEBUGOUT("ixgb_detach: begin");
 390
 391 #ifdef DEVICE_POLLING
 392         if (ifp->if_capenable & IFCAP_POLLING)
 393                 ether_poll_deregister(ifp);
 394 #endif
 395
 396         IXGB_LOCK(adapter);
 397         adapter->in_detach = 1;
 398
 399         ixgb_stop(adapter);
 400         IXGB_UNLOCK(adapter);
 401
 402 #if __FreeBSD_version < 500000
 403         ether_ifdetach(adapter->ifp, ETHER_BPF_SUPPORTED);
 404 #else
 405         ether_ifdetach(adapter->ifp);
 406 #endif
 407         ixgb_free_pci_resources(adapter);
 408 #if __FreeBSD_version >= 500000
 409         if_free(adapter->ifp);
 410 #endif
 411
 412         /* Free Transmit Descriptor ring */
 413         if (adapter->tx_desc_base) {
 414                 ixgb_dma_free(adapter, &adapter->txdma);
 415                 adapter->tx_desc_base = NULL;
 416         }
 417         /* Free Receive Descriptor ring */
 418         if (adapter->rx_desc_base) {
 419                 ixgb_dma_free(adapter, &adapter->rxdma);
 420                 adapter->rx_desc_base = NULL;
 421         }
 422         /* Remove from the adapter list */
 423         if (ixgb_adapter_list == adapter)
 424                 ixgb_adapter_list = adapter->next;
 425         if (adapter->next != NULL)
 426                 adapter->next->prev = adapter->prev;
 427         if (adapter->prev != NULL)
 428                 adapter->prev->next = adapter->next;
 429         free(adapter->mta, M_DEVBUF);
 430
 431         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
 432         ifp->if_timer = 0;
 433
 434         IXGB_LOCK_DESTROY(adapter);
 435         return (0);
 436 }
 437
 438 /*********************************************************************
 439  *
 440  *  Shutdown entry point
 441  *
 442  **********************************************************************/
 443
 444 static int
 445 ixgb_shutdown(device_t dev)
 446 {
 447         struct adapter *adapter = device_get_softc(dev);
 448         IXGB_LOCK(adapter);
 449         ixgb_stop(adapter);
 450         IXGB_UNLOCK(adapter);
 451         return (0);
 452 }
 453
 454
 455 /*********************************************************************
 456  *  Transmit entry point
 457  *
 458  *  ixgb_start is called by the stack to initiate a transmit.
 459  *  The driver will remain in this routine as long as there are
 460  *  packets to transmit and transmit resources are available.
 461  *  In case resources are not available stack is notified and
 462  *  the packet is requeued.
 463  **********************************************************************/
 464
 465 static void
 466 ixgb_start_locked(struct ifnet * ifp)
 467 {
 468         struct mbuf    *m_head;
 469         struct adapter *adapter = ifp->if_softc;
 470
 471         IXGB_LOCK_ASSERT(adapter);
 472
 473         if (!adapter->link_active)
 474                 return;
 475
 476         while (ifp->if_snd.ifq_head != NULL) {
 477                 IF_DEQUEUE(&ifp->if_snd, m_head);
 478
 479                 if (m_head == NULL)
 480                         break;
 481
 482                 if (ixgb_encap(adapter, m_head)) {
 483                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 484                         IF_PREPEND(&ifp->if_snd, m_head);
 485                         break;
 486                 }
 487                 /* Send a copy of the frame to the BPF listener */
 488 #if __FreeBSD_version < 500000
 489                 if (ifp->if_bpf)
 490                         bpf_mtap(ifp, m_head);
 491 #else
 492                 ETHER_BPF_MTAP(ifp, m_head);
 493 #endif
 494                 /* Set timeout in case hardware has problems transmitting */
 495                 ifp->if_timer = IXGB_TX_TIMEOUT;
 496
 497         }
 498         return;
 499 }
 500
 501 static void
 502 ixgb_start(struct ifnet *ifp)
 503 {
 504         struct adapter *adapter = ifp->if_softc;
 505
 506         IXGB_LOCK(adapter);
 507         ixgb_start_locked(ifp);
 508         IXGB_UNLOCK(adapter);
 509         return;
 510 }
 511
 512 /*********************************************************************
 513  *  Ioctl entry point
 514  *
 515  *  ixgb_ioctl is called when the user wants to configure the
 516  *  interface.
 517  *
 518  *  return 0 on success, positive on failure
 519  **********************************************************************/
 520
 521 static int
 522 ixgb_ioctl(struct ifnet * ifp, IOCTL_CMD_TYPE command, caddr_t data)
 523 {
 524         int             mask, error = 0;
 525         struct ifreq   *ifr = (struct ifreq *) data;
 526         struct adapter *adapter = ifp->if_softc;
 527
 528         if (adapter->in_detach)
 529                 goto out;
 530
 531         switch (command) {
 532         case SIOCSIFADDR:
 533         case SIOCGIFADDR:
 534                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCxIFADDR (Get/Set Interface Addr)");
 535                 ether_ioctl(ifp, command, data);
 536                 break;
 537         case SIOCSIFMTU:
 538                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
 539                 if (ifr->ifr_mtu > IXGB_MAX_JUMBO_FRAME_SIZE - ETHER_HDR_LEN) {
 540                         error = EINVAL;
 541                 } else {
 542                         IXGB_LOCK(adapter);
 543                         ifp->if_mtu = ifr->ifr_mtu;
 544                         adapter->hw.max_frame_size =
 545                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 546
 547                         ixgb_init_locked(adapter);
 548                         IXGB_UNLOCK(adapter);
 549                 }
 550                 break;
 551         case SIOCSIFFLAGS:
 552                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFFLAGS (Set Interface Flags)");
 553                 IXGB_LOCK(adapter);
 554                 if (ifp->if_flags & IFF_UP) {
 555                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 556                                 ixgb_init_locked(adapter);
 557                         }
 558                         ixgb_disable_promisc(adapter);
 559                         ixgb_set_promisc(adapter);
 560                 } else {
 561                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 562                                 ixgb_stop(adapter);
 563                         }
 564                 }
 565                 IXGB_UNLOCK(adapter);
 566                 break;
 567         case SIOCADDMULTI:
 568         case SIOCDELMULTI:
 569                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
 570                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 571                         IXGB_LOCK(adapter);
 572                         ixgb_disable_intr(adapter);
 573                         ixgb_set_multi(adapter);
 574                         ixgb_enable_intr(adapter);
 575                         IXGB_UNLOCK(adapter);
 576                 }
 577                 break;
 578         case SIOCSIFMEDIA:
 579         case SIOCGIFMEDIA:
 580                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCxIFMEDIA (Get/Set Interface Media)");
 581                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
 582                 break;
 583         case SIOCSIFCAP:
 584                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
 585                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 586 #ifdef DEVICE_POLLING
 587                 if (mask & IFCAP_POLLING) {
 588                         if (ifr->ifr_reqcap & IFCAP_POLLING) {
 589                                 error = ether_poll_register(ixgb_poll, ifp);
 590                                 if (error)
 591                                         return(error);
 592                                 IXGB_LOCK(adapter);
 593                                 ixgb_disable_intr(adapter);
 594                                 ifp->if_capenable |= IFCAP_POLLING;
 595                                 IXGB_UNLOCK(adapter);
 596                         } else {
 597                                 error = ether_poll_deregister(ifp);
 598                                 /* Enable interrupt even in error case */
 599                                 IXGB_LOCK(adapter);
 600                                 ixgb_enable_intr(adapter);
 601                                 ifp->if_capenable &= ~IFCAP_POLLING;
 602                                 IXGB_UNLOCK(adapter);
 603                         }
 604                 }
 605 #endif /* DEVICE_POLLING */
 606                 if (mask & IFCAP_HWCSUM) {
 607                         if (IFCAP_HWCSUM & ifp->if_capenable)
 608                                 ifp->if_capenable &= ~IFCAP_HWCSUM;
 609                         else
 610                                 ifp->if_capenable |= IFCAP_HWCSUM;
 611                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 612                                 ixgb_init(adapter);
 613                 }
 614                 break;
 615         default:
 616                 IOCTL_DEBUGOUT1("ioctl received: UNKNOWN (0x%X)\n", (int)command);
 617                 error = EINVAL;
 618         }
 619
 620 out:
 621         return (error);
 622 }
 623
 624 /*********************************************************************
 625  *  Watchdog entry point
 626  *
 627  *  This routine is called whenever hardware quits transmitting.
 628  *
 629  **********************************************************************/
 630
 631 static void
 632 ixgb_watchdog(struct ifnet * ifp)
 633 {
 634         struct adapter *adapter;
 635         adapter = ifp->if_softc;
 636
 637         /*
 638          * If we are in this routine because of pause frames, then don't
 639          * reset the hardware.
 640          */
 641         if (IXGB_READ_REG(&adapter->hw, STATUS) & IXGB_STATUS_TXOFF) {
 642                 ifp->if_timer = IXGB_TX_TIMEOUT;
 643                 return;
 644         }
 645         printf("ixgb%d: watchdog timeout -- resetting\n", adapter->unit);
 646
 647         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 648
 649
 650         ixgb_stop(adapter);
 651         ixgb_init(adapter);
 652
 653
 654         ifp->if_oerrors++;
 655
 656         return;
 657 }
 658
 659 /*********************************************************************
 660  *  Init entry point
 661  *
 662  *  This routine is used in two ways. It is used by the stack as
 663  *  init entry point in network interface structure. It is also used
 664  *  by the driver as a hw/sw initialization routine to get to a
 665  *  consistent state.
 666  *
 667  *  return 0 on success, positive on failure
 668  **********************************************************************/
 669
 670 static void
 671 ixgb_init_locked(struct adapter *adapter)
 672 {
 673         struct ifnet   *ifp;
 674
 675         INIT_DEBUGOUT("ixgb_init: begin");
 676
 677         IXGB_LOCK_ASSERT(adapter);
 678
 679         ixgb_stop(adapter);
 680
 681         /* Get the latest mac address, User can use a LAA */
 682         bcopy(IF_LLADDR(adapter->ifp), adapter->hw.curr_mac_addr,
 683               IXGB_ETH_LENGTH_OF_ADDRESS);
 684
 685         /* Initialize the hardware */
 686         if (ixgb_hardware_init(adapter)) {
 687                 printf("ixgb%d: Unable to initialize the hardware\n",
 688                        adapter->unit);
 689                 return;
 690         }
 691         ixgb_enable_vlans(adapter);
 692
 693         /* Prepare transmit descriptors and buffers */
 694         if (ixgb_setup_transmit_structures(adapter)) {
 695                 printf("ixgb%d: Could not setup transmit structures\n",
 696                        adapter->unit);
 697                 ixgb_stop(adapter);
 698                 return;
 699         }
 700         ixgb_initialize_transmit_unit(adapter);
 701
 702         /* Setup Multicast table */
 703         ixgb_set_multi(adapter);
 704
 705         /* Prepare receive descriptors and buffers */
 706         if (ixgb_setup_receive_structures(adapter)) {
 707                 printf("ixgb%d: Could not setup receive structures\n",
 708                        adapter->unit);
 709                 ixgb_stop(adapter);
 710                 return;
 711         }
 712         ixgb_initialize_receive_unit(adapter);
 713
 714         /* Don't lose promiscuous settings */
 715         ixgb_set_promisc(adapter);
 716
 717         ifp = adapter->ifp;
 718         ifp->if_drv_flags |= IFF_DRV_RUNNING;
 719         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 720
 721
 722         if (ifp->if_capenable & IFCAP_TXCSUM)
 723                 ifp->if_hwassist = IXGB_CHECKSUM_FEATURES;
 724         else
 725                 ifp->if_hwassist = 0;
 726
 727
 728         /* Enable jumbo frames */
 729         if (ifp->if_mtu > ETHERMTU) {
 730                 uint32_t        temp_reg;
 731                 IXGB_WRITE_REG(&adapter->hw, MFS,
 732                                adapter->hw.max_frame_size << IXGB_MFS_SHIFT);
 733                 temp_reg = IXGB_READ_REG(&adapter->hw, CTRL0);
 734                 temp_reg |= IXGB_CTRL0_JFE;
 735                 IXGB_WRITE_REG(&adapter->hw, CTRL0, temp_reg);
 736         }
 737         callout_reset(&adapter->timer, 2 * hz, ixgb_local_timer, adapter);
 738         ixgb_clear_hw_cntrs(&adapter->hw);
 739 #ifdef DEVICE_POLLING
 740         /*
 741          * Only disable interrupts if we are polling, make sure they are on
 742          * otherwise.
 743          */
 744         if (ifp->if_capenable & IFCAP_POLLING)
 745                 ixgb_disable_intr(adapter);
 746         else
 747 #endif
 748                 ixgb_enable_intr(adapter);
 749
 750         return;
 751 }
 752
 753 static void
 754 ixgb_init(void *arg)
 755 {
 756         struct adapter *adapter = arg;
 757
 758         IXGB_LOCK(adapter);
 759         ixgb_init_locked(adapter);
 760         IXGB_UNLOCK(adapter);
 761         return;
 762 }
 763
 764 #ifdef DEVICE_POLLING
 765 static int
 766 ixgb_poll_locked(struct ifnet * ifp, enum poll_cmd cmd, int count)
 767 {
 768         struct adapter *adapter = ifp->if_softc;
 769         u_int32_t       reg_icr;
 770         int             rx_npkts;
 771
 772         IXGB_LOCK_ASSERT(adapter);
 773
 774         if (cmd == POLL_AND_CHECK_STATUS) {
 775                 reg_icr = IXGB_READ_REG(&adapter->hw, ICR);
 776                 if (reg_icr & (IXGB_INT_RXSEQ | IXGB_INT_LSC)) {
 777                         callout_stop(&adapter->timer);
 778                         ixgb_check_for_link(&adapter->hw);
 779                         ixgb_print_link_status(adapter);
 780                         callout_reset(&adapter->timer, 2 * hz, ixgb_local_timer,
 781                             adapter);
 782                 }
 783         }
 784         rx_npkts = ixgb_process_receive_interrupts(adapter, count);
 785         ixgb_clean_transmit_interrupts(adapter);
 786
 787         if (ifp->if_snd.ifq_head != NULL)
 788                 ixgb_start_locked(ifp);
 789         return (rx_npkts);
 790 }
 791
 792 static int
 793 ixgb_poll(struct ifnet * ifp, enum poll_cmd cmd, int count)
 794 {
 795         struct adapter *adapter = ifp->if_softc;
 796         int rx_npkts = 0;
 797
 798         IXGB_LOCK(adapter);
 799         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 800                 rx_npkts = ixgb_poll_locked(ifp, cmd, count);
 801         IXGB_UNLOCK(adapter);
 802         return (rx_npkts);
 803 }
 804 #endif /* DEVICE_POLLING */
 805
 806 /*********************************************************************
 807  *
 808  *  Interrupt Service routine
 809  *
 810  **********************************************************************/
 811
 812 static void
 813 ixgb_intr(void *arg)
 814 {
 815         u_int32_t       loop_cnt = IXGB_MAX_INTR;
 816         u_int32_t       reg_icr;
 817         struct ifnet   *ifp;
 818         struct adapter *adapter = arg;
 819         boolean_t       rxdmt0 = FALSE;
 820
 821         IXGB_LOCK(adapter);
 822
 823         ifp = adapter->ifp;
 824
 825 #ifdef DEVICE_POLLING
 826         if (ifp->if_capenable & IFCAP_POLLING) {
 827                 IXGB_UNLOCK(adapter);
 828                 return;
 829         }
 830 #endif
 831
 832         reg_icr = IXGB_READ_REG(&adapter->hw, ICR);
 833         if (reg_icr == 0) {
 834                 IXGB_UNLOCK(adapter);
 835                 return;
 836         }
 837
 838         if (reg_icr & IXGB_INT_RXDMT0)
 839                 rxdmt0 = TRUE;
 840
 841 #ifdef _SV_
 842         if (reg_icr & IXGB_INT_RXDMT0)
 843                 adapter->sv_stats.icr_rxdmt0++;
 844         if (reg_icr & IXGB_INT_RXO)
 845                 adapter->sv_stats.icr_rxo++;
 846         if (reg_icr & IXGB_INT_RXT0)
 847                 adapter->sv_stats.icr_rxt0++;
 848         if (reg_icr & IXGB_INT_TXDW)
 849                 adapter->sv_stats.icr_TXDW++;
 850 #endif                          /* _SV_ */
 851
 852         /* Link status change */
 853         if (reg_icr & (IXGB_INT_RXSEQ | IXGB_INT_LSC)) {
 854                 callout_stop(&adapter->timer);
 855                 ixgb_check_for_link(&adapter->hw);
 856                 ixgb_print_link_status(adapter);
 857                 callout_reset(&adapter->timer, 2 * hz, ixgb_local_timer,
 858                     adapter);
 859         }
 860         while (loop_cnt > 0) {
 861                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 862                         ixgb_process_receive_interrupts(adapter, -1);
 863                         ixgb_clean_transmit_interrupts(adapter);
 864                 }
 865                 loop_cnt--;
 866         }
 867
 868         if (rxdmt0 && adapter->raidc) {
 869                 IXGB_WRITE_REG(&adapter->hw, IMC, IXGB_INT_RXDMT0);
 870                 IXGB_WRITE_REG(&adapter->hw, IMS, IXGB_INT_RXDMT0);
 871         }
 872         if (ifp->if_drv_flags & IFF_DRV_RUNNING && ifp->if_snd.ifq_head != NULL)
 873                 ixgb_start_locked(ifp);
 874
 875         IXGB_UNLOCK(adapter);
 876         return;
 877 }
 878
 879
 880 /*********************************************************************
 881  *
 882  *  Media Ioctl callback
 883  *
 884  *  This routine is called whenever the user queries the status of
 885  *  the interface using ifconfig.
 886  *
 887  **********************************************************************/
 888 static void
 889 ixgb_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
 890 {
 891         struct adapter *adapter = ifp->if_softc;
 892
 893         INIT_DEBUGOUT("ixgb_media_status: begin");
 894
 895         ixgb_check_for_link(&adapter->hw);
 896         ixgb_print_link_status(adapter);
 897
 898         ifmr->ifm_status = IFM_AVALID;
 899         ifmr->ifm_active = IFM_ETHER;
 900
 901         if (!adapter->hw.link_up)
 902                 return;
 903
 904         ifmr->ifm_status |= IFM_ACTIVE;
 905         ifmr->ifm_active |= IFM_1000_SX | IFM_FDX;
 906
 907         return;
 908 }
 909
 910 /*********************************************************************
 911  *
 912  *  Media Ioctl callback
 913  *
 914  *  This routine is called when the user changes speed/duplex using
 915  *  media/mediopt option with ifconfig.
 916  *
 917  **********************************************************************/
 918 static int
 919 ixgb_media_change(struct ifnet * ifp)
 920 {
 921         struct adapter *adapter = ifp->if_softc;
 922         struct ifmedia *ifm = &adapter->media;
 923
 924         INIT_DEBUGOUT("ixgb_media_change: begin");
 925
 926         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
 927                 return (EINVAL);
 928
 929         return (0);
 930 }
 931
 932 /*********************************************************************
 933  *
 934  *  This routine maps the mbufs to tx descriptors.
 935  *
 936  *  return 0 on success, positive on failure
 937  **********************************************************************/
 938
 939 static int
 940 ixgb_encap(struct adapter * adapter, struct mbuf * m_head)
 941 {
 942         u_int8_t        txd_popts;
 943         int             i, j, error, nsegs;
 944
 945 #if __FreeBSD_version < 500000
 946         struct ifvlan  *ifv = NULL;
 947 #endif
 948         bus_dma_segment_t segs[IXGB_MAX_SCATTER];
 949         bus_dmamap_t    map;
 950         struct ixgb_buffer *tx_buffer = NULL;
 951         struct ixgb_tx_desc *current_tx_desc = NULL;
 952         struct ifnet   *ifp = adapter->ifp;
 953
 954         /*
 955          * Force a cleanup if number of TX descriptors available hits the
 956          * threshold
 957          */
 958         if (adapter->num_tx_desc_avail <= IXGB_TX_CLEANUP_THRESHOLD) {
 959                 ixgb_clean_transmit_interrupts(adapter);
 960         }
 961         if (adapter->num_tx_desc_avail <= IXGB_TX_CLEANUP_THRESHOLD) {
 962                 adapter->no_tx_desc_avail1++;
 963                 return (ENOBUFS);
 964         }
 965         /*
 966          * Map the packet for DMA.
 967          */
 968         if (bus_dmamap_create(adapter->txtag, BUS_DMA_NOWAIT, &map)) {
 969                 adapter->no_tx_map_avail++;
 970                 return (ENOMEM);
 971         }
 972         error = bus_dmamap_load_mbuf_sg(adapter->txtag, map, m_head, segs,
 973                                         &nsegs, BUS_DMA_NOWAIT);
 974         if (error != 0) {
 975                 adapter->no_tx_dma_setup++;
 976                 printf("ixgb%d: ixgb_encap: bus_dmamap_load_mbuf failed; "
 977                        "error %u\n", adapter->unit, error);
 978                 bus_dmamap_destroy(adapter->txtag, map);
 979                 return (error);
 980         }
 981         KASSERT(nsegs != 0, ("ixgb_encap: empty packet"));
 982
 983         if (nsegs > adapter->num_tx_desc_avail) {
 984                 adapter->no_tx_desc_avail2++;
 985                 bus_dmamap_destroy(adapter->txtag, map);
 986                 return (ENOBUFS);
 987         }
 988         if (ifp->if_hwassist > 0) {
 989                 ixgb_transmit_checksum_setup(adapter, m_head,
 990                                              &txd_popts);
 991         } else
 992                 txd_popts = 0;
 993
 994         /* Find out if we are in vlan mode */
 995 #if __FreeBSD_version < 500000
 996         if ((m_head->m_flags & (M_PROTO1 | M_PKTHDR)) == (M_PROTO1 | M_PKTHDR) &&
 997             m_head->m_pkthdr.rcvif != NULL &&
 998             m_head->m_pkthdr.rcvif->if_type == IFT_L2VLAN)
 999                 ifv = m_head->m_pkthdr.rcvif->if_softc;
1000 #elseif __FreeBSD_version < 700000
1001         mtag = VLAN_OUTPUT_TAG(ifp, m_head);
1002 #endif
1003         i = adapter->next_avail_tx_desc;
1004         for (j = 0; j < nsegs; j++) {
1005                 tx_buffer = &adapter->tx_buffer_area[i];
1006                 current_tx_desc = &adapter->tx_desc_base[i];
1007
1008                 current_tx_desc->buff_addr = htole64(segs[j].ds_addr);
1009                 current_tx_desc->cmd_type_len = (adapter->txd_cmd | segs[j].ds_len);
1010                 current_tx_desc->popts = txd_popts;
1011                 if (++i == adapter->num_tx_desc)
1012                         i = 0;
1013
1014                 tx_buffer->m_head = NULL;
1015         }
1016
1017         adapter->num_tx_desc_avail -= nsegs;
1018         adapter->next_avail_tx_desc = i;
1019
1020 #if __FreeBSD_version < 500000
1021         if (ifv != NULL) {
1022                 /* Set the vlan id */
1023                 current_tx_desc->vlan = ifv->ifv_tag;
1024 #elseif __FreeBSD_version < 700000
1025         if (mtag != NULL) {
1026                 /* Set the vlan id */
1027                 current_tx_desc->vlan = VLAN_TAG_VALUE(mtag);
1028 #else
1029         if (m_head->m_flags & M_VLANTAG) {
1030                 current_tx_desc->vlan = m_head->m_pkthdr.ether_vtag;
1031 #endif
1032
1033                 /* Tell hardware to add tag */
1034                 current_tx_desc->cmd_type_len |= IXGB_TX_DESC_CMD_VLE;
1035         }
1036         tx_buffer->m_head = m_head;
1037         tx_buffer->map = map;
1038         bus_dmamap_sync(adapter->txtag, map, BUS_DMASYNC_PREWRITE);
1039
1040         /*
1041          * Last Descriptor of Packet needs End Of Packet (EOP)
1042          */
1043         current_tx_desc->cmd_type_len |= (IXGB_TX_DESC_CMD_EOP);
1044
1045         /*
1046          * Advance the Transmit Descriptor Tail (Tdt), this tells the E1000
1047          * that this frame is available to transmit.
1048          */
1049         IXGB_WRITE_REG(&adapter->hw, TDT, i);
1050
1051         return (0);
1052 }
1053
1054 static void
1055 ixgb_set_promisc(struct adapter * adapter)
1056 {
1057
1058         u_int32_t       reg_rctl;
1059         struct ifnet   *ifp = adapter->ifp;
1060
1061         reg_rctl = IXGB_READ_REG(&adapter->hw, RCTL);
1062
1063         if (ifp->if_flags & IFF_PROMISC) {
1064                 reg_rctl |= (IXGB_RCTL_UPE | IXGB_RCTL_MPE);
1065                 IXGB_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1066         } else if (ifp->if_flags & IFF_ALLMULTI) {
1067                 reg_rctl |= IXGB_RCTL_MPE;
1068                 reg_rctl &= ~IXGB_RCTL_UPE;
1069                 IXGB_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1070         }
1071         return;
1072 }
1073
1074 static void
1075 ixgb_disable_promisc(struct adapter * adapter)
1076 {
1077         u_int32_t       reg_rctl;
1078
1079         reg_rctl = IXGB_READ_REG(&adapter->hw, RCTL);
1080
1081         reg_rctl &= (~IXGB_RCTL_UPE);
1082         reg_rctl &= (~IXGB_RCTL_MPE);
1083         IXGB_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1084
1085         return;
1086 }
1087
1088
1089 /*********************************************************************
1090  *  Multicast Update
1091  *
1092  *  This routine is called whenever multicast address list is updated.
1093  *
1094  **********************************************************************/
1095
1096 static void
1097 ixgb_set_multi(struct adapter * adapter)
1098 {
1099         u_int32_t       reg_rctl = 0;
1100         u_int8_t        *mta;
1101         struct ifmultiaddr *ifma;
1102         int             mcnt = 0;
1103         struct ifnet   *ifp = adapter->ifp;
1104
1105         IOCTL_DEBUGOUT("ixgb_set_multi: begin");
1106
1107         mta = adapter->mta;
1108         bzero(mta, sizeof(u_int8_t) * IXGB_ETH_LENGTH_OF_ADDRESS *
1109             MAX_NUM_MULTICAST_ADDRESSES);
1110
1111         if_maddr_rlock(ifp);
1112 #if __FreeBSD_version < 500000
1113         LIST_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1114 #else
1115         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1116 #endif
1117                 if (ifma->ifma_addr->sa_family != AF_LINK)
1118                         continue;
1119
1120                 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1121                       &mta[mcnt * IXGB_ETH_LENGTH_OF_ADDRESS], IXGB_ETH_LENGTH_OF_ADDRESS);
1122                 mcnt++;
1123         }
1124         if_maddr_runlock(ifp);
1125
1126         if (mcnt > MAX_NUM_MULTICAST_ADDRESSES) {
1127                 reg_rctl = IXGB_READ_REG(&adapter->hw, RCTL);
1128                 reg_rctl |= IXGB_RCTL_MPE;
1129                 IXGB_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1130         } else
1131                 ixgb_mc_addr_list_update(&adapter->hw, mta, mcnt, 0);
1132
1133         return;
1134 }
1135
1136
1137 /*********************************************************************
1138  *  Timer routine
1139  *
1140  *  This routine checks for link status and updates statistics.
1141  *
1142  **********************************************************************/
1143
1144 static void
1145 ixgb_local_timer(void *arg)
1146 {
1147         struct ifnet   *ifp;
1148         struct adapter *adapter = arg;
1149         ifp = adapter->ifp;
1150
1151         IXGB_LOCK(adapter);
1152
1153         ixgb_check_for_link(&adapter->hw);
1154         ixgb_print_link_status(adapter);
1155         ixgb_update_stats_counters(adapter);
1156         if (ixgb_display_debug_stats && ifp->if_drv_flags & IFF_DRV_RUNNING) {
1157                 ixgb_print_hw_stats(adapter);
1158         }
1159         callout_reset(&adapter->timer, 2 * hz, ixgb_local_timer, adapter);
1160
1161         IXGB_UNLOCK(adapter);
1162         return;
1163 }
1164
1165 static void
1166 ixgb_print_link_status(struct adapter * adapter)
1167 {
1168         if (adapter->hw.link_up) {
1169                 if (!adapter->link_active) {
1170                         printf("ixgb%d: Link is up %d Mbps %s \n",
1171                                adapter->unit,
1172                                10000,
1173                                "Full Duplex");
1174                         adapter->link_active = 1;
1175                 }
1176         } else {
1177                 if (adapter->link_active) {
1178                         printf("ixgb%d: Link is Down \n", adapter->unit);
1179                         adapter->link_active = 0;
1180                 }
1181         }
1182
1183         return;
1184 }
1185
1186
1187
1188 /*********************************************************************
1189  *
1190  *  This routine disables all traffic on the adapter by issuing a
1191  *  global reset on the MAC and deallocates TX/RX buffers.
1192  *
1193  **********************************************************************/
1194
1195 static void
1196 ixgb_stop(void *arg)
1197 {
1198         struct ifnet   *ifp;
1199         struct adapter *adapter = arg;
1200         ifp = adapter->ifp;
1201
1202         IXGB_LOCK_ASSERT(adapter);
1203
1204         INIT_DEBUGOUT("ixgb_stop: begin\n");
1205         ixgb_disable_intr(adapter);
1206         adapter->hw.adapter_stopped = FALSE;
1207         ixgb_adapter_stop(&adapter->hw);
1208         callout_stop(&adapter->timer);
1209         ixgb_free_transmit_structures(adapter);
1210         ixgb_free_receive_structures(adapter);
1211
1212
1213         /* Tell the stack that the interface is no longer active */
1214         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1215
1216         return;
1217 }
1218
1219
1220 /*********************************************************************
1221  *
1222  *  Determine hardware revision.
1223  *
1224  **********************************************************************/
1225 static void
1226 ixgb_identify_hardware(struct adapter * adapter)
1227 {
1228         device_t        dev = adapter->dev;
1229
1230         /* Make sure our PCI config space has the necessary stuff set */
1231         adapter->hw.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
1232         if (!((adapter->hw.pci_cmd_word & PCIM_CMD_BUSMASTEREN) &&
1233               (adapter->hw.pci_cmd_word & PCIM_CMD_MEMEN))) {
1234                 printf("ixgb%d: Memory Access and/or Bus Master bits were not set!\n",
1235                        adapter->unit);
1236                 adapter->hw.pci_cmd_word |=
1237                         (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN);
1238                 pci_write_config(dev, PCIR_COMMAND, adapter->hw.pci_cmd_word, 2);
1239         }
1240         /* Save off the information about this board */
1241         adapter->hw.vendor_id = pci_get_vendor(dev);
1242         adapter->hw.device_id = pci_get_device(dev);
1243         adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
1244         adapter->hw.subsystem_vendor_id = pci_read_config(dev, PCIR_SUBVEND_0, 2);
1245         adapter->hw.subsystem_id = pci_read_config(dev, PCIR_SUBDEV_0, 2);
1246
1247         /* Set MacType, etc. based on this PCI info */
1248         switch (adapter->hw.device_id) {
1249         case IXGB_DEVICE_ID_82597EX:
1250         case IXGB_DEVICE_ID_82597EX_SR:
1251                 adapter->hw.mac_type = ixgb_82597;
1252                 break;
1253         default:
1254                 INIT_DEBUGOUT1("Unknown device if 0x%x", adapter->hw.device_id);
1255                 printf("ixgb%d: unsupported device id 0x%x\n", adapter->unit, adapter->hw.device_id);
1256         }
1257
1258         return;
1259 }
1260
1261 static int
1262 ixgb_allocate_pci_resources(struct adapter * adapter)
1263 {
1264         int             rid;
1265         device_t        dev = adapter->dev;
1266
1267         rid = IXGB_MMBA;
1268         adapter->res_memory = bus_alloc_resource(dev, SYS_RES_MEMORY,
1269                                                  &rid, 0, ~0, 1,
1270                                                  RF_ACTIVE);
1271         if (!(adapter->res_memory)) {
1272                 printf("ixgb%d: Unable to allocate bus resource: memory\n",
1273                        adapter->unit);
1274                 return (ENXIO);
1275         }
1276         adapter->osdep.mem_bus_space_tag =
1277                 rman_get_bustag(adapter->res_memory);
1278         adapter->osdep.mem_bus_space_handle =
1279                 rman_get_bushandle(adapter->res_memory);
1280         adapter->hw.hw_addr = (uint8_t *) & adapter->osdep.mem_bus_space_handle;
1281
1282         rid = 0x0;
1283         adapter->res_interrupt = bus_alloc_resource(dev, SYS_RES_IRQ,
1284                                                     &rid, 0, ~0, 1,
1285                                                   RF_SHAREABLE | RF_ACTIVE);
1286         if (!(adapter->res_interrupt)) {
1287                 printf("ixgb%d: Unable to allocate bus resource: interrupt\n",
1288                        adapter->unit);
1289                 return (ENXIO);
1290         }
1291         if (bus_setup_intr(dev, adapter->res_interrupt,
1292                            INTR_TYPE_NET | INTR_MPSAFE,
1293                            NULL, (void (*) (void *))ixgb_intr, adapter,
1294                            &adapter->int_handler_tag)) {
1295                 printf("ixgb%d: Error registering interrupt handler!\n",
1296                        adapter->unit);
1297                 return (ENXIO);
1298         }
1299         adapter->hw.back = &adapter->osdep;
1300
1301         return (0);
1302 }
1303
1304 static void
1305 ixgb_free_pci_resources(struct adapter * adapter)
1306 {
1307         device_t        dev = adapter->dev;
1308
1309         if (adapter->res_interrupt != NULL) {
1310                 bus_teardown_intr(dev, adapter->res_interrupt,
1311                                   adapter->int_handler_tag);
1312                 bus_release_resource(dev, SYS_RES_IRQ, 0,
1313                                      adapter->res_interrupt);
1314         }
1315         if (adapter->res_memory != NULL) {
1316                 bus_release_resource(dev, SYS_RES_MEMORY, IXGB_MMBA,
1317                                      adapter->res_memory);
1318         }
1319         if (adapter->res_ioport != NULL) {
1320                 bus_release_resource(dev, SYS_RES_IOPORT, adapter->io_rid,
1321                                      adapter->res_ioport);
1322         }
1323         return;
1324 }
1325
1326 /*********************************************************************
1327  *
1328  *  Initialize the hardware to a configuration as specified by the
1329  *  adapter structure. The controller is reset, the EEPROM is
1330  *  verified, the MAC address is set, then the shared initialization
1331  *  routines are called.
1332  *
1333  **********************************************************************/
1334 static int
1335 ixgb_hardware_init(struct adapter * adapter)
1336 {
1337         /* Issue a global reset */
1338         adapter->hw.adapter_stopped = FALSE;
1339         ixgb_adapter_stop(&adapter->hw);
1340
1341         /* Make sure we have a good EEPROM before we read from it */
1342         if (!ixgb_validate_eeprom_checksum(&adapter->hw)) {
1343                 printf("ixgb%d: The EEPROM Checksum Is Not Valid\n",
1344                        adapter->unit);
1345                 return (EIO);
1346         }
1347         if (!ixgb_init_hw(&adapter->hw)) {
1348                 printf("ixgb%d: Hardware Initialization Failed",
1349                        adapter->unit);
1350                 return (EIO);
1351         }
1352
1353         return (0);
1354 }
1355
1356 /*********************************************************************
1357  *
1358  *  Setup networking device structure and register an interface.
1359  *
1360  **********************************************************************/
1361 static int
1362 ixgb_setup_interface(device_t dev, struct adapter * adapter)
1363 {
1364         struct ifnet   *ifp;
1365         INIT_DEBUGOUT("ixgb_setup_interface: begin");
1366
1367         ifp = adapter->ifp = if_alloc(IFT_ETHER);
1368         if (ifp == NULL) {
1369                 device_printf(dev, "can not allocate ifnet structure\n");
1370                 return (-1);
1371         }
1372 #if __FreeBSD_version >= 502000
1373         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1374 #else
1375         ifp->if_unit = adapter->unit;
1376         ifp->if_name = "ixgb";
1377 #endif
1378         ifp->if_mtu = ETHERMTU;
1379         ifp->if_baudrate = 1000000000;
1380         ifp->if_init = ixgb_init;
1381         ifp->if_softc = adapter;
1382         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1383         ifp->if_ioctl = ixgb_ioctl;
1384         ifp->if_start = ixgb_start;
1385         ifp->if_watchdog = ixgb_watchdog;
1386         ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 1;
1387
1388 #if __FreeBSD_version < 500000
1389         ether_ifattach(ifp, ETHER_BPF_SUPPORTED);
1390 #else
1391         ether_ifattach(ifp, adapter->hw.curr_mac_addr);
1392 #endif
1393
1394         ifp->if_capabilities = IFCAP_HWCSUM;
1395
1396         /*
1397          * Tell the upper layer(s) we support long frames.
1398          */
1399         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
1400
1401 #if __FreeBSD_version >= 500000
1402         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
1403 #endif
1404
1405         ifp->if_capenable = ifp->if_capabilities;
1406
1407 #ifdef DEVICE_POLLING
1408         ifp->if_capabilities |= IFCAP_POLLING;
1409 #endif
1410
1411         /*
1412          * Specify the media types supported by this adapter and register
1413          * callbacks to update media and link information
1414          */
1415         ifmedia_init(&adapter->media, IFM_IMASK, ixgb_media_change,
1416                      ixgb_media_status);
1417         ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX | IFM_FDX,
1418                     0, NULL);
1419         ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX,
1420                     0, NULL);
1421         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
1422         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
1423
1424         return (0);
1425 }
1426
1427 /********************************************************************
1428  * Manage DMA'able memory.
1429  *******************************************************************/
1430 static void
1431 ixgb_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
1432 {
1433         if (error)
1434                 return;
1435         *(bus_addr_t *) arg = segs->ds_addr;
1436         return;
1437 }
1438
1439 static int
1440 ixgb_dma_malloc(struct adapter * adapter, bus_size_t size,
1441                 struct ixgb_dma_alloc * dma, int mapflags)
1442 {
1443         int             r;
1444
1445         r = bus_dma_tag_create(NULL,    /* parent */
1446                                PAGE_SIZE, 0,    /* alignment, bounds */
1447                                BUS_SPACE_MAXADDR,       /* lowaddr */
1448                                BUS_SPACE_MAXADDR,       /* highaddr */
1449                                NULL, NULL,      /* filter, filterarg */
1450                                size,    /* maxsize */
1451                                1,       /* nsegments */
1452                                size,    /* maxsegsize */
1453                                BUS_DMA_ALLOCNOW,        /* flags */
1454 #if __FreeBSD_version >= 502000
1455                                NULL,    /* lockfunc */
1456                                NULL,    /* lockfuncarg */
1457 #endif
1458                                &dma->dma_tag);
1459         if (r != 0) {
1460                 printf("ixgb%d: ixgb_dma_malloc: bus_dma_tag_create failed; "
1461                        "error %u\n", adapter->unit, r);
1462                 goto fail_0;
1463         }
1464         r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
1465                              BUS_DMA_NOWAIT, &dma->dma_map);
1466         if (r != 0) {
1467                 printf("ixgb%d: ixgb_dma_malloc: bus_dmamem_alloc failed; "
1468                        "error %u\n", adapter->unit, r);
1469                 goto fail_1;
1470         }
1471         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
1472                             size,
1473                             ixgb_dmamap_cb,
1474                             &dma->dma_paddr,
1475                             mapflags | BUS_DMA_NOWAIT);
1476         if (r != 0) {
1477                 printf("ixgb%d: ixgb_dma_malloc: bus_dmamap_load failed; "
1478                        "error %u\n", adapter->unit, r);
1479                 goto fail_2;
1480         }
1481         dma->dma_size = size;
1482         return (0);
1483 fail_2:
1484         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1485 fail_1:
1486         bus_dma_tag_destroy(dma->dma_tag);
1487 fail_0:
1488         dma->dma_map = NULL;
1489         dma->dma_tag = NULL;
1490         return (r);
1491 }
1492
1493
1494
1495 static void
1496 ixgb_dma_free(struct adapter * adapter, struct ixgb_dma_alloc * dma)
1497 {
1498         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
1499         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1500         bus_dma_tag_destroy(dma->dma_tag);
1501 }
1502
1503 /*********************************************************************
1504  *
1505  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
1506  *  the information needed to transmit a packet on the wire.
1507  *
1508  **********************************************************************/
1509 static int
1510 ixgb_allocate_transmit_structures(struct adapter * adapter)
1511 {
1512         if (!(adapter->tx_buffer_area =
1513               (struct ixgb_buffer *) malloc(sizeof(struct ixgb_buffer) *
1514                                             adapter->num_tx_desc, M_DEVBUF,
1515                                             M_NOWAIT | M_ZERO))) {
1516                 printf("ixgb%d: Unable to allocate tx_buffer memory\n",
1517                        adapter->unit);
1518                 return ENOMEM;
1519         }
1520         bzero(adapter->tx_buffer_area,
1521               sizeof(struct ixgb_buffer) * adapter->num_tx_desc);
1522
1523         return 0;
1524 }
1525
1526 /*********************************************************************
1527  *
1528  *  Allocate and initialize transmit structures.
1529  *
1530  **********************************************************************/
1531 static int
1532 ixgb_setup_transmit_structures(struct adapter * adapter)
1533 {
1534         /*
1535          * Setup DMA descriptor areas.
1536          */
1537         if (bus_dma_tag_create(NULL,    /* parent */
1538                                PAGE_SIZE, 0,    /* alignment, bounds */
1539                                BUS_SPACE_MAXADDR,       /* lowaddr */
1540                                BUS_SPACE_MAXADDR,       /* highaddr */
1541                                NULL, NULL,      /* filter, filterarg */
1542                                MCLBYTES * IXGB_MAX_SCATTER,     /* maxsize */
1543                                IXGB_MAX_SCATTER,        /* nsegments */
1544                                MCLBYTES,        /* maxsegsize */
1545                                BUS_DMA_ALLOCNOW,        /* flags */
1546 #if __FreeBSD_version >= 502000
1547                                NULL,    /* lockfunc */
1548                                NULL,    /* lockfuncarg */
1549 #endif
1550                                &adapter->txtag)) {
1551                 printf("ixgb%d: Unable to allocate TX DMA tag\n", adapter->unit);
1552                 return (ENOMEM);
1553         }
1554         if (ixgb_allocate_transmit_structures(adapter))
1555                 return ENOMEM;
1556
1557         bzero((void *)adapter->tx_desc_base,
1558               (sizeof(struct ixgb_tx_desc)) * adapter->num_tx_desc);
1559
1560         adapter->next_avail_tx_desc = 0;
1561         adapter->oldest_used_tx_desc = 0;
1562
1563         /* Set number of descriptors available */
1564         adapter->num_tx_desc_avail = adapter->num_tx_desc;
1565
1566         /* Set checksum context */
1567         adapter->active_checksum_context = OFFLOAD_NONE;
1568
1569         return 0;
1570 }
1571
1572 /*********************************************************************
1573  *
1574  *  Enable transmit unit.
1575  *
1576  **********************************************************************/
1577 static void
1578 ixgb_initialize_transmit_unit(struct adapter * adapter)
1579 {
1580         u_int32_t       reg_tctl;
1581         u_int64_t       tdba = adapter->txdma.dma_paddr;
1582
1583         /* Setup the Base and Length of the Tx Descriptor Ring */
1584         IXGB_WRITE_REG(&adapter->hw, TDBAL,
1585                        (tdba & 0x00000000ffffffffULL));
1586         IXGB_WRITE_REG(&adapter->hw, TDBAH, (tdba >> 32));
1587         IXGB_WRITE_REG(&adapter->hw, TDLEN,
1588                        adapter->num_tx_desc *
1589                        sizeof(struct ixgb_tx_desc));
1590
1591         /* Setup the HW Tx Head and Tail descriptor pointers */
1592         IXGB_WRITE_REG(&adapter->hw, TDH, 0);
1593         IXGB_WRITE_REG(&adapter->hw, TDT, 0);
1594
1595
1596         HW_DEBUGOUT2("Base = %x, Length = %x\n",
1597                      IXGB_READ_REG(&adapter->hw, TDBAL),
1598                      IXGB_READ_REG(&adapter->hw, TDLEN));
1599
1600         IXGB_WRITE_REG(&adapter->hw, TIDV, adapter->tx_int_delay);
1601
1602
1603         /* Program the Transmit Control Register */
1604         reg_tctl = IXGB_READ_REG(&adapter->hw, TCTL);
1605         reg_tctl = IXGB_TCTL_TCE | IXGB_TCTL_TXEN | IXGB_TCTL_TPDE;
1606         IXGB_WRITE_REG(&adapter->hw, TCTL, reg_tctl);
1607
1608         /* Setup Transmit Descriptor Settings for this adapter */
1609         adapter->txd_cmd = IXGB_TX_DESC_TYPE | IXGB_TX_DESC_CMD_RS;
1610
1611         if (adapter->tx_int_delay > 0)
1612                 adapter->txd_cmd |= IXGB_TX_DESC_CMD_IDE;
1613         return;
1614 }
1615
1616 /*********************************************************************
1617  *
1618  *  Free all transmit related data structures.
1619  *
1620  **********************************************************************/
1621 static void
1622 ixgb_free_transmit_structures(struct adapter * adapter)
1623 {
1624         struct ixgb_buffer *tx_buffer;
1625         int             i;
1626
1627         INIT_DEBUGOUT("free_transmit_structures: begin");
1628
1629         if (adapter->tx_buffer_area != NULL) {
1630                 tx_buffer = adapter->tx_buffer_area;
1631                 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
1632                         if (tx_buffer->m_head != NULL) {
1633                                 bus_dmamap_unload(adapter->txtag, tx_buffer->map);
1634                                 bus_dmamap_destroy(adapter->txtag, tx_buffer->map);
1635                                 m_freem(tx_buffer->m_head);
1636                         }
1637                         tx_buffer->m_head = NULL;
1638                 }
1639         }
1640         if (adapter->tx_buffer_area != NULL) {
1641                 free(adapter->tx_buffer_area, M_DEVBUF);
1642                 adapter->tx_buffer_area = NULL;
1643         }
1644         if (adapter->txtag != NULL) {
1645                 bus_dma_tag_destroy(adapter->txtag);
1646                 adapter->txtag = NULL;
1647         }
1648         return;
1649 }
1650
1651 /*********************************************************************
1652  *
1653  *  The offload context needs to be set when we transfer the first
1654  *  packet of a particular protocol (TCP/UDP). We change the
1655  *  context only if the protocol type changes.
1656  *
1657  **********************************************************************/
1658 static void
1659 ixgb_transmit_checksum_setup(struct adapter * adapter,
1660                              struct mbuf * mp,
1661                              u_int8_t * txd_popts)
1662 {
1663         struct ixgb_context_desc *TXD;
1664         struct ixgb_buffer *tx_buffer;
1665         int             curr_txd;
1666
1667         if (mp->m_pkthdr.csum_flags) {
1668
1669                 if (mp->m_pkthdr.csum_flags & CSUM_TCP) {
1670                         *txd_popts = IXGB_TX_DESC_POPTS_TXSM;
1671                         if (adapter->active_checksum_context == OFFLOAD_TCP_IP)
1672                                 return;
1673                         else
1674                                 adapter->active_checksum_context = OFFLOAD_TCP_IP;
1675                 } else if (mp->m_pkthdr.csum_flags & CSUM_UDP) {
1676                         *txd_popts = IXGB_TX_DESC_POPTS_TXSM;
1677                         if (adapter->active_checksum_context == OFFLOAD_UDP_IP)
1678                                 return;
1679                         else
1680                                 adapter->active_checksum_context = OFFLOAD_UDP_IP;
1681                 } else {
1682                         *txd_popts = 0;
1683                         return;
1684                 }
1685         } else {
1686                 *txd_popts = 0;
1687                 return;
1688         }
1689
1690         /*
1691          * If we reach this point, the checksum offload context needs to be
1692          * reset.
1693          */
1694         curr_txd = adapter->next_avail_tx_desc;
1695         tx_buffer = &adapter->tx_buffer_area[curr_txd];
1696         TXD = (struct ixgb_context_desc *) & adapter->tx_desc_base[curr_txd];
1697
1698
1699         TXD->tucss = ENET_HEADER_SIZE + sizeof(struct ip);
1700         TXD->tucse = 0;
1701
1702         TXD->mss = 0;
1703
1704         if (adapter->active_checksum_context == OFFLOAD_TCP_IP) {
1705                 TXD->tucso =
1706                         ENET_HEADER_SIZE + sizeof(struct ip) +
1707                         offsetof(struct tcphdr, th_sum);
1708         } else if (adapter->active_checksum_context == OFFLOAD_UDP_IP) {
1709                 TXD->tucso =
1710                         ENET_HEADER_SIZE + sizeof(struct ip) +
1711                         offsetof(struct udphdr, uh_sum);
1712         }
1713         TXD->cmd_type_len = IXGB_CONTEXT_DESC_CMD_TCP | IXGB_TX_DESC_CMD_RS | IXGB_CONTEXT_DESC_CMD_IDE;
1714
1715         tx_buffer->m_head = NULL;
1716
1717         if (++curr_txd == adapter->num_tx_desc)
1718                 curr_txd = 0;
1719
1720         adapter->num_tx_desc_avail--;
1721         adapter->next_avail_tx_desc = curr_txd;
1722         return;
1723 }
1724
1725 /**********************************************************************
1726  *
1727  *  Examine each tx_buffer in the used queue. If the hardware is done
1728  *  processing the packet then free associated resources. The
1729  *  tx_buffer is put back on the free queue.
1730  *
1731  **********************************************************************/
1732 static void
1733 ixgb_clean_transmit_interrupts(struct adapter * adapter)
1734 {
1735         int             i, num_avail;
1736         struct ixgb_buffer *tx_buffer;
1737         struct ixgb_tx_desc *tx_desc;
1738
1739         IXGB_LOCK_ASSERT(adapter);
1740
1741         if (adapter->num_tx_desc_avail == adapter->num_tx_desc)
1742                 return;
1743
1744 #ifdef _SV_
1745         adapter->clean_tx_interrupts++;
1746 #endif
1747         num_avail = adapter->num_tx_desc_avail;
1748         i = adapter->oldest_used_tx_desc;
1749
1750         tx_buffer = &adapter->tx_buffer_area[i];
1751         tx_desc = &adapter->tx_desc_base[i];
1752
1753         while (tx_desc->status & IXGB_TX_DESC_STATUS_DD) {
1754
1755                 tx_desc->status = 0;
1756                 num_avail++;
1757
1758                 if (tx_buffer->m_head) {
1759                         bus_dmamap_sync(adapter->txtag, tx_buffer->map,
1760                                         BUS_DMASYNC_POSTWRITE);
1761                         bus_dmamap_unload(adapter->txtag, tx_buffer->map);
1762                         bus_dmamap_destroy(adapter->txtag, tx_buffer->map);
1763                         m_freem(tx_buffer->m_head);
1764                         tx_buffer->m_head = NULL;
1765                 }
1766                 if (++i == adapter->num_tx_desc)
1767                         i = 0;
1768
1769                 tx_buffer = &adapter->tx_buffer_area[i];
1770                 tx_desc = &adapter->tx_desc_base[i];
1771         }
1772
1773         adapter->oldest_used_tx_desc = i;
1774
1775         /*
1776          * If we have enough room, clear IFF_DRV_OACTIVE to tell the stack that
1777          * it is OK to send packets. If there are no pending descriptors,
1778          * clear the timeout. Otherwise, if some descriptors have been freed,
1779          * restart the timeout.
1780          */
1781         if (num_avail > IXGB_TX_CLEANUP_THRESHOLD) {
1782                 struct ifnet   *ifp = adapter->ifp;
1783
1784                 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1785                 if (num_avail == adapter->num_tx_desc)
1786                         ifp->if_timer = 0;
1787                 else if (num_avail == adapter->num_tx_desc_avail)
1788                         ifp->if_timer = IXGB_TX_TIMEOUT;
1789         }
1790         adapter->num_tx_desc_avail = num_avail;
1791         return;
1792 }
1793
1794
1795 /*********************************************************************
1796  *
1797  *  Get a buffer from system mbuf buffer pool.
1798  *
1799  **********************************************************************/
1800 static int
1801 ixgb_get_buf(int i, struct adapter * adapter,
1802              struct mbuf * nmp)
1803 {
1804         register struct mbuf *mp = nmp;
1805         struct ixgb_buffer *rx_buffer;
1806         struct ifnet   *ifp;
1807         bus_addr_t      paddr;
1808         int             error;
1809
1810         ifp = adapter->ifp;
1811
1812         if (mp == NULL) {
1813
1814                 mp = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR);
1815
1816                 if (mp == NULL) {
1817                         adapter->mbuf_alloc_failed++;
1818                         return (ENOBUFS);
1819                 }
1820                 mp->m_len = mp->m_pkthdr.len = MCLBYTES;
1821         } else {
1822                 mp->m_len = mp->m_pkthdr.len = MCLBYTES;
1823                 mp->m_data = mp->m_ext.ext_buf;
1824                 mp->m_next = NULL;
1825         }
1826
1827         if (ifp->if_mtu <= ETHERMTU) {
1828                 m_adj(mp, ETHER_ALIGN);
1829         }
1830         rx_buffer = &adapter->rx_buffer_area[i];
1831
1832         /*
1833          * Using memory from the mbuf cluster pool, invoke the bus_dma
1834          * machinery to arrange the memory mapping.
1835          */
1836         error = bus_dmamap_load(adapter->rxtag, rx_buffer->map,
1837                                 mtod(mp, void *), mp->m_len,
1838                                 ixgb_dmamap_cb, &paddr, 0);
1839         if (error) {
1840                 m_free(mp);
1841                 return (error);
1842         }
1843         rx_buffer->m_head = mp;
1844         adapter->rx_desc_base[i].buff_addr = htole64(paddr);
1845         bus_dmamap_sync(adapter->rxtag, rx_buffer->map, BUS_DMASYNC_PREREAD);
1846
1847         return (0);
1848 }
1849
1850 /*********************************************************************
1851  *
1852  *  Allocate memory for rx_buffer structures. Since we use one
1853  *  rx_buffer per received packet, the maximum number of rx_buffer's
1854  *  that we'll need is equal to the number of receive descriptors
1855  *  that we've allocated.
1856  *
1857  **********************************************************************/
1858 static int
1859 ixgb_allocate_receive_structures(struct adapter * adapter)
1860 {
1861         int             i, error;
1862         struct ixgb_buffer *rx_buffer;
1863
1864         if (!(adapter->rx_buffer_area =
1865               (struct ixgb_buffer *) malloc(sizeof(struct ixgb_buffer) *
1866                                             adapter->num_rx_desc, M_DEVBUF,
1867                                             M_NOWAIT | M_ZERO))) {
1868                 printf("ixgb%d: Unable to allocate rx_buffer memory\n",
1869                        adapter->unit);
1870                 return (ENOMEM);
1871         }
1872         bzero(adapter->rx_buffer_area,
1873               sizeof(struct ixgb_buffer) * adapter->num_rx_desc);
1874
1875         error = bus_dma_tag_create(NULL,        /* parent */
1876                                    PAGE_SIZE, 0,        /* alignment, bounds */
1877                                    BUS_SPACE_MAXADDR,   /* lowaddr */
1878                                    BUS_SPACE_MAXADDR,   /* highaddr */
1879                                    NULL, NULL,  /* filter, filterarg */
1880                                    MCLBYTES,    /* maxsize */
1881                                    1,   /* nsegments */
1882                                    MCLBYTES,    /* maxsegsize */
1883                                    BUS_DMA_ALLOCNOW,    /* flags */
1884 #if __FreeBSD_version >= 502000
1885                                    NULL,        /* lockfunc */
1886                                    NULL,        /* lockfuncarg */
1887 #endif
1888                                    &adapter->rxtag);
1889         if (error != 0) {
1890                 printf("ixgb%d: ixgb_allocate_receive_structures: "
1891                        "bus_dma_tag_create failed; error %u\n",
1892                        adapter->unit, error);
1893                 goto fail_0;
1894         }
1895         rx_buffer = adapter->rx_buffer_area;
1896         for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
1897                 error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT,
1898                                           &rx_buffer->map);
1899                 if (error != 0) {
1900                         printf("ixgb%d: ixgb_allocate_receive_structures: "
1901                                "bus_dmamap_create failed; error %u\n",
1902                                adapter->unit, error);
1903                         goto fail_1;
1904                 }
1905         }
1906
1907         for (i = 0; i < adapter->num_rx_desc; i++) {
1908                 if (ixgb_get_buf(i, adapter, NULL) == ENOBUFS) {
1909                         adapter->rx_buffer_area[i].m_head = NULL;
1910                         adapter->rx_desc_base[i].buff_addr = 0;
1911                         return (ENOBUFS);
1912                 }
1913         }
1914
1915         return (0);
1916 fail_1:
1917         bus_dma_tag_destroy(adapter->rxtag);
1918 fail_0:
1919         adapter->rxtag = NULL;
1920         free(adapter->rx_buffer_area, M_DEVBUF);
1921         adapter->rx_buffer_area = NULL;
1922         return (error);
1923 }
1924
1925 /*********************************************************************
1926  *
1927  *  Allocate and initialize receive structures.
1928  *
1929  **********************************************************************/
1930 static int
1931 ixgb_setup_receive_structures(struct adapter * adapter)
1932 {
1933         bzero((void *)adapter->rx_desc_base,
1934               (sizeof(struct ixgb_rx_desc)) * adapter->num_rx_desc);
1935
1936         if (ixgb_allocate_receive_structures(adapter))
1937                 return ENOMEM;
1938
1939         /* Setup our descriptor pointers */
1940         adapter->next_rx_desc_to_check = 0;
1941         adapter->next_rx_desc_to_use = 0;
1942         return (0);
1943 }
1944
1945 /*********************************************************************
1946  *
1947  *  Enable receive unit.
1948  *
1949  **********************************************************************/
1950 static void
1951 ixgb_initialize_receive_unit(struct adapter * adapter)
1952 {
1953         u_int32_t       reg_rctl;
1954         u_int32_t       reg_rxcsum;
1955         u_int32_t       reg_rxdctl;
1956         struct ifnet   *ifp;
1957         u_int64_t       rdba = adapter->rxdma.dma_paddr;
1958
1959         ifp = adapter->ifp;
1960
1961         /*
1962          * Make sure receives are disabled while setting up the descriptor
1963          * ring
1964          */
1965         reg_rctl = IXGB_READ_REG(&adapter->hw, RCTL);
1966         IXGB_WRITE_REG(&adapter->hw, RCTL, reg_rctl & ~IXGB_RCTL_RXEN);
1967
1968         /* Set the Receive Delay Timer Register */
1969         IXGB_WRITE_REG(&adapter->hw, RDTR,
1970                        adapter->rx_int_delay);
1971
1972
1973         /* Setup the Base and Length of the Rx Descriptor Ring */
1974         IXGB_WRITE_REG(&adapter->hw, RDBAL,
1975                        (rdba & 0x00000000ffffffffULL));
1976         IXGB_WRITE_REG(&adapter->hw, RDBAH, (rdba >> 32));
1977         IXGB_WRITE_REG(&adapter->hw, RDLEN, adapter->num_rx_desc *
1978                        sizeof(struct ixgb_rx_desc));
1979
1980         /* Setup the HW Rx Head and Tail Descriptor Pointers */
1981         IXGB_WRITE_REG(&adapter->hw, RDH, 0);
1982
1983         IXGB_WRITE_REG(&adapter->hw, RDT, adapter->num_rx_desc - 1);
1984
1985
1986
1987         reg_rxdctl = RXDCTL_WTHRESH_DEFAULT << IXGB_RXDCTL_WTHRESH_SHIFT
1988                 | RXDCTL_HTHRESH_DEFAULT << IXGB_RXDCTL_HTHRESH_SHIFT
1989                 | RXDCTL_PTHRESH_DEFAULT << IXGB_RXDCTL_PTHRESH_SHIFT;
1990         IXGB_WRITE_REG(&adapter->hw, RXDCTL, reg_rxdctl);
1991
1992
1993         adapter->raidc = 1;
1994         if (adapter->raidc) {
1995                 uint32_t        raidc;
1996                 uint8_t         poll_threshold;
1997 #define IXGB_RAIDC_POLL_DEFAULT 120
1998
1999                 poll_threshold = ((adapter->num_rx_desc - 1) >> 3);
2000                 poll_threshold >>= 1;
2001                 poll_threshold &= 0x3F;
2002                 raidc = IXGB_RAIDC_EN | IXGB_RAIDC_RXT_GATE |
2003                         (IXGB_RAIDC_POLL_DEFAULT << IXGB_RAIDC_POLL_SHIFT) |
2004                         (adapter->rx_int_delay << IXGB_RAIDC_DELAY_SHIFT) |
2005                         poll_threshold;
2006                 IXGB_WRITE_REG(&adapter->hw, RAIDC, raidc);
2007         }
2008         /* Enable Receive Checksum Offload for TCP and UDP ? */
2009         if (ifp->if_capenable & IFCAP_RXCSUM) {
2010                 reg_rxcsum = IXGB_READ_REG(&adapter->hw, RXCSUM);
2011                 reg_rxcsum |= IXGB_RXCSUM_TUOFL;
2012                 IXGB_WRITE_REG(&adapter->hw, RXCSUM, reg_rxcsum);
2013         }
2014         /* Setup the Receive Control Register */
2015         reg_rctl = IXGB_READ_REG(&adapter->hw, RCTL);
2016         reg_rctl &= ~(3 << IXGB_RCTL_MO_SHIFT);
2017         reg_rctl |= IXGB_RCTL_BAM | IXGB_RCTL_RDMTS_1_2 | IXGB_RCTL_SECRC |
2018                 IXGB_RCTL_CFF |
2019                 (adapter->hw.mc_filter_type << IXGB_RCTL_MO_SHIFT);
2020
2021         switch (adapter->rx_buffer_len) {
2022         default:
2023         case IXGB_RXBUFFER_2048:
2024                 reg_rctl |= IXGB_RCTL_BSIZE_2048;
2025                 break;
2026         case IXGB_RXBUFFER_4096:
2027                 reg_rctl |= IXGB_RCTL_BSIZE_4096;
2028                 break;
2029         case IXGB_RXBUFFER_8192:
2030                 reg_rctl |= IXGB_RCTL_BSIZE_8192;
2031                 break;
2032         case IXGB_RXBUFFER_16384:
2033                 reg_rctl |= IXGB_RCTL_BSIZE_16384;
2034                 break;
2035         }
2036
2037         reg_rctl |= IXGB_RCTL_RXEN;
2038
2039
2040         /* Enable Receives */
2041         IXGB_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
2042
2043         return;
2044 }
2045
2046 /*********************************************************************
2047  *
2048  *  Free receive related data structures.
2049  *
2050  **********************************************************************/
2051 static void
2052 ixgb_free_receive_structures(struct adapter * adapter)
2053 {
2054         struct ixgb_buffer *rx_buffer;
2055         int             i;
2056
2057         INIT_DEBUGOUT("free_receive_structures: begin");
2058
2059         if (adapter->rx_buffer_area != NULL) {
2060                 rx_buffer = adapter->rx_buffer_area;
2061                 for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
2062                         if (rx_buffer->map != NULL) {
2063                                 bus_dmamap_unload(adapter->rxtag, rx_buffer->map);
2064                                 bus_dmamap_destroy(adapter->rxtag, rx_buffer->map);
2065                         }
2066                         if (rx_buffer->m_head != NULL)
2067                                 m_freem(rx_buffer->m_head);
2068                         rx_buffer->m_head = NULL;
2069                 }
2070         }
2071         if (adapter->rx_buffer_area != NULL) {
2072                 free(adapter->rx_buffer_area, M_DEVBUF);
2073                 adapter->rx_buffer_area = NULL;
2074         }
2075         if (adapter->rxtag != NULL) {
2076                 bus_dma_tag_destroy(adapter->rxtag);
2077                 adapter->rxtag = NULL;
2078         }
2079         return;
2080 }
2081
2082 /*********************************************************************
2083  *
2084  *  This routine executes in interrupt context. It replenishes
2085  *  the mbufs in the descriptor and sends data which has been
2086  *  dma'ed into host memory to upper layer.
2087  *
2088  *  We loop at most count times if count is > 0, or until done if
2089  *  count < 0.
2090  *
2091  *********************************************************************/
2092 static int
2093 ixgb_process_receive_interrupts(struct adapter * adapter, int count)
2094 {
2095         struct ifnet   *ifp;
2096         struct mbuf    *mp;
2097 #if __FreeBSD_version < 500000
2098         struct ether_header *eh;
2099 #endif
2100         int             eop = 0;
2101         int             len;
2102         u_int8_t        accept_frame = 0;
2103         int             i;
2104         int             next_to_use = 0;
2105         int             eop_desc;
2106         int             rx_npkts = 0;
2107         /* Pointer to the receive descriptor being examined. */
2108         struct ixgb_rx_desc *current_desc;
2109
2110         IXGB_LOCK_ASSERT(adapter);
2111
2112         ifp = adapter->ifp;
2113         i = adapter->next_rx_desc_to_check;
2114         next_to_use = adapter->next_rx_desc_to_use;
2115         eop_desc = adapter->next_rx_desc_to_check;
2116         current_desc = &adapter->rx_desc_base[i];
2117
2118         if (!((current_desc->status) & IXGB_RX_DESC_STATUS_DD)) {
2119 #ifdef _SV_
2120                 adapter->no_pkts_avail++;
2121 #endif
2122                 return (rx_npkts);
2123         }
2124         while ((current_desc->status & IXGB_RX_DESC_STATUS_DD) && (count != 0)) {
2125
2126                 mp = adapter->rx_buffer_area[i].m_head;
2127                 bus_dmamap_sync(adapter->rxtag, adapter->rx_buffer_area[i].map,
2128                                 BUS_DMASYNC_POSTREAD);
2129                 accept_frame = 1;
2130                 if (current_desc->status & IXGB_RX_DESC_STATUS_EOP) {
2131                         count--;
2132                         eop = 1;
2133                 } else {
2134                         eop = 0;
2135                 }
2136                 len = current_desc->length;
2137
2138                 if (current_desc->errors & (IXGB_RX_DESC_ERRORS_CE |
2139                             IXGB_RX_DESC_ERRORS_SE | IXGB_RX_DESC_ERRORS_P |
2140                                             IXGB_RX_DESC_ERRORS_RXE)) {
2141                         accept_frame = 0;
2142                 }
2143                 if (accept_frame) {
2144
2145                         /* Assign correct length to the current fragment */
2146                         mp->m_len = len;
2147
2148                         if (adapter->fmp == NULL) {
2149                                 mp->m_pkthdr.len = len;
2150                                 adapter->fmp = mp;      /* Store the first mbuf */
2151                                 adapter->lmp = mp;
2152                         } else {
2153                                 /* Chain mbuf's together */
2154                                 mp->m_flags &= ~M_PKTHDR;
2155                                 adapter->lmp->m_next = mp;
2156                                 adapter->lmp = adapter->lmp->m_next;
2157                                 adapter->fmp->m_pkthdr.len += len;
2158                         }
2159
2160                         if (eop) {
2161                                 eop_desc = i;
2162                                 adapter->fmp->m_pkthdr.rcvif = ifp;
2163
2164 #if __FreeBSD_version < 500000
2165                                 eh = mtod(adapter->fmp, struct ether_header *);
2166
2167                                 /* Remove ethernet header from mbuf */
2168                                 m_adj(adapter->fmp, sizeof(struct ether_header));
2169                                 ixgb_receive_checksum(adapter, current_desc,
2170                                                       adapter->fmp);
2171
2172                                 if (current_desc->status & IXGB_RX_DESC_STATUS_VP)
2173                                         VLAN_INPUT_TAG(eh, adapter->fmp,
2174                                                      current_desc->special);
2175                                 else
2176                                         ether_input(ifp, eh, adapter->fmp);
2177 #else
2178                                 ixgb_receive_checksum(adapter, current_desc,
2179                                                       adapter->fmp);
2180 #if __FreeBSD_version < 700000
2181                                 if (current_desc->status & IXGB_RX_DESC_STATUS_VP)
2182                                         VLAN_INPUT_TAG(ifp, adapter->fmp,
2183                                                        current_desc->special);
2184 #else
2185                                 if (current_desc->status & IXGB_RX_DESC_STATUS_VP) {
2186                                         adapter->fmp->m_pkthdr.ether_vtag =
2187                                             current_desc->special;
2188                                         adapter->fmp->m_flags |= M_VLANTAG;
2189                                 }
2190 #endif
2191
2192                                 if (adapter->fmp != NULL) {
2193                                         IXGB_UNLOCK(adapter);
2194                                         (*ifp->if_input) (ifp, adapter->fmp);
2195                                         IXGB_LOCK(adapter);
2196                                         rx_npkts++;
2197                                 }
2198 #endif
2199                                 adapter->fmp = NULL;
2200                                 adapter->lmp = NULL;
2201                         }
2202                         adapter->rx_buffer_area[i].m_head = NULL;
2203                 } else {
2204                         adapter->dropped_pkts++;
2205                         if (adapter->fmp != NULL)
2206                                 m_freem(adapter->fmp);
2207                         adapter->fmp = NULL;
2208                         adapter->lmp = NULL;
2209                 }
2210
2211                 /* Zero out the receive descriptors status  */
2212                 current_desc->status = 0;
2213
2214                 /* Advance our pointers to the next descriptor */
2215                 if (++i == adapter->num_rx_desc) {
2216                         i = 0;
2217                         current_desc = adapter->rx_desc_base;
2218                 } else
2219                         current_desc++;
2220         }
2221         adapter->next_rx_desc_to_check = i;
2222
2223         if (--i < 0)
2224                 i = (adapter->num_rx_desc - 1);
2225
2226         /*
2227          * 82597EX: Workaround for redundent write back in receive descriptor ring (causes
2228          * memory corruption). Avoid using and re-submitting the most recently received RX
2229          * descriptor back to hardware.
2230          *
2231          * if(Last written back descriptor == EOP bit set descriptor)
2232          *      then avoid re-submitting the most recently received RX descriptor
2233          *      back to hardware.
2234          * if(Last written back descriptor != EOP bit set descriptor)
2235          *      then avoid re-submitting the most recently received RX descriptors
2236          *      till last EOP bit set descriptor.
2237          */
2238         if (eop_desc != i) {
2239                 if (++eop_desc == adapter->num_rx_desc)
2240                         eop_desc = 0;
2241                 i = eop_desc;
2242         }
2243         /* Replenish the descriptors with new mbufs till last EOP bit set descriptor */
2244         while (next_to_use != i) {
2245                 current_desc = &adapter->rx_desc_base[next_to_use];
2246                 if ((current_desc->errors & (IXGB_RX_DESC_ERRORS_CE |
2247                             IXGB_RX_DESC_ERRORS_SE | IXGB_RX_DESC_ERRORS_P |
2248                                              IXGB_RX_DESC_ERRORS_RXE))) {
2249                         mp = adapter->rx_buffer_area[next_to_use].m_head;
2250                         ixgb_get_buf(next_to_use, adapter, mp);
2251                 } else {
2252                         if (ixgb_get_buf(next_to_use, adapter, NULL) == ENOBUFS)
2253                                 break;
2254                 }
2255                 /* Advance our pointers to the next descriptor */
2256                 if (++next_to_use == adapter->num_rx_desc) {
2257                         next_to_use = 0;
2258                         current_desc = adapter->rx_desc_base;
2259                 } else
2260                         current_desc++;
2261         }
2262         adapter->next_rx_desc_to_use = next_to_use;
2263         if (--next_to_use < 0)
2264                 next_to_use = (adapter->num_rx_desc - 1);
2265         /* Advance the IXGB's Receive Queue #0  "Tail Pointer" */
2266         IXGB_WRITE_REG(&adapter->hw, RDT, next_to_use);
2267
2268         return (rx_npkts);
2269 }
2270
2271 /*********************************************************************
2272  *
2273  *  Verify that the hardware indicated that the checksum is valid.
2274  *  Inform the stack about the status of checksum so that stack
2275  *  doesn't spend time verifying the checksum.
2276  *
2277  *********************************************************************/
2278 static void
2279 ixgb_receive_checksum(struct adapter * adapter,
2280                       struct ixgb_rx_desc * rx_desc,
2281                       struct mbuf * mp)
2282 {
2283         if (rx_desc->status & IXGB_RX_DESC_STATUS_IXSM) {
2284                 mp->m_pkthdr.csum_flags = 0;
2285                 return;
2286         }
2287         if (rx_desc->status & IXGB_RX_DESC_STATUS_IPCS) {
2288                 /* Did it pass? */
2289                 if (!(rx_desc->errors & IXGB_RX_DESC_ERRORS_IPE)) {
2290                         /* IP Checksum Good */
2291                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
2292                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
2293
2294                 } else {
2295                         mp->m_pkthdr.csum_flags = 0;
2296                 }
2297         }
2298         if (rx_desc->status & IXGB_RX_DESC_STATUS_TCPCS) {
2299                 /* Did it pass? */
2300                 if (!(rx_desc->errors & IXGB_RX_DESC_ERRORS_TCPE)) {
2301                         mp->m_pkthdr.csum_flags |=
2302                                 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
2303                         mp->m_pkthdr.csum_data = htons(0xffff);
2304                 }
2305         }
2306         return;
2307 }
2308
2309
2310 static void
2311 ixgb_enable_vlans(struct adapter * adapter)
2312 {
2313         uint32_t        ctrl;
2314
2315         ctrl = IXGB_READ_REG(&adapter->hw, CTRL0);
2316         ctrl |= IXGB_CTRL0_VME;
2317         IXGB_WRITE_REG(&adapter->hw, CTRL0, ctrl);
2318
2319         return;
2320 }
2321
2322
2323 static void
2324 ixgb_enable_intr(struct adapter * adapter)
2325 {
2326         IXGB_WRITE_REG(&adapter->hw, IMS, (IXGB_INT_RXT0 | IXGB_INT_TXDW |
2327                             IXGB_INT_RXDMT0 | IXGB_INT_LSC | IXGB_INT_RXO));
2328         return;
2329 }
2330
2331 static void
2332 ixgb_disable_intr(struct adapter * adapter)
2333 {
2334         IXGB_WRITE_REG(&adapter->hw, IMC, ~0);
2335         return;
2336 }
2337
2338 void
2339 ixgb_write_pci_cfg(struct ixgb_hw * hw,
2340                    uint32_t reg,
2341                    uint16_t * value)
2342 {
2343         pci_write_config(((struct ixgb_osdep *) hw->back)->dev, reg,
2344                          *value, 2);
2345 }
2346
2347 /**********************************************************************
2348  *
2349  *  Update the board statistics counters.
2350  *
2351  **********************************************************************/
2352 static void
2353 ixgb_update_stats_counters(struct adapter * adapter)
2354 {
2355         struct ifnet   *ifp;
2356
2357         adapter->stats.crcerrs += IXGB_READ_REG(&adapter->hw, CRCERRS);
2358         adapter->stats.gprcl += IXGB_READ_REG(&adapter->hw, GPRCL);
2359         adapter->stats.gprch += IXGB_READ_REG(&adapter->hw, GPRCH);
2360         adapter->stats.gorcl += IXGB_READ_REG(&adapter->hw, GORCL);
2361         adapter->stats.gorch += IXGB_READ_REG(&adapter->hw, GORCH);
2362         adapter->stats.bprcl += IXGB_READ_REG(&adapter->hw, BPRCL);
2363         adapter->stats.bprch += IXGB_READ_REG(&adapter->hw, BPRCH);
2364         adapter->stats.mprcl += IXGB_READ_REG(&adapter->hw, MPRCL);
2365         adapter->stats.mprch += IXGB_READ_REG(&adapter->hw, MPRCH);
2366         adapter->stats.roc += IXGB_READ_REG(&adapter->hw, ROC);
2367
2368         adapter->stats.mpc += IXGB_READ_REG(&adapter->hw, MPC);
2369         adapter->stats.dc += IXGB_READ_REG(&adapter->hw, DC);
2370         adapter->stats.rlec += IXGB_READ_REG(&adapter->hw, RLEC);
2371         adapter->stats.xonrxc += IXGB_READ_REG(&adapter->hw, XONRXC);
2372         adapter->stats.xontxc += IXGB_READ_REG(&adapter->hw, XONTXC);
2373         adapter->stats.xoffrxc += IXGB_READ_REG(&adapter->hw, XOFFRXC);
2374         adapter->stats.xofftxc += IXGB_READ_REG(&adapter->hw, XOFFTXC);
2375         adapter->stats.gptcl += IXGB_READ_REG(&adapter->hw, GPTCL);
2376         adapter->stats.gptch += IXGB_READ_REG(&adapter->hw, GPTCH);
2377         adapter->stats.gotcl += IXGB_READ_REG(&adapter->hw, GOTCL);
2378         adapter->stats.gotch += IXGB_READ_REG(&adapter->hw, GOTCH);
2379         adapter->stats.ruc += IXGB_READ_REG(&adapter->hw, RUC);
2380         adapter->stats.rfc += IXGB_READ_REG(&adapter->hw, RFC);
2381         adapter->stats.rjc += IXGB_READ_REG(&adapter->hw, RJC);
2382         adapter->stats.torl += IXGB_READ_REG(&adapter->hw, TORL);
2383         adapter->stats.torh += IXGB_READ_REG(&adapter->hw, TORH);
2384         adapter->stats.totl += IXGB_READ_REG(&adapter->hw, TOTL);
2385         adapter->stats.toth += IXGB_READ_REG(&adapter->hw, TOTH);
2386         adapter->stats.tprl += IXGB_READ_REG(&adapter->hw, TPRL);
2387         adapter->stats.tprh += IXGB_READ_REG(&adapter->hw, TPRH);
2388         adapter->stats.tptl += IXGB_READ_REG(&adapter->hw, TPTL);
2389         adapter->stats.tpth += IXGB_READ_REG(&adapter->hw, TPTH);
2390         adapter->stats.plt64c += IXGB_READ_REG(&adapter->hw, PLT64C);
2391         adapter->stats.mptcl += IXGB_READ_REG(&adapter->hw, MPTCL);
2392         adapter->stats.mptch += IXGB_READ_REG(&adapter->hw, MPTCH);
2393         adapter->stats.bptcl += IXGB_READ_REG(&adapter->hw, BPTCL);
2394         adapter->stats.bptch += IXGB_READ_REG(&adapter->hw, BPTCH);
2395
2396         adapter->stats.uprcl += IXGB_READ_REG(&adapter->hw, UPRCL);
2397         adapter->stats.uprch += IXGB_READ_REG(&adapter->hw, UPRCH);
2398         adapter->stats.vprcl += IXGB_READ_REG(&adapter->hw, VPRCL);
2399         adapter->stats.vprch += IXGB_READ_REG(&adapter->hw, VPRCH);
2400         adapter->stats.jprcl += IXGB_READ_REG(&adapter->hw, JPRCL);
2401         adapter->stats.jprch += IXGB_READ_REG(&adapter->hw, JPRCH);
2402         adapter->stats.rnbc += IXGB_READ_REG(&adapter->hw, RNBC);
2403         adapter->stats.icbc += IXGB_READ_REG(&adapter->hw, ICBC);
2404         adapter->stats.ecbc += IXGB_READ_REG(&adapter->hw, ECBC);
2405         adapter->stats.uptcl += IXGB_READ_REG(&adapter->hw, UPTCL);
2406         adapter->stats.uptch += IXGB_READ_REG(&adapter->hw, UPTCH);
2407         adapter->stats.vptcl += IXGB_READ_REG(&adapter->hw, VPTCL);
2408         adapter->stats.vptch += IXGB_READ_REG(&adapter->hw, VPTCH);
2409         adapter->stats.jptcl += IXGB_READ_REG(&adapter->hw, JPTCL);
2410         adapter->stats.jptch += IXGB_READ_REG(&adapter->hw, JPTCH);
2411         adapter->stats.tsctc += IXGB_READ_REG(&adapter->hw, TSCTC);
2412         adapter->stats.tsctfc += IXGB_READ_REG(&adapter->hw, TSCTFC);
2413         adapter->stats.ibic += IXGB_READ_REG(&adapter->hw, IBIC);
2414         adapter->stats.lfc += IXGB_READ_REG(&adapter->hw, LFC);
2415         adapter->stats.pfrc += IXGB_READ_REG(&adapter->hw, PFRC);
2416         adapter->stats.pftc += IXGB_READ_REG(&adapter->hw, PFTC);
2417         adapter->stats.mcfrc += IXGB_READ_REG(&adapter->hw, MCFRC);
2418
2419         ifp = adapter->ifp;
2420
2421         /* Fill out the OS statistics structure */
2422         ifp->if_ipackets = adapter->stats.gprcl;
2423         ifp->if_opackets = adapter->stats.gptcl;
2424         ifp->if_ibytes = adapter->stats.gorcl;
2425         ifp->if_obytes = adapter->stats.gotcl;
2426         ifp->if_imcasts = adapter->stats.mprcl;
2427         ifp->if_collisions = 0;
2428
2429         /* Rx Errors */
2430         ifp->if_ierrors =
2431                 adapter->dropped_pkts +
2432                 adapter->stats.crcerrs +
2433                 adapter->stats.rnbc +
2434                 adapter->stats.mpc +
2435                 adapter->stats.rlec;
2436
2437
2438 }
2439
2440
2441 /**********************************************************************
2442  *
2443  *  This routine is called only when ixgb_display_debug_stats is enabled.
2444  *  This routine provides a way to take a look at important statistics
2445  *  maintained by the driver and hardware.
2446  *
2447  **********************************************************************/
2448 static void
2449 ixgb_print_hw_stats(struct adapter * adapter)
2450 {
2451         char            buf_speed[100], buf_type[100];
2452         ixgb_bus_speed  bus_speed;
2453         ixgb_bus_type   bus_type;
2454         int             unit = adapter->unit;
2455
2456 #ifdef _SV_
2457         printf("ixgb%d: Packets not Avail = %ld\n", unit,
2458                adapter->no_pkts_avail);
2459         printf("ixgb%d: CleanTxInterrupts = %ld\n", unit,
2460                adapter->clean_tx_interrupts);
2461         printf("ixgb%d: ICR RXDMT0 = %lld\n", unit,
2462                (long long)adapter->sv_stats.icr_rxdmt0);
2463         printf("ixgb%d: ICR RXO = %lld\n", unit,
2464                (long long)adapter->sv_stats.icr_rxo);
2465         printf("ixgb%d: ICR RXT0 = %lld\n", unit,
2466                (long long)adapter->sv_stats.icr_rxt0);
2467         printf("ixgb%d: ICR TXDW = %lld\n", unit,
2468                (long long)adapter->sv_stats.icr_TXDW);
2469 #endif                          /* _SV_ */
2470
2471         bus_speed = adapter->hw.bus.speed;
2472         bus_type = adapter->hw.bus.type;
2473         sprintf(buf_speed,
2474                 bus_speed == ixgb_bus_speed_33 ? "33MHz" :
2475                 bus_speed == ixgb_bus_speed_66 ? "66MHz" :
2476                 bus_speed == ixgb_bus_speed_100 ? "100MHz" :
2477                 bus_speed == ixgb_bus_speed_133 ? "133MHz" :
2478                 "UNKNOWN");
2479         printf("ixgb%d: PCI_Bus_Speed = %s\n", unit,
2480                buf_speed);
2481
2482         sprintf(buf_type,
2483                 bus_type == ixgb_bus_type_pci ? "PCI" :
2484                 bus_type == ixgb_bus_type_pcix ? "PCI-X" :
2485                 "UNKNOWN");
2486         printf("ixgb%d: PCI_Bus_Type = %s\n", unit,
2487                buf_type);
2488
2489         printf("ixgb%d: Tx Descriptors not Avail1 = %ld\n", unit,
2490                adapter->no_tx_desc_avail1);
2491         printf("ixgb%d: Tx Descriptors not Avail2 = %ld\n", unit,
2492                adapter->no_tx_desc_avail2);
2493         printf("ixgb%d: Std Mbuf Failed = %ld\n", unit,
2494                adapter->mbuf_alloc_failed);
2495         printf("ixgb%d: Std Cluster Failed = %ld\n", unit,
2496                adapter->mbuf_cluster_failed);
2497
2498         printf("ixgb%d: Defer count = %lld\n", unit,
2499                (long long)adapter->stats.dc);
2500         printf("ixgb%d: Missed Packets = %lld\n", unit,
2501                (long long)adapter->stats.mpc);
2502         printf("ixgb%d: Receive No Buffers = %lld\n", unit,
2503                (long long)adapter->stats.rnbc);
2504         printf("ixgb%d: Receive length errors = %lld\n", unit,
2505                (long long)adapter->stats.rlec);
2506         printf("ixgb%d: Crc errors = %lld\n", unit,
2507                (long long)adapter->stats.crcerrs);
2508         printf("ixgb%d: Driver dropped packets = %ld\n", unit,
2509                adapter->dropped_pkts);
2510
2511         printf("ixgb%d: XON Rcvd = %lld\n", unit,
2512                (long long)adapter->stats.xonrxc);
2513         printf("ixgb%d: XON Xmtd = %lld\n", unit,
2514                (long long)adapter->stats.xontxc);
2515         printf("ixgb%d: XOFF Rcvd = %lld\n", unit,
2516                (long long)adapter->stats.xoffrxc);
2517         printf("ixgb%d: XOFF Xmtd = %lld\n", unit,
2518                (long long)adapter->stats.xofftxc);
2519
2520         printf("ixgb%d: Good Packets Rcvd = %lld\n", unit,
2521                (long long)adapter->stats.gprcl);
2522         printf("ixgb%d: Good Packets Xmtd = %lld\n", unit,
2523                (long long)adapter->stats.gptcl);
2524
2525         printf("ixgb%d: Jumbo frames recvd = %lld\n", unit,
2526                (long long)adapter->stats.jprcl);
2527         printf("ixgb%d: Jumbo frames Xmtd = %lld\n", unit,
2528                (long long)adapter->stats.jptcl);
2529
2530         return;
2531
2532 }
2533
2534 static int
2535 ixgb_sysctl_stats(SYSCTL_HANDLER_ARGS)
2536 {
2537         int             error;
2538         int             result;
2539         struct adapter *adapter;
2540
2541         result = -1;
2542         error = sysctl_handle_int(oidp, &result, 0, req);
2543
2544         if (error || !req->newptr)
2545                 return (error);
2546
2547         if (result == 1) {
2548                 adapter = (struct adapter *) arg1;
2549                 ixgb_print_hw_stats(adapter);
2550         }
2551         return error;
2552 }