sys/dev/nxge/if_nxge.c

   1 /*-
   2  * Copyright (c) 2002-2007 Neterion, Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  *
  26  * $FreeBSD$
  27  */
  28
  29 #include <dev/nxge/if_nxge.h>
  30 #include <dev/nxge/xge-osdep.h>
  31 #include <net/if_arp.h>
  32 #include <sys/types.h>
  33 #include <net/if.h>
  34 #include <net/if_var.h>
  35 #include <net/if_vlan_var.h>
  36
  37 int       copyright_print       = 0;
  38 int       hal_driver_init_count = 0;
  39 size_t    size                  = sizeof(int);
  40
  41 static void inline xge_flush_txds(xge_hal_channel_h);
  42
  43 /**
  44  * xge_probe
  45  * Probes for Xframe devices
  46  *
  47  * @dev Device handle
  48  *
  49  * Returns
  50  * BUS_PROBE_DEFAULT if device is supported
  51  * ENXIO if device is not supported
  52  */
  53 int
  54 xge_probe(device_t dev)
  55 {
  56         int  devid    = pci_get_device(dev);
  57         int  vendorid = pci_get_vendor(dev);
  58         int  retValue = ENXIO;
  59
  60         if(vendorid == XGE_PCI_VENDOR_ID) {
  61             if((devid == XGE_PCI_DEVICE_ID_XENA_2) ||
  62                 (devid == XGE_PCI_DEVICE_ID_HERC_2)) {
  63                 if(!copyright_print) {
  64                     xge_os_printf(XGE_COPYRIGHT);
  65                     copyright_print = 1;
  66                 }
  67                 device_set_desc_copy(dev,
  68                     "Neterion Xframe 10 Gigabit Ethernet Adapter");
  69                 retValue = BUS_PROBE_DEFAULT;
  70             }
  71         }
  72
  73         return retValue;
  74 }
  75
  76 /**
  77  * xge_init_params
  78  * Sets HAL parameter values (from kenv).
  79  *
  80  * @dconfig Device Configuration
  81  * @dev Device Handle
  82  */
  83 void
  84 xge_init_params(xge_hal_device_config_t *dconfig, device_t dev)
  85 {
  86         int qindex, tindex, revision;
  87         device_t checkdev;
  88         xge_lldev_t *lldev = (xge_lldev_t *)device_get_softc(dev);
  89
  90         dconfig->mtu                   = XGE_DEFAULT_INITIAL_MTU;
  91         dconfig->pci_freq_mherz        = XGE_DEFAULT_USER_HARDCODED;
  92         dconfig->device_poll_millis    = XGE_HAL_DEFAULT_DEVICE_POLL_MILLIS;
  93         dconfig->link_stability_period = XGE_HAL_DEFAULT_LINK_STABILITY_PERIOD;
  94         dconfig->mac.rmac_bcast_en     = XGE_DEFAULT_MAC_RMAC_BCAST_EN;
  95         dconfig->fifo.alignment_size   = XGE_DEFAULT_FIFO_ALIGNMENT_SIZE;
  96
  97         XGE_GET_PARAM("hw.xge.enable_tso", (*lldev), enabled_tso,
  98             XGE_DEFAULT_ENABLED_TSO);
  99         XGE_GET_PARAM("hw.xge.enable_lro", (*lldev), enabled_lro,
 100             XGE_DEFAULT_ENABLED_LRO);
 101         XGE_GET_PARAM("hw.xge.enable_msi", (*lldev), enabled_msi,
 102             XGE_DEFAULT_ENABLED_MSI);
 103
 104         XGE_GET_PARAM("hw.xge.latency_timer", (*dconfig), latency_timer,
 105             XGE_DEFAULT_LATENCY_TIMER);
 106         XGE_GET_PARAM("hw.xge.max_splits_trans", (*dconfig), max_splits_trans,
 107             XGE_DEFAULT_MAX_SPLITS_TRANS);
 108         XGE_GET_PARAM("hw.xge.mmrb_count", (*dconfig), mmrb_count,
 109             XGE_DEFAULT_MMRB_COUNT);
 110         XGE_GET_PARAM("hw.xge.shared_splits", (*dconfig), shared_splits,
 111             XGE_DEFAULT_SHARED_SPLITS);
 112         XGE_GET_PARAM("hw.xge.isr_polling_cnt", (*dconfig), isr_polling_cnt,
 113             XGE_DEFAULT_ISR_POLLING_CNT);
 114         XGE_GET_PARAM("hw.xge.stats_refresh_time_sec", (*dconfig),
 115             stats_refresh_time_sec, XGE_DEFAULT_STATS_REFRESH_TIME_SEC);
 116
 117         XGE_GET_PARAM_MAC("hw.xge.mac_tmac_util_period", tmac_util_period,
 118             XGE_DEFAULT_MAC_TMAC_UTIL_PERIOD);
 119         XGE_GET_PARAM_MAC("hw.xge.mac_rmac_util_period", rmac_util_period,
 120             XGE_DEFAULT_MAC_RMAC_UTIL_PERIOD);
 121         XGE_GET_PARAM_MAC("hw.xge.mac_rmac_pause_gen_en", rmac_pause_gen_en,
 122             XGE_DEFAULT_MAC_RMAC_PAUSE_GEN_EN);
 123         XGE_GET_PARAM_MAC("hw.xge.mac_rmac_pause_rcv_en", rmac_pause_rcv_en,
 124             XGE_DEFAULT_MAC_RMAC_PAUSE_RCV_EN);
 125         XGE_GET_PARAM_MAC("hw.xge.mac_rmac_pause_time", rmac_pause_time,
 126             XGE_DEFAULT_MAC_RMAC_PAUSE_TIME);
 127         XGE_GET_PARAM_MAC("hw.xge.mac_mc_pause_threshold_q0q3",
 128             mc_pause_threshold_q0q3, XGE_DEFAULT_MAC_MC_PAUSE_THRESHOLD_Q0Q3);
 129         XGE_GET_PARAM_MAC("hw.xge.mac_mc_pause_threshold_q4q7",
 130             mc_pause_threshold_q4q7, XGE_DEFAULT_MAC_MC_PAUSE_THRESHOLD_Q4Q7);
 131
 132         XGE_GET_PARAM_FIFO("hw.xge.fifo_memblock_size", memblock_size,
 133             XGE_DEFAULT_FIFO_MEMBLOCK_SIZE);
 134         XGE_GET_PARAM_FIFO("hw.xge.fifo_reserve_threshold", reserve_threshold,
 135             XGE_DEFAULT_FIFO_RESERVE_THRESHOLD);
 136         XGE_GET_PARAM_FIFO("hw.xge.fifo_max_frags", max_frags,
 137             XGE_DEFAULT_FIFO_MAX_FRAGS);
 138
 139         for(qindex = 0; qindex < XGE_FIFO_COUNT; qindex++) {
 140             XGE_GET_PARAM_FIFO_QUEUE("hw.xge.fifo_queue_intr", intr, qindex,
 141                 XGE_DEFAULT_FIFO_QUEUE_INTR);
 142             XGE_GET_PARAM_FIFO_QUEUE("hw.xge.fifo_queue_max", max, qindex,
 143                 XGE_DEFAULT_FIFO_QUEUE_MAX);
 144             XGE_GET_PARAM_FIFO_QUEUE("hw.xge.fifo_queue_initial", initial,
 145                 qindex, XGE_DEFAULT_FIFO_QUEUE_INITIAL);
 146
 147             for (tindex = 0; tindex < XGE_HAL_MAX_FIFO_TTI_NUM; tindex++) {
 148                 dconfig->fifo.queue[qindex].tti[tindex].enabled  = 1;
 149                 dconfig->fifo.queue[qindex].configured = 1;
 150
 151                 XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_urange_a",
 152                     urange_a, qindex, tindex,
 153                     XGE_DEFAULT_FIFO_QUEUE_TTI_URANGE_A);
 154                 XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_urange_b",
 155                     urange_b, qindex, tindex,
 156                     XGE_DEFAULT_FIFO_QUEUE_TTI_URANGE_B);
 157                 XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_urange_c",
 158                     urange_c, qindex, tindex,
 159                     XGE_DEFAULT_FIFO_QUEUE_TTI_URANGE_C);
 160                 XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_ufc_a",
 161                     ufc_a, qindex, tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_UFC_A);
 162                 XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_ufc_b",
 163                     ufc_b, qindex, tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_UFC_B);
 164                 XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_ufc_c",
 165                     ufc_c, qindex, tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_UFC_C);
 166                 XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_ufc_d",
 167                     ufc_d, qindex, tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_UFC_D);
 168                 XGE_GET_PARAM_FIFO_QUEUE_TTI(
 169                     "hw.xge.fifo_queue_tti_timer_ci_en", timer_ci_en, qindex,
 170                     tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_TIMER_CI_EN);
 171                 XGE_GET_PARAM_FIFO_QUEUE_TTI(
 172                     "hw.xge.fifo_queue_tti_timer_ac_en", timer_ac_en, qindex,
 173                     tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_TIMER_AC_EN);
 174                 XGE_GET_PARAM_FIFO_QUEUE_TTI(
 175                     "hw.xge.fifo_queue_tti_timer_val_us", timer_val_us, qindex,
 176                     tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_TIMER_VAL_US);
 177             }
 178         }
 179
 180         XGE_GET_PARAM_RING("hw.xge.ring_memblock_size", memblock_size,
 181             XGE_DEFAULT_RING_MEMBLOCK_SIZE);
 182
 183         XGE_GET_PARAM_RING("hw.xge.ring_strip_vlan_tag", strip_vlan_tag,
 184             XGE_DEFAULT_RING_STRIP_VLAN_TAG);
 185
 186         XGE_GET_PARAM("hw.xge.buffer_mode", (*lldev), buffer_mode,
 187             XGE_DEFAULT_BUFFER_MODE);
 188         if((lldev->buffer_mode < XGE_HAL_RING_QUEUE_BUFFER_MODE_1) ||
 189             (lldev->buffer_mode > XGE_HAL_RING_QUEUE_BUFFER_MODE_2)) {
 190             xge_trace(XGE_ERR, "Supported buffer modes are 1 and 2");
 191             lldev->buffer_mode = XGE_HAL_RING_QUEUE_BUFFER_MODE_1;
 192         }
 193
 194         for (qindex = 0; qindex < XGE_RING_COUNT; qindex++) {
 195             dconfig->ring.queue[qindex].max_frm_len  = XGE_HAL_RING_USE_MTU;
 196             dconfig->ring.queue[qindex].priority     = 0;
 197             dconfig->ring.queue[qindex].configured   = 1;
 198             dconfig->ring.queue[qindex].buffer_mode  =
 199                 (lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_2) ?
 200                 XGE_HAL_RING_QUEUE_BUFFER_MODE_3 : lldev->buffer_mode;
 201
 202             XGE_GET_PARAM_RING_QUEUE("hw.xge.ring_queue_max", max, qindex,
 203                 XGE_DEFAULT_RING_QUEUE_MAX);
 204             XGE_GET_PARAM_RING_QUEUE("hw.xge.ring_queue_initial", initial,
 205                 qindex, XGE_DEFAULT_RING_QUEUE_INITIAL);
 206             XGE_GET_PARAM_RING_QUEUE("hw.xge.ring_queue_dram_size_mb",
 207                 dram_size_mb, qindex, XGE_DEFAULT_RING_QUEUE_DRAM_SIZE_MB);
 208             XGE_GET_PARAM_RING_QUEUE("hw.xge.ring_queue_indicate_max_pkts",
 209                 indicate_max_pkts, qindex,
 210                 XGE_DEFAULT_RING_QUEUE_INDICATE_MAX_PKTS);
 211             XGE_GET_PARAM_RING_QUEUE("hw.xge.ring_queue_backoff_interval_us",
 212                 backoff_interval_us, qindex,
 213                 XGE_DEFAULT_RING_QUEUE_BACKOFF_INTERVAL_US);
 214
 215             XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_ufc_a", ufc_a,
 216                 qindex, XGE_DEFAULT_RING_QUEUE_RTI_UFC_A);
 217             XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_ufc_b", ufc_b,
 218                 qindex, XGE_DEFAULT_RING_QUEUE_RTI_UFC_B);
 219             XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_ufc_c", ufc_c,
 220                 qindex, XGE_DEFAULT_RING_QUEUE_RTI_UFC_C);
 221             XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_ufc_d", ufc_d,
 222                 qindex, XGE_DEFAULT_RING_QUEUE_RTI_UFC_D);
 223             XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_timer_ac_en",
 224                 timer_ac_en, qindex, XGE_DEFAULT_RING_QUEUE_RTI_TIMER_AC_EN);
 225             XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_timer_val_us",
 226                 timer_val_us, qindex, XGE_DEFAULT_RING_QUEUE_RTI_TIMER_VAL_US);
 227             XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_urange_a",
 228                 urange_a, qindex, XGE_DEFAULT_RING_QUEUE_RTI_URANGE_A);
 229             XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_urange_b",
 230                 urange_b, qindex, XGE_DEFAULT_RING_QUEUE_RTI_URANGE_B);
 231             XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_urange_c",
 232                 urange_c, qindex, XGE_DEFAULT_RING_QUEUE_RTI_URANGE_C);
 233         }
 234
 235         if(dconfig->fifo.max_frags > (PAGE_SIZE/32)) {
 236             xge_os_printf("fifo_max_frags = %d", dconfig->fifo.max_frags)
 237             xge_os_printf("fifo_max_frags should be <= (PAGE_SIZE / 32) = %d",
 238                 (int)(PAGE_SIZE / 32))
 239             xge_os_printf("Using fifo_max_frags = %d", (int)(PAGE_SIZE / 32))
 240             dconfig->fifo.max_frags = (PAGE_SIZE / 32);
 241         }
 242
 243         checkdev = pci_find_device(VENDOR_ID_AMD, DEVICE_ID_8131_PCI_BRIDGE);
 244         if(checkdev != NULL) {
 245             /* Check Revision for 0x12 */
 246             revision = pci_read_config(checkdev,
 247                 xge_offsetof(xge_hal_pci_config_t, revision), 1);
 248             if(revision <= 0x12) {
 249                 /* Set mmrb_count to 1k and max splits = 2 */
 250                 dconfig->mmrb_count       = 1;
 251                 dconfig->max_splits_trans = XGE_HAL_THREE_SPLIT_TRANSACTION;
 252             }
 253         }
 254 }
 255
 256 /**
 257  * xge_buffer_sizes_set
 258  * Set buffer sizes based on Rx buffer mode
 259  *
 260  * @lldev Per-adapter Data
 261  * @buffer_mode Rx Buffer Mode
 262  */
 263 void
 264 xge_rx_buffer_sizes_set(xge_lldev_t *lldev, int buffer_mode, int mtu)
 265 {
 266         int index = 0;
 267         int frame_header = XGE_HAL_MAC_HEADER_MAX_SIZE;
 268         int buffer_size = mtu + frame_header;
 269
 270         xge_os_memzero(lldev->rxd_mbuf_len, sizeof(lldev->rxd_mbuf_len));
 271
 272         if(buffer_mode != XGE_HAL_RING_QUEUE_BUFFER_MODE_5)
 273             lldev->rxd_mbuf_len[buffer_mode - 1] = mtu;
 274
 275         lldev->rxd_mbuf_len[0] = (buffer_mode == 1) ? buffer_size:frame_header;
 276
 277         if(buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_5)
 278             lldev->rxd_mbuf_len[1] = XGE_HAL_TCPIP_HEADER_MAX_SIZE;
 279
 280         if(buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_5) {
 281             index = 2;
 282             buffer_size -= XGE_HAL_TCPIP_HEADER_MAX_SIZE;
 283             while(buffer_size > MJUMPAGESIZE) {
 284                 lldev->rxd_mbuf_len[index++] = MJUMPAGESIZE;
 285                 buffer_size -= MJUMPAGESIZE;
 286             }
 287             XGE_ALIGN_TO(buffer_size, 128);
 288             lldev->rxd_mbuf_len[index] = buffer_size;
 289             lldev->rxd_mbuf_cnt = index + 1;
 290         }
 291
 292         for(index = 0; index < buffer_mode; index++)
 293             xge_trace(XGE_TRACE, "Buffer[%d] %d\n", index,
 294                 lldev->rxd_mbuf_len[index]);
 295 }
 296
 297 /**
 298  * xge_buffer_mode_init
 299  * Init Rx buffer mode
 300  *
 301  * @lldev Per-adapter Data
 302  * @mtu Interface MTU
 303  */
 304 void
 305 xge_buffer_mode_init(xge_lldev_t *lldev, int mtu)
 306 {
 307         int index = 0, buffer_size = 0;
 308         xge_hal_ring_config_t *ring_config = &((lldev->devh)->config.ring);
 309
 310         buffer_size = mtu + XGE_HAL_MAC_HEADER_MAX_SIZE;
 311
 312         if(lldev->enabled_lro)
 313             (lldev->ifnetp)->if_capenable |= IFCAP_LRO;
 314         else
 315             (lldev->ifnetp)->if_capenable &= ~IFCAP_LRO;
 316
 317         lldev->rxd_mbuf_cnt = lldev->buffer_mode;
 318         if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_2) {
 319             XGE_SET_BUFFER_MODE_IN_RINGS(XGE_HAL_RING_QUEUE_BUFFER_MODE_3);
 320             ring_config->scatter_mode = XGE_HAL_RING_QUEUE_SCATTER_MODE_B;
 321         }
 322         else {
 323             XGE_SET_BUFFER_MODE_IN_RINGS(lldev->buffer_mode);
 324             ring_config->scatter_mode = XGE_HAL_RING_QUEUE_SCATTER_MODE_A;
 325         }
 326         xge_rx_buffer_sizes_set(lldev, lldev->buffer_mode, mtu);
 327
 328         xge_os_printf("%s: TSO %s", device_get_nameunit(lldev->device),
 329             ((lldev->enabled_tso) ? "Enabled":"Disabled"));
 330         xge_os_printf("%s: LRO %s", device_get_nameunit(lldev->device),
 331             ((lldev->ifnetp)->if_capenable & IFCAP_LRO) ? "Enabled":"Disabled");
 332         xge_os_printf("%s: Rx %d Buffer Mode Enabled",
 333             device_get_nameunit(lldev->device), lldev->buffer_mode);
 334 }
 335
 336 /**
 337  * xge_driver_initialize
 338  * Initializes HAL driver (common for all devices)
 339  *
 340  * Returns
 341  * XGE_HAL_OK if success
 342  * XGE_HAL_ERR_BAD_DRIVER_CONFIG if driver configuration parameters are invalid
 343  */
 344 int
 345 xge_driver_initialize(void)
 346 {
 347         xge_hal_uld_cbs_t       uld_callbacks;
 348         xge_hal_driver_config_t driver_config;
 349         xge_hal_status_e        status = XGE_HAL_OK;
 350
 351         /* Initialize HAL driver */
 352         if(!hal_driver_init_count) {
 353             xge_os_memzero(&uld_callbacks, sizeof(xge_hal_uld_cbs_t));
 354             xge_os_memzero(&driver_config, sizeof(xge_hal_driver_config_t));
 355
 356             /*
 357              * Initial and maximum size of the queue used to store the events
 358              * like Link up/down (xge_hal_event_e)
 359              */
 360             driver_config.queue_size_initial = XGE_HAL_MIN_QUEUE_SIZE_INITIAL;
 361             driver_config.queue_size_max     = XGE_HAL_MAX_QUEUE_SIZE_MAX;
 362
 363             uld_callbacks.link_up   = xge_callback_link_up;
 364             uld_callbacks.link_down = xge_callback_link_down;
 365             uld_callbacks.crit_err  = xge_callback_crit_err;
 366             uld_callbacks.event     = xge_callback_event;
 367
 368             status = xge_hal_driver_initialize(&driver_config, &uld_callbacks);
 369             if(status != XGE_HAL_OK) {
 370                 XGE_EXIT_ON_ERR("xgeX: Initialization of HAL driver failed",
 371                     xdi_out, status);
 372             }
 373         }
 374         hal_driver_init_count = hal_driver_init_count + 1;
 375
 376         xge_hal_driver_debug_module_mask_set(0xffffffff);
 377         xge_hal_driver_debug_level_set(XGE_TRACE);
 378
 379 xdi_out:
 380         return status;
 381 }
 382
 383 /**
 384  * xge_media_init
 385  * Initializes, adds and sets media
 386  *
 387  * @devc Device Handle
 388  */
 389 void
 390 xge_media_init(device_t devc)
 391 {
 392         xge_lldev_t *lldev = (xge_lldev_t *)device_get_softc(devc);
 393
 394         /* Initialize Media */
 395         ifmedia_init(&lldev->media, IFM_IMASK, xge_ifmedia_change,
 396             xge_ifmedia_status);
 397
 398         /* Add supported media */
 399         ifmedia_add(&lldev->media, IFM_ETHER | IFM_1000_SX | IFM_FDX, 0, NULL);
 400         ifmedia_add(&lldev->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
 401         ifmedia_add(&lldev->media, IFM_ETHER | IFM_AUTO,    0, NULL);
 402         ifmedia_add(&lldev->media, IFM_ETHER | IFM_10G_SR,  0, NULL);
 403         ifmedia_add(&lldev->media, IFM_ETHER | IFM_10G_LR,  0, NULL);
 404
 405         /* Set media */
 406         ifmedia_set(&lldev->media, IFM_ETHER | IFM_AUTO);
 407 }
 408
 409 /**
 410  * xge_pci_space_save
 411  * Save PCI configuration space
 412  *
 413  * @dev Device Handle
 414  */
 415 void
 416 xge_pci_space_save(device_t dev)
 417 {
 418         struct pci_devinfo *dinfo = NULL;
 419
 420         dinfo = device_get_ivars(dev);
 421         xge_trace(XGE_TRACE, "Saving PCI configuration space");
 422         pci_cfg_save(dev, dinfo, 0);
 423 }
 424
 425 /**
 426  * xge_pci_space_restore
 427  * Restore saved PCI configuration space
 428  *
 429  * @dev Device Handle
 430  */
 431 void
 432 xge_pci_space_restore(device_t dev)
 433 {
 434         struct pci_devinfo *dinfo = NULL;
 435
 436         dinfo = device_get_ivars(dev);
 437         xge_trace(XGE_TRACE, "Restoring PCI configuration space");
 438         pci_cfg_restore(dev, dinfo);
 439 }
 440
 441 /**
 442  * xge_msi_info_save
 443  * Save MSI info
 444  *
 445  * @lldev Per-adapter Data
 446  */
 447 void
 448 xge_msi_info_save(xge_lldev_t * lldev)
 449 {
 450         xge_os_pci_read16(lldev->pdev, NULL,
 451             xge_offsetof(xge_hal_pci_config_le_t, msi_control),
 452             &lldev->msi_info.msi_control);
 453         xge_os_pci_read32(lldev->pdev, NULL,
 454             xge_offsetof(xge_hal_pci_config_le_t, msi_lower_address),
 455             &lldev->msi_info.msi_lower_address);
 456         xge_os_pci_read32(lldev->pdev, NULL,
 457             xge_offsetof(xge_hal_pci_config_le_t, msi_higher_address),
 458             &lldev->msi_info.msi_higher_address);
 459         xge_os_pci_read16(lldev->pdev, NULL,
 460             xge_offsetof(xge_hal_pci_config_le_t, msi_data),
 461             &lldev->msi_info.msi_data);
 462 }
 463
 464 /**
 465  * xge_msi_info_restore
 466  * Restore saved MSI info
 467  *
 468  * @dev Device Handle
 469  */
 470 void
 471 xge_msi_info_restore(xge_lldev_t *lldev)
 472 {
 473         /*
 474          * If interface is made down and up, traffic fails. It was observed that
 475          * MSI information were getting reset on down. Restoring them.
 476          */
 477         xge_os_pci_write16(lldev->pdev, NULL,
 478             xge_offsetof(xge_hal_pci_config_le_t, msi_control),
 479             lldev->msi_info.msi_control);
 480
 481         xge_os_pci_write32(lldev->pdev, NULL,
 482             xge_offsetof(xge_hal_pci_config_le_t, msi_lower_address),
 483             lldev->msi_info.msi_lower_address);
 484
 485         xge_os_pci_write32(lldev->pdev, NULL,
 486             xge_offsetof(xge_hal_pci_config_le_t, msi_higher_address),
 487             lldev->msi_info.msi_higher_address);
 488
 489         xge_os_pci_write16(lldev->pdev, NULL,
 490             xge_offsetof(xge_hal_pci_config_le_t, msi_data),
 491             lldev->msi_info.msi_data);
 492 }
 493
 494 /**
 495  * xge_init_mutex
 496  * Initializes mutexes used in driver
 497  *
 498  * @lldev  Per-adapter Data
 499  */
 500 void
 501 xge_mutex_init(xge_lldev_t *lldev)
 502 {
 503         int qindex;
 504
 505         sprintf(lldev->mtx_name_drv, "%s_drv",
 506             device_get_nameunit(lldev->device));
 507         mtx_init(&lldev->mtx_drv, lldev->mtx_name_drv, MTX_NETWORK_LOCK,
 508             MTX_DEF);
 509
 510         for(qindex = 0; qindex < XGE_FIFO_COUNT; qindex++) {
 511             sprintf(lldev->mtx_name_tx[qindex], "%s_tx_%d",
 512                 device_get_nameunit(lldev->device), qindex);
 513             mtx_init(&lldev->mtx_tx[qindex], lldev->mtx_name_tx[qindex], NULL,
 514                 MTX_DEF);
 515         }
 516 }
 517
 518 /**
 519  * xge_mutex_destroy
 520  * Destroys mutexes used in driver
 521  *
 522  * @lldev Per-adapter Data
 523  */
 524 void
 525 xge_mutex_destroy(xge_lldev_t *lldev)
 526 {
 527         int qindex;
 528
 529         for(qindex = 0; qindex < XGE_FIFO_COUNT; qindex++)
 530             mtx_destroy(&lldev->mtx_tx[qindex]);
 531         mtx_destroy(&lldev->mtx_drv);
 532 }
 533
 534 /**
 535  * xge_print_info
 536  * Print device and driver information
 537  *
 538  * @lldev Per-adapter Data
 539  */
 540 void
 541 xge_print_info(xge_lldev_t *lldev)
 542 {
 543         device_t dev = lldev->device;
 544         xge_hal_device_t *hldev = lldev->devh;
 545         xge_hal_status_e status = XGE_HAL_OK;
 546         u64 val64 = 0;
 547         const char *xge_pci_bus_speeds[17] = {
 548             "PCI 33MHz Bus",
 549             "PCI 66MHz Bus",
 550             "PCIX(M1) 66MHz Bus",
 551             "PCIX(M1) 100MHz Bus",
 552             "PCIX(M1) 133MHz Bus",
 553             "PCIX(M2) 133MHz Bus",
 554             "PCIX(M2) 200MHz Bus",
 555             "PCIX(M2) 266MHz Bus",
 556             "PCIX(M1) Reserved",
 557             "PCIX(M1) 66MHz Bus (Not Supported)",
 558             "PCIX(M1) 100MHz Bus (Not Supported)",
 559             "PCIX(M1) 133MHz Bus (Not Supported)",
 560             "PCIX(M2) Reserved",
 561             "PCIX 533 Reserved",
 562             "PCI Basic Mode",
 563             "PCIX Basic Mode",
 564             "PCI Invalid Mode"
 565         };
 566
 567         xge_os_printf("%s: Xframe%s %s Revision %d Driver v%s",
 568             device_get_nameunit(dev),
 569             ((hldev->device_id == XGE_PCI_DEVICE_ID_XENA_2) ? "I" : "II"),
 570             hldev->vpd_data.product_name, hldev->revision, XGE_DRIVER_VERSION);
 571         xge_os_printf("%s: Serial Number %s",
 572             device_get_nameunit(dev), hldev->vpd_data.serial_num);
 573
 574         if(pci_get_device(dev) == XGE_PCI_DEVICE_ID_HERC_2) {
 575             status = xge_hal_mgmt_reg_read(hldev, 0,
 576                 xge_offsetof(xge_hal_pci_bar0_t, pci_info), &val64);
 577             if(status != XGE_HAL_OK)
 578                 xge_trace(XGE_ERR, "Error for getting bus speed");
 579
 580             xge_os_printf("%s: Adapter is on %s bit %s",
 581                 device_get_nameunit(dev), ((val64 & BIT(8)) ? "32":"64"),
 582                 (xge_pci_bus_speeds[((val64 & XGE_HAL_PCI_INFO) >> 60)]));
 583         }
 584
 585         xge_os_printf("%s: Using %s Interrupts",
 586             device_get_nameunit(dev),
 587             (lldev->enabled_msi == XGE_HAL_INTR_MODE_MSI) ? "MSI":"Line");
 588 }
 589
 590 /**
 591  * xge_create_dma_tags
 592  * Creates DMA tags for both Tx and Rx
 593  *
 594  * @dev Device Handle
 595  *
 596  * Returns XGE_HAL_OK or XGE_HAL_FAIL (if errors)
 597  */
 598 xge_hal_status_e
 599 xge_create_dma_tags(device_t dev)
 600 {
 601         xge_lldev_t *lldev = (xge_lldev_t *)device_get_softc(dev);
 602         xge_hal_status_e status = XGE_HAL_FAIL;
 603         int mtu = (lldev->ifnetp)->if_mtu, maxsize;
 604
 605         /* DMA tag for Tx */
 606         status = bus_dma_tag_create(
 607             bus_get_dma_tag(dev),                /* Parent                    */
 608             PAGE_SIZE,                           /* Alignment                 */
 609             0,                                   /* Bounds                    */
 610             BUS_SPACE_MAXADDR,                   /* Low Address               */
 611             BUS_SPACE_MAXADDR,                   /* High Address              */
 612             NULL,                                /* Filter Function           */
 613             NULL,                                /* Filter Function Arguments */
 614             MCLBYTES * XGE_MAX_SEGS,             /* Maximum Size              */
 615             XGE_MAX_SEGS,                        /* Number of Segments        */
 616             MCLBYTES,                            /* Maximum Segment Size      */
 617             BUS_DMA_ALLOCNOW,                    /* Flags                     */
 618             NULL,                                /* Lock Function             */
 619             NULL,                                /* Lock Function Arguments   */
 620             (&lldev->dma_tag_tx));               /* DMA Tag                   */
 621         if(status != 0)
 622             goto _exit;
 623
 624         maxsize = mtu + XGE_HAL_MAC_HEADER_MAX_SIZE;
 625         if(maxsize <= MCLBYTES) {
 626             maxsize = MCLBYTES;
 627         }
 628         else {
 629             if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_5)
 630                 maxsize = MJUMPAGESIZE;
 631             else
 632                 maxsize = (maxsize <= MJUMPAGESIZE) ? MJUMPAGESIZE : MJUM9BYTES;
 633         }
 634
 635         /* DMA tag for Rx */
 636         status = bus_dma_tag_create(
 637             bus_get_dma_tag(dev),                /* Parent                    */
 638             PAGE_SIZE,                           /* Alignment                 */
 639             0,                                   /* Bounds                    */
 640             BUS_SPACE_MAXADDR,                   /* Low Address               */
 641             BUS_SPACE_MAXADDR,                   /* High Address              */
 642             NULL,                                /* Filter Function           */
 643             NULL,                                /* Filter Function Arguments */
 644             maxsize,                             /* Maximum Size              */
 645             1,                                   /* Number of Segments        */
 646             maxsize,                             /* Maximum Segment Size      */
 647             BUS_DMA_ALLOCNOW,                    /* Flags                     */
 648             NULL,                                /* Lock Function             */
 649             NULL,                                /* Lock Function Arguments   */
 650             (&lldev->dma_tag_rx));               /* DMA Tag                   */
 651         if(status != 0)
 652             goto _exit1;
 653
 654         status = bus_dmamap_create(lldev->dma_tag_rx, BUS_DMA_NOWAIT,
 655             &lldev->extra_dma_map);
 656         if(status != 0)
 657             goto _exit2;
 658
 659         status = XGE_HAL_OK;
 660         goto _exit;
 661
 662 _exit2:
 663         status = bus_dma_tag_destroy(lldev->dma_tag_rx);
 664         if(status != 0)
 665             xge_trace(XGE_ERR, "Rx DMA tag destroy failed");
 666 _exit1:
 667         status = bus_dma_tag_destroy(lldev->dma_tag_tx);
 668         if(status != 0)
 669             xge_trace(XGE_ERR, "Tx DMA tag destroy failed");
 670         status = XGE_HAL_FAIL;
 671 _exit:
 672         return status;
 673 }
 674
 675 /**
 676  * xge_confirm_changes
 677  * Disables and Enables interface to apply requested change
 678  *
 679  * @lldev Per-adapter Data
 680  * @mtu_set Is it called for changing MTU? (Yes: 1, No: 0)
 681  *
 682  * Returns 0 or Error Number
 683  */
 684 void
 685 xge_confirm_changes(xge_lldev_t *lldev, xge_option_e option)
 686 {
 687         if(lldev->initialized == 0) goto _exit1;
 688
 689         mtx_lock(&lldev->mtx_drv);
 690         if_down(lldev->ifnetp);
 691         xge_device_stop(lldev, XGE_HAL_CHANNEL_OC_NORMAL);
 692
 693         if(option == XGE_SET_MTU)
 694             (lldev->ifnetp)->if_mtu = lldev->mtu;
 695         else
 696             xge_buffer_mode_init(lldev, lldev->mtu);
 697
 698         xge_device_init(lldev, XGE_HAL_CHANNEL_OC_NORMAL);
 699         if_up(lldev->ifnetp);
 700         mtx_unlock(&lldev->mtx_drv);
 701         goto _exit;
 702
 703 _exit1:
 704         /* Request was to change MTU and device not initialized */
 705         if(option == XGE_SET_MTU) {
 706             (lldev->ifnetp)->if_mtu = lldev->mtu;
 707             xge_buffer_mode_init(lldev, lldev->mtu);
 708         }
 709 _exit:
 710         return;
 711 }
 712
 713 /**
 714  * xge_change_lro_status
 715  * Enable/Disable LRO feature
 716  *
 717  * @SYSCTL_HANDLER_ARGS sysctl_oid structure with arguments
 718  *
 719  * Returns 0 or error number.
 720  */
 721 static int
 722 xge_change_lro_status(SYSCTL_HANDLER_ARGS)
 723 {
 724         xge_lldev_t *lldev = (xge_lldev_t *)arg1;
 725         int request = lldev->enabled_lro, status = XGE_HAL_OK;
 726
 727         status = sysctl_handle_int(oidp, &request, arg2, req);
 728         if((status != XGE_HAL_OK) || (!req->newptr))
 729             goto _exit;
 730
 731         if((request < 0) || (request > 1)) {
 732             status = EINVAL;
 733             goto _exit;
 734         }
 735
 736         /* Return if current and requested states are same */
 737         if(request == lldev->enabled_lro){
 738             xge_trace(XGE_ERR, "LRO is already %s",
 739                 ((request) ? "enabled" : "disabled"));
 740             goto _exit;
 741         }
 742
 743         lldev->enabled_lro = request;
 744         xge_confirm_changes(lldev, XGE_CHANGE_LRO);
 745         arg2 = lldev->enabled_lro;
 746
 747 _exit:
 748         return status;
 749 }
 750
 751 /**
 752  * xge_add_sysctl_handlers
 753  * Registers sysctl parameter value update handlers
 754  *
 755  * @lldev Per-adapter data
 756  */
 757 void
 758 xge_add_sysctl_handlers(xge_lldev_t *lldev)
 759 {
 760         struct sysctl_ctx_list *context_list =
 761             device_get_sysctl_ctx(lldev->device);
 762         struct sysctl_oid *oid = device_get_sysctl_tree(lldev->device);
 763
 764         SYSCTL_ADD_PROC(context_list, SYSCTL_CHILDREN(oid), OID_AUTO,
 765             "enable_lro", CTLTYPE_INT | CTLFLAG_RW, lldev, 0,
 766             xge_change_lro_status, "I", "Enable or disable LRO feature");
 767 }
 768
 769 /**
 770  * xge_attach
 771  * Connects driver to the system if probe was success
 772  *
 773  * @dev Device Handle
 774  */
 775 int
 776 xge_attach(device_t dev)
 777 {
 778         xge_hal_device_config_t *device_config;
 779         xge_hal_device_attr_t   attr;
 780         xge_lldev_t             *lldev;
 781         xge_hal_device_t        *hldev;
 782         xge_pci_info_t          *pci_info;
 783         struct ifnet            *ifnetp;
 784         int                     rid, rid0, rid1, error;
 785         int                     msi_count = 0, status = XGE_HAL_OK;
 786         int                     enable_msi = XGE_HAL_INTR_MODE_IRQLINE;
 787
 788         device_config = xge_os_malloc(NULL, sizeof(xge_hal_device_config_t));
 789         if(!device_config) {
 790             XGE_EXIT_ON_ERR("Memory allocation for device configuration failed",
 791                 attach_out_config, ENOMEM);
 792         }
 793
 794         lldev = (xge_lldev_t *) device_get_softc(dev);
 795         if(!lldev) {
 796             XGE_EXIT_ON_ERR("Adapter softc is NULL", attach_out, ENOMEM);
 797         }
 798         lldev->device = dev;
 799
 800         xge_mutex_init(lldev);
 801
 802         error = xge_driver_initialize();
 803         if(error != XGE_HAL_OK) {
 804             xge_resources_free(dev, xge_free_mutex);
 805             XGE_EXIT_ON_ERR("Initializing driver failed", attach_out, ENXIO);
 806         }
 807
 808         /* HAL device */
 809         hldev =
 810             (xge_hal_device_t *)xge_os_malloc(NULL, sizeof(xge_hal_device_t));
 811         if(!hldev) {
 812             xge_resources_free(dev, xge_free_terminate_hal_driver);
 813             XGE_EXIT_ON_ERR("Memory allocation for HAL device failed",
 814                 attach_out, ENOMEM);
 815         }
 816         lldev->devh = hldev;
 817
 818         /* Our private structure */
 819         pci_info =
 820             (xge_pci_info_t*) xge_os_malloc(NULL, sizeof(xge_pci_info_t));
 821         if(!pci_info) {
 822             xge_resources_free(dev, xge_free_hal_device);
 823             XGE_EXIT_ON_ERR("Memory allocation for PCI info. failed",
 824                 attach_out, ENOMEM);
 825         }
 826         lldev->pdev      = pci_info;
 827         pci_info->device = dev;
 828
 829         /* Set bus master */
 830         pci_enable_busmaster(dev);
 831
 832         /* Get virtual address for BAR0 */
 833         rid0 = PCIR_BAR(0);
 834         pci_info->regmap0 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid0,
 835             RF_ACTIVE);
 836         if(pci_info->regmap0 == NULL) {
 837             xge_resources_free(dev, xge_free_pci_info);
 838             XGE_EXIT_ON_ERR("Bus resource allocation for BAR0 failed",
 839                 attach_out, ENOMEM);
 840         }
 841         attr.bar0 = (char *)pci_info->regmap0;
 842
 843         pci_info->bar0resource = (xge_bus_resource_t*)
 844             xge_os_malloc(NULL, sizeof(xge_bus_resource_t));
 845         if(pci_info->bar0resource == NULL) {
 846             xge_resources_free(dev, xge_free_bar0);
 847             XGE_EXIT_ON_ERR("Memory allocation for BAR0 Resources failed",
 848                 attach_out, ENOMEM);
 849         }
 850         ((xge_bus_resource_t *)(pci_info->bar0resource))->bus_tag =
 851             rman_get_bustag(pci_info->regmap0);
 852         ((xge_bus_resource_t *)(pci_info->bar0resource))->bus_handle =
 853             rman_get_bushandle(pci_info->regmap0);
 854         ((xge_bus_resource_t *)(pci_info->bar0resource))->bar_start_addr =
 855             pci_info->regmap0;
 856
 857         /* Get virtual address for BAR1 */
 858         rid1 = PCIR_BAR(2);
 859         pci_info->regmap1 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid1,
 860             RF_ACTIVE);
 861         if(pci_info->regmap1 == NULL) {
 862             xge_resources_free(dev, xge_free_bar0_resource);
 863             XGE_EXIT_ON_ERR("Bus resource allocation for BAR1 failed",
 864                 attach_out, ENOMEM);
 865         }
 866         attr.bar1 = (char *)pci_info->regmap1;
 867
 868         pci_info->bar1resource = (xge_bus_resource_t*)
 869             xge_os_malloc(NULL, sizeof(xge_bus_resource_t));
 870         if(pci_info->bar1resource == NULL) {
 871             xge_resources_free(dev, xge_free_bar1);
 872             XGE_EXIT_ON_ERR("Memory allocation for BAR1 Resources failed",
 873                 attach_out, ENOMEM);
 874         }
 875         ((xge_bus_resource_t *)(pci_info->bar1resource))->bus_tag =
 876             rman_get_bustag(pci_info->regmap1);
 877         ((xge_bus_resource_t *)(pci_info->bar1resource))->bus_handle =
 878             rman_get_bushandle(pci_info->regmap1);
 879         ((xge_bus_resource_t *)(pci_info->bar1resource))->bar_start_addr =
 880             pci_info->regmap1;
 881
 882         /* Save PCI config space */
 883         xge_pci_space_save(dev);
 884
 885         attr.regh0 = (xge_bus_resource_t *) pci_info->bar0resource;
 886         attr.regh1 = (xge_bus_resource_t *) pci_info->bar1resource;
 887         attr.irqh  = lldev->irqhandle;
 888         attr.cfgh  = pci_info;
 889         attr.pdev  = pci_info;
 890
 891         /* Initialize device configuration parameters */
 892         xge_init_params(device_config, dev);
 893
 894         rid = 0;
 895         if(lldev->enabled_msi) {
 896             /* Number of MSI messages supported by device */
 897             msi_count = pci_msi_count(dev);
 898             if(msi_count > 1) {
 899                 /* Device supports MSI */
 900                 if(bootverbose) {
 901                     xge_trace(XGE_ERR, "MSI count: %d", msi_count);
 902                     xge_trace(XGE_ERR, "Now, driver supporting 1 message");
 903                 }
 904                 msi_count = 1;
 905                 error = pci_alloc_msi(dev, &msi_count);
 906                 if(error == 0) {
 907                     if(bootverbose)
 908                         xge_trace(XGE_ERR, "Allocated messages: %d", msi_count);
 909                     enable_msi = XGE_HAL_INTR_MODE_MSI;
 910                     rid = 1;
 911                 }
 912                 else {
 913                     if(bootverbose)
 914                         xge_trace(XGE_ERR, "pci_alloc_msi failed, %d", error);
 915                 }
 916             }
 917         }
 918         lldev->enabled_msi = enable_msi;
 919
 920         /* Allocate resource for irq */
 921         lldev->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 922             (RF_SHAREABLE | RF_ACTIVE));
 923         if(lldev->irq == NULL) {
 924             xge_trace(XGE_ERR, "Allocating irq resource for %s failed",
 925                 ((rid == 0) ? "line interrupt" : "MSI"));
 926             if(rid == 1) {
 927                 error = pci_release_msi(dev);
 928                 if(error != 0) {
 929                     xge_trace(XGE_ERR, "Releasing MSI resources failed %d",
 930                         error);
 931                     xge_trace(XGE_ERR, "Requires reboot to use MSI again");
 932                 }
 933                 xge_trace(XGE_ERR, "Trying line interrupts");
 934                 rid = 0;
 935                 lldev->enabled_msi = XGE_HAL_INTR_MODE_IRQLINE;
 936                 lldev->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 937                     (RF_SHAREABLE | RF_ACTIVE));
 938             }
 939             if(lldev->irq == NULL) {
 940                 xge_trace(XGE_ERR, "Allocating irq resource failed");
 941                 xge_resources_free(dev, xge_free_bar1_resource);
 942                 status = ENOMEM;
 943                 goto attach_out;
 944             }
 945         }
 946
 947         device_config->intr_mode = lldev->enabled_msi;
 948         if(bootverbose) {
 949             xge_trace(XGE_TRACE, "rid: %d, Mode: %d, MSI count: %d", rid,
 950                 lldev->enabled_msi, msi_count);
 951         }
 952
 953         /* Initialize HAL device */
 954         error = xge_hal_device_initialize(hldev, &attr, device_config);
 955         if(error != XGE_HAL_OK) {
 956             xge_resources_free(dev, xge_free_irq_resource);
 957             XGE_EXIT_ON_ERR("Initializing HAL device failed", attach_out,
 958                 ENXIO);
 959         }
 960
 961         xge_hal_device_private_set(hldev, lldev);
 962
 963         error = xge_interface_setup(dev);
 964         if(error != 0) {
 965             status = error;
 966             goto attach_out;
 967         }
 968
 969         ifnetp         = lldev->ifnetp;
 970         ifnetp->if_mtu = device_config->mtu;
 971
 972         xge_media_init(dev);
 973
 974         /* Associate interrupt handler with the device */
 975         if(lldev->enabled_msi == XGE_HAL_INTR_MODE_MSI) {
 976             error = bus_setup_intr(dev, lldev->irq,
 977                 (INTR_TYPE_NET | INTR_MPSAFE),
 978 #if __FreeBSD_version > 700030
 979                 NULL,
 980 #endif
 981                 xge_isr_msi, lldev, &lldev->irqhandle);
 982             xge_msi_info_save(lldev);
 983         }
 984         else {
 985             error = bus_setup_intr(dev, lldev->irq,
 986                 (INTR_TYPE_NET | INTR_MPSAFE),
 987 #if __FreeBSD_version > 700030
 988                 xge_isr_filter,
 989 #endif
 990                 xge_isr_line, lldev, &lldev->irqhandle);
 991         }
 992         if(error != 0) {
 993             xge_resources_free(dev, xge_free_media_interface);
 994             XGE_EXIT_ON_ERR("Associating interrupt handler with device failed",
 995                 attach_out, ENXIO);
 996         }
 997
 998         xge_print_info(lldev);
 999
1000         xge_add_sysctl_handlers(lldev);
1001
1002         xge_buffer_mode_init(lldev, device_config->mtu);
1003
1004 attach_out:
1005         xge_os_free(NULL, device_config, sizeof(xge_hal_device_config_t));
1006 attach_out_config:
1007         gone_in_dev(dev, 12, "nxge(4) driver");
1008         return status;
1009 }
1010
1011 /**
1012  * xge_resources_free
1013  * Undo what-all we did during load/attach
1014  *
1015  * @dev Device Handle
1016  * @error Identifies what-all to undo
1017  */
1018 void
1019 xge_resources_free(device_t dev, xge_lables_e error)
1020 {
1021         xge_lldev_t *lldev;
1022         xge_pci_info_t *pci_info;
1023         xge_hal_device_t *hldev;
1024         int rid, status;
1025
1026         /* LL Device */
1027         lldev = (xge_lldev_t *) device_get_softc(dev);
1028         pci_info = lldev->pdev;
1029
1030         /* HAL Device */
1031         hldev = lldev->devh;
1032
1033         switch(error) {
1034             case xge_free_all:
1035                 /* Teardown interrupt handler - device association */
1036                 bus_teardown_intr(dev, lldev->irq, lldev->irqhandle);
1037
1038             case xge_free_media_interface:
1039                 /* Media */
1040                 ifmedia_removeall(&lldev->media);
1041
1042                 /* Detach Ether */
1043                 ether_ifdetach(lldev->ifnetp);
1044                 if_free(lldev->ifnetp);
1045
1046                 xge_hal_device_private_set(hldev, NULL);
1047                 xge_hal_device_disable(hldev);
1048
1049             case xge_free_terminate_hal_device:
1050                 /* HAL Device */
1051                 xge_hal_device_terminate(hldev);
1052
1053             case xge_free_irq_resource:
1054                 /* Release IRQ resource */
1055                 bus_release_resource(dev, SYS_RES_IRQ,
1056                     ((lldev->enabled_msi == XGE_HAL_INTR_MODE_IRQLINE) ? 0:1),
1057                     lldev->irq);
1058
1059                 if(lldev->enabled_msi == XGE_HAL_INTR_MODE_MSI) {
1060                     status = pci_release_msi(dev);
1061                     if(status != 0) {
1062                         if(bootverbose) {
1063                             xge_trace(XGE_ERR,
1064                                 "pci_release_msi returned %d", status);
1065                         }
1066                     }
1067                 }
1068
1069             case xge_free_bar1_resource:
1070                 /* Restore PCI configuration space */
1071                 xge_pci_space_restore(dev);
1072
1073                 /* Free bar1resource */
1074                 xge_os_free(NULL, pci_info->bar1resource,
1075                     sizeof(xge_bus_resource_t));
1076
1077             case xge_free_bar1:
1078                 /* Release BAR1 */
1079                 rid = PCIR_BAR(2);
1080                 bus_release_resource(dev, SYS_RES_MEMORY, rid,
1081                     pci_info->regmap1);
1082
1083             case xge_free_bar0_resource:
1084                 /* Free bar0resource */
1085                 xge_os_free(NULL, pci_info->bar0resource,
1086                     sizeof(xge_bus_resource_t));
1087
1088             case xge_free_bar0:
1089                 /* Release BAR0 */
1090                 rid = PCIR_BAR(0);
1091                 bus_release_resource(dev, SYS_RES_MEMORY, rid,
1092                     pci_info->regmap0);
1093
1094             case xge_free_pci_info:
1095                 /* Disable Bus Master */
1096                 pci_disable_busmaster(dev);
1097
1098                 /* Free pci_info_t */
1099                 lldev->pdev = NULL;
1100                 xge_os_free(NULL, pci_info, sizeof(xge_pci_info_t));
1101
1102             case xge_free_hal_device:
1103                 /* Free device configuration struct and HAL device */
1104                 xge_os_free(NULL, hldev, sizeof(xge_hal_device_t));
1105
1106             case xge_free_terminate_hal_driver:
1107                 /* Terminate HAL driver */
1108                 hal_driver_init_count = hal_driver_init_count - 1;
1109                 if(!hal_driver_init_count) {
1110                     xge_hal_driver_terminate();
1111                 }
1112
1113             case xge_free_mutex:
1114                 xge_mutex_destroy(lldev);
1115         }
1116 }
1117
1118 /**
1119  * xge_detach
1120  * Detaches driver from the Kernel subsystem
1121  *
1122  * @dev Device Handle
1123  */
1124 int
1125 xge_detach(device_t dev)
1126 {
1127         xge_lldev_t *lldev = (xge_lldev_t *)device_get_softc(dev);
1128
1129         if(lldev->in_detach == 0) {
1130             lldev->in_detach = 1;
1131             xge_stop(lldev);
1132             xge_resources_free(dev, xge_free_all);
1133         }
1134
1135         return 0;
1136 }
1137
1138 /**
1139  * xge_shutdown
1140  * To shutdown device before system shutdown
1141  *
1142  * @dev Device Handle
1143  */
1144 int
1145 xge_shutdown(device_t dev)
1146 {
1147         xge_lldev_t *lldev = (xge_lldev_t *) device_get_softc(dev);
1148         xge_stop(lldev);
1149
1150         return 0;
1151 }
1152
1153 /**
1154  * xge_interface_setup
1155  * Setup interface
1156  *
1157  * @dev Device Handle
1158  *
1159  * Returns 0 on success, ENXIO/ENOMEM on failure
1160  */
1161 int
1162 xge_interface_setup(device_t dev)
1163 {
1164         u8 mcaddr[ETHER_ADDR_LEN];
1165         xge_hal_status_e status;
1166         xge_lldev_t *lldev = (xge_lldev_t *)device_get_softc(dev);
1167         struct ifnet *ifnetp;
1168         xge_hal_device_t *hldev = lldev->devh;
1169
1170         /* Get the MAC address of the device */
1171         status = xge_hal_device_macaddr_get(hldev, 0, &mcaddr);
1172         if(status != XGE_HAL_OK) {
1173             xge_resources_free(dev, xge_free_terminate_hal_device);
1174             XGE_EXIT_ON_ERR("Getting MAC address failed", ifsetup_out, ENXIO);
1175         }
1176
1177         /* Get interface ifnet structure for this Ether device */
1178         ifnetp = lldev->ifnetp = if_alloc(IFT_ETHER);
1179         if(ifnetp == NULL) {
1180             xge_resources_free(dev, xge_free_terminate_hal_device);
1181             XGE_EXIT_ON_ERR("Allocation ifnet failed", ifsetup_out, ENOMEM);
1182         }
1183
1184         /* Initialize interface ifnet structure */
1185         if_initname(ifnetp, device_get_name(dev), device_get_unit(dev));
1186         ifnetp->if_mtu      = XGE_HAL_DEFAULT_MTU;
1187         ifnetp->if_baudrate = XGE_BAUDRATE;
1188         ifnetp->if_init     = xge_init;
1189         ifnetp->if_softc    = lldev;
1190         ifnetp->if_flags    = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1191         ifnetp->if_ioctl    = xge_ioctl;
1192         ifnetp->if_start    = xge_send;
1193
1194         /* TODO: Check and assign optimal value */
1195         ifnetp->if_snd.ifq_maxlen = ifqmaxlen;
1196
1197         ifnetp->if_capabilities = IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU |
1198             IFCAP_HWCSUM;
1199         if(lldev->enabled_tso)
1200             ifnetp->if_capabilities |= IFCAP_TSO4;
1201         if(lldev->enabled_lro)
1202             ifnetp->if_capabilities |= IFCAP_LRO;
1203
1204         ifnetp->if_capenable = ifnetp->if_capabilities;
1205
1206         /* Attach the interface */
1207         ether_ifattach(ifnetp, mcaddr);
1208
1209 ifsetup_out:
1210         return status;
1211 }
1212
1213 /**
1214  * xge_callback_link_up
1215  * Callback for Link-up indication from HAL
1216  *
1217  * @userdata Per-adapter data
1218  */
1219 void
1220 xge_callback_link_up(void *userdata)
1221 {
1222         xge_lldev_t  *lldev  = (xge_lldev_t *)userdata;
1223         struct ifnet *ifnetp = lldev->ifnetp;
1224
1225         ifnetp->if_flags  &= ~IFF_DRV_OACTIVE;
1226         if_link_state_change(ifnetp, LINK_STATE_UP);
1227 }
1228
1229 /**
1230  * xge_callback_link_down
1231  * Callback for Link-down indication from HAL
1232  *
1233  * @userdata Per-adapter data
1234  */
1235 void
1236 xge_callback_link_down(void *userdata)
1237 {
1238         xge_lldev_t  *lldev  = (xge_lldev_t *)userdata;
1239         struct ifnet *ifnetp = lldev->ifnetp;
1240
1241         ifnetp->if_flags  |= IFF_DRV_OACTIVE;
1242         if_link_state_change(ifnetp, LINK_STATE_DOWN);
1243 }
1244
1245 /**
1246  * xge_callback_crit_err
1247  * Callback for Critical error indication from HAL
1248  *
1249  * @userdata Per-adapter data
1250  * @type Event type (Enumerated hardware error)
1251  * @serr_data Hardware status
1252  */
1253 void
1254 xge_callback_crit_err(void *userdata, xge_hal_event_e type, u64 serr_data)
1255 {
1256         xge_trace(XGE_ERR, "Critical Error");
1257         xge_reset(userdata);
1258 }
1259
1260 /**
1261  * xge_callback_event
1262  * Callback from HAL indicating that some event has been queued
1263  *
1264  * @item Queued event item
1265  */
1266 void
1267 xge_callback_event(xge_queue_item_t *item)
1268 {
1269         xge_lldev_t      *lldev  = NULL;
1270         xge_hal_device_t *hldev  = NULL;
1271         struct ifnet     *ifnetp = NULL;
1272
1273         hldev  = item->context;
1274         lldev  = xge_hal_device_private(hldev);
1275         ifnetp = lldev->ifnetp;
1276
1277         switch((int)item->event_type) {
1278             case XGE_LL_EVENT_TRY_XMIT_AGAIN:
1279                 if(lldev->initialized) {
1280                     if(xge_hal_channel_dtr_count(lldev->fifo_channel[0]) > 0) {
1281                         ifnetp->if_flags  &= ~IFF_DRV_OACTIVE;
1282                     }
1283                     else {
1284                         xge_queue_produce_context(
1285                             xge_hal_device_queue(lldev->devh),
1286                             XGE_LL_EVENT_TRY_XMIT_AGAIN, lldev->devh);
1287                     }
1288                 }
1289                 break;
1290
1291             case XGE_LL_EVENT_DEVICE_RESETTING:
1292                 xge_reset(item->context);
1293                 break;
1294
1295             default:
1296                 break;
1297         }
1298 }
1299
1300 /**
1301  * xge_ifmedia_change
1302  * Media change driver callback
1303  *
1304  * @ifnetp Interface Handle
1305  *
1306  * Returns 0 if media is Ether else EINVAL
1307  */
1308 int
1309 xge_ifmedia_change(struct ifnet *ifnetp)
1310 {
1311         xge_lldev_t    *lldev    = ifnetp->if_softc;
1312         struct ifmedia *ifmediap = &lldev->media;
1313
1314         return (IFM_TYPE(ifmediap->ifm_media) != IFM_ETHER) ?  EINVAL:0;
1315 }
1316
1317 /**
1318  * xge_ifmedia_status
1319  * Media status driver callback
1320  *
1321  * @ifnetp Interface Handle
1322  * @ifmr Interface Media Settings
1323  */
1324 void
1325 xge_ifmedia_status(struct ifnet *ifnetp, struct ifmediareq *ifmr)
1326 {
1327         xge_hal_status_e status;
1328         u64              regvalue;
1329         xge_lldev_t      *lldev = ifnetp->if_softc;
1330         xge_hal_device_t *hldev = lldev->devh;
1331
1332         ifmr->ifm_status = IFM_AVALID;
1333         ifmr->ifm_active = IFM_ETHER;
1334
1335         status = xge_hal_mgmt_reg_read(hldev, 0,
1336             xge_offsetof(xge_hal_pci_bar0_t, adapter_status), &regvalue);
1337         if(status != XGE_HAL_OK) {
1338             xge_trace(XGE_TRACE, "Getting adapter status failed");
1339             goto _exit;
1340         }
1341
1342         if((regvalue & (XGE_HAL_ADAPTER_STATUS_RMAC_REMOTE_FAULT |
1343             XGE_HAL_ADAPTER_STATUS_RMAC_LOCAL_FAULT)) == 0) {
1344             ifmr->ifm_status |= IFM_ACTIVE;
1345             ifmr->ifm_active |= IFM_10G_SR | IFM_FDX;
1346             if_link_state_change(ifnetp, LINK_STATE_UP);
1347         }
1348         else {
1349             if_link_state_change(ifnetp, LINK_STATE_DOWN);
1350         }
1351 _exit:
1352         return;
1353 }
1354
1355 /**
1356  * xge_ioctl_stats
1357  * IOCTL to get statistics
1358  *
1359  * @lldev Per-adapter data
1360  * @ifreqp Interface request
1361  */
1362 int
1363 xge_ioctl_stats(xge_lldev_t *lldev, struct ifreq *ifreqp)
1364 {
1365         xge_hal_status_e status = XGE_HAL_OK;
1366         char cmd, mode;
1367         void *info = NULL;
1368         int retValue;
1369
1370         cmd = retValue = fubyte(ifr_data_get_ptr(ifreqp));
1371         if (retValue == -1)
1372                 return (EFAULT);
1373
1374         retValue = EINVAL;
1375         switch(cmd) {
1376             case XGE_QUERY_STATS:
1377                 mtx_lock(&lldev->mtx_drv);
1378                 status = xge_hal_stats_hw(lldev->devh,
1379                     (xge_hal_stats_hw_info_t **)&info);
1380                 mtx_unlock(&lldev->mtx_drv);
1381                 if(status == XGE_HAL_OK) {
1382                     if(copyout(info, ifr_data_get_ptr(ifreqp),
1383                         sizeof(xge_hal_stats_hw_info_t)) == 0)
1384                         retValue = 0;
1385                 }
1386                 else {
1387                     xge_trace(XGE_ERR, "Getting statistics failed (Status: %d)",
1388                         status);
1389                 }
1390                 break;
1391
1392             case XGE_QUERY_PCICONF:
1393                 info = xge_os_malloc(NULL, sizeof(xge_hal_pci_config_t));
1394                 if(info != NULL) {
1395                     mtx_lock(&lldev->mtx_drv);
1396                     status = xge_hal_mgmt_pci_config(lldev->devh, info,
1397                         sizeof(xge_hal_pci_config_t));
1398                     mtx_unlock(&lldev->mtx_drv);
1399                     if(status == XGE_HAL_OK) {
1400                         if(copyout(info, ifr_data_get_ptr(ifreqp),
1401                             sizeof(xge_hal_pci_config_t)) == 0)
1402                             retValue = 0;
1403                     }
1404                     else {
1405                         xge_trace(XGE_ERR,
1406                             "Getting PCI configuration failed (%d)", status);
1407                     }
1408                     xge_os_free(NULL, info, sizeof(xge_hal_pci_config_t));
1409                 }
1410                 break;
1411
1412             case XGE_QUERY_DEVSTATS:
1413                 info = xge_os_malloc(NULL, sizeof(xge_hal_stats_device_info_t));
1414                 if(info != NULL) {
1415                     mtx_lock(&lldev->mtx_drv);
1416                     status =xge_hal_mgmt_device_stats(lldev->devh, info,
1417                         sizeof(xge_hal_stats_device_info_t));
1418                     mtx_unlock(&lldev->mtx_drv);
1419                     if(status == XGE_HAL_OK) {
1420                         if(copyout(info, ifr_data_get_ptr(ifreqp),
1421                             sizeof(xge_hal_stats_device_info_t)) == 0)
1422                             retValue = 0;
1423                     }
1424                     else {
1425                         xge_trace(XGE_ERR, "Getting device info failed (%d)",
1426                             status);
1427                     }
1428                     xge_os_free(NULL, info,
1429                         sizeof(xge_hal_stats_device_info_t));
1430                 }
1431                 break;
1432
1433             case XGE_QUERY_SWSTATS:
1434                 info = xge_os_malloc(NULL, sizeof(xge_hal_stats_sw_err_t));
1435                 if(info != NULL) {
1436                     mtx_lock(&lldev->mtx_drv);
1437                     status =xge_hal_mgmt_sw_stats(lldev->devh, info,
1438                         sizeof(xge_hal_stats_sw_err_t));
1439                     mtx_unlock(&lldev->mtx_drv);
1440                     if(status == XGE_HAL_OK) {
1441                         if(copyout(info, ifr_data_get_ptr(ifreqp),
1442                             sizeof(xge_hal_stats_sw_err_t)) == 0)
1443                             retValue = 0;
1444                     }
1445                     else {
1446                         xge_trace(XGE_ERR,
1447                             "Getting tcode statistics failed (%d)", status);
1448                     }
1449                     xge_os_free(NULL, info, sizeof(xge_hal_stats_sw_err_t));
1450                 }
1451                 break;
1452
1453             case XGE_QUERY_DRIVERSTATS:
1454                 if(copyout(&lldev->driver_stats, ifr_data_get_ptr(ifreqp),
1455                     sizeof(xge_driver_stats_t)) == 0) {
1456                     retValue = 0;
1457                 }
1458                 else {
1459                     xge_trace(XGE_ERR,
1460                         "Copyout of driver statistics failed (%d)", status);
1461                 }
1462                 break;
1463
1464             case XGE_READ_VERSION:
1465                 info = xge_os_malloc(NULL, XGE_BUFFER_SIZE);
1466                 if(info != NULL) {
1467                     strcpy(info, XGE_DRIVER_VERSION);
1468                     if(copyout(info, ifr_data_get_ptr(ifreqp),
1469                         XGE_BUFFER_SIZE) == 0)
1470                         retValue = 0;
1471                     xge_os_free(NULL, info, XGE_BUFFER_SIZE);
1472                 }
1473                 break;
1474
1475             case XGE_QUERY_DEVCONF:
1476                 info = xge_os_malloc(NULL, sizeof(xge_hal_device_config_t));
1477                 if(info != NULL) {
1478                     mtx_lock(&lldev->mtx_drv);
1479                     status = xge_hal_mgmt_device_config(lldev->devh, info,
1480                         sizeof(xge_hal_device_config_t));
1481                     mtx_unlock(&lldev->mtx_drv);
1482                     if(status == XGE_HAL_OK) {
1483                         if(copyout(info, ifr_data_get_ptr(ifreqp),
1484                             sizeof(xge_hal_device_config_t)) == 0)
1485                             retValue = 0;
1486                     }
1487                     else {
1488                         xge_trace(XGE_ERR, "Getting devconfig failed (%d)",
1489                             status);
1490                     }
1491                     xge_os_free(NULL, info, sizeof(xge_hal_device_config_t));
1492                 }
1493                 break;
1494
1495             case XGE_QUERY_BUFFER_MODE:
1496                 if(copyout(&lldev->buffer_mode, ifr_data_get_ptr(ifreqp),
1497                     sizeof(int)) == 0)
1498                     retValue = 0;
1499                 break;
1500
1501             case XGE_SET_BUFFER_MODE_1:
1502             case XGE_SET_BUFFER_MODE_2:
1503             case XGE_SET_BUFFER_MODE_5:
1504                 mode = (cmd == XGE_SET_BUFFER_MODE_1) ? 'Y':'N';
1505                 if(copyout(&mode, ifr_data_get_ptr(ifreqp), sizeof(mode)) == 0)
1506                     retValue = 0;
1507                 break;
1508             default:
1509                 xge_trace(XGE_TRACE, "Nothing is matching");
1510                 retValue = ENOTTY;
1511                 break;
1512         }
1513         return retValue;
1514 }
1515
1516 /**
1517  * xge_ioctl_registers
1518  * IOCTL to get registers
1519  *
1520  * @lldev Per-adapter data
1521  * @ifreqp Interface request
1522  */
1523 int
1524 xge_ioctl_registers(xge_lldev_t *lldev, struct ifreq *ifreqp)
1525 {
1526         xge_register_t tmpdata;
1527         xge_register_t *data;
1528         xge_hal_status_e status = XGE_HAL_OK;
1529         int retValue = EINVAL, offset = 0, index = 0;
1530         int error;
1531         u64 val64 = 0;
1532
1533         error = copyin(ifr_data_get_ptr(ifreqp), &tmpdata, sizeof(tmpdata));
1534         if (error != 0)
1535                 return (error);
1536         data = &tmpdata;
1537
1538         /* Reading a register */
1539         if(strcmp(data->option, "-r") == 0) {
1540             data->value = 0x0000;
1541             mtx_lock(&lldev->mtx_drv);
1542             status = xge_hal_mgmt_reg_read(lldev->devh, 0, data->offset,
1543                 &data->value);
1544             mtx_unlock(&lldev->mtx_drv);
1545             if(status == XGE_HAL_OK) {
1546                 if(copyout(data, ifr_data_get_ptr(ifreqp),
1547                     sizeof(xge_register_t)) == 0)
1548                     retValue = 0;
1549             }
1550         }
1551         /* Writing to a register */
1552         else if(strcmp(data->option, "-w") == 0) {
1553             mtx_lock(&lldev->mtx_drv);
1554             status = xge_hal_mgmt_reg_write(lldev->devh, 0, data->offset,
1555                 data->value);
1556             if(status == XGE_HAL_OK) {
1557                 val64 = 0x0000;
1558                 status = xge_hal_mgmt_reg_read(lldev->devh, 0, data->offset,
1559                     &val64);
1560                 if(status != XGE_HAL_OK) {
1561                     xge_trace(XGE_ERR, "Reading back updated register failed");
1562                 }
1563                 else {
1564                     if(val64 != data->value) {
1565                         xge_trace(XGE_ERR,
1566                             "Read and written register values mismatched");
1567                     }
1568                     else retValue = 0;
1569                 }
1570             }
1571             else {
1572                 xge_trace(XGE_ERR, "Getting register value failed");
1573             }
1574             mtx_unlock(&lldev->mtx_drv);
1575         }
1576         else {
1577             mtx_lock(&lldev->mtx_drv);
1578             for(index = 0, offset = 0; offset <= XGE_OFFSET_OF_LAST_REG;
1579                 index++, offset += 0x0008) {
1580                 val64 = 0;
1581                 status = xge_hal_mgmt_reg_read(lldev->devh, 0, offset, &val64);
1582                 if(status != XGE_HAL_OK) {
1583                     xge_trace(XGE_ERR, "Getting register value failed");
1584                     break;
1585                 }
1586                 *((u64 *)((u64 *)data + index)) = val64;
1587                 retValue = 0;
1588             }
1589             mtx_unlock(&lldev->mtx_drv);
1590
1591             if(retValue == 0) {
1592                 if(copyout(data, ifr_data_get_ptr(ifreqp),
1593                     sizeof(xge_hal_pci_bar0_t)) != 0) {
1594                     xge_trace(XGE_ERR, "Copyout of register values failed");
1595                     retValue = EINVAL;
1596                 }
1597             }
1598             else {
1599                 xge_trace(XGE_ERR, "Getting register values failed");
1600             }
1601         }
1602         return retValue;
1603 }
1604
1605 /**
1606  * xge_ioctl
1607  * Callback to control the device - Interface configuration
1608  *
1609  * @ifnetp Interface Handle
1610  * @command Device control command
1611  * @data Parameters associated with command (if any)
1612  */
1613 int
1614 xge_ioctl(struct ifnet *ifnetp, unsigned long command, caddr_t data)
1615 {
1616         struct ifreq   *ifreqp   = (struct ifreq *)data;
1617         xge_lldev_t    *lldev    = ifnetp->if_softc;
1618         struct ifmedia *ifmediap = &lldev->media;
1619         int             retValue = 0, mask = 0;
1620
1621         if(lldev->in_detach) {
1622             return retValue;
1623         }
1624
1625         switch(command) {
1626             /* Set/Get ifnet address */
1627             case SIOCSIFADDR:
1628             case SIOCGIFADDR:
1629                 ether_ioctl(ifnetp, command, data);
1630                 break;
1631
1632             /* Set ifnet MTU */
1633             case SIOCSIFMTU:
1634                 retValue = xge_change_mtu(lldev, ifreqp->ifr_mtu);
1635                 break;
1636
1637             /* Set ifnet flags */
1638             case SIOCSIFFLAGS:
1639                 if(ifnetp->if_flags & IFF_UP) {
1640                     /* Link status is UP */
1641                     if(!(ifnetp->if_drv_flags & IFF_DRV_RUNNING)) {
1642                         xge_init(lldev);
1643                     }
1644                     xge_disable_promisc(lldev);
1645                     xge_enable_promisc(lldev);
1646                 }
1647                 else {
1648                     /* Link status is DOWN */
1649                     /* If device is in running, make it down */
1650                     if(ifnetp->if_drv_flags & IFF_DRV_RUNNING) {
1651                         xge_stop(lldev);
1652                     }
1653                 }
1654                 break;
1655
1656             /* Add/delete multicast address */
1657             case SIOCADDMULTI:
1658             case SIOCDELMULTI:
1659                 if(ifnetp->if_drv_flags & IFF_DRV_RUNNING) {
1660                     xge_setmulti(lldev);
1661                 }
1662                 break;
1663
1664             /* Set/Get net media */
1665             case SIOCSIFMEDIA:
1666             case SIOCGIFMEDIA:
1667                 retValue = ifmedia_ioctl(ifnetp, ifreqp, ifmediap, command);
1668                 break;
1669
1670             /* Set capabilities */
1671             case SIOCSIFCAP:
1672                 mtx_lock(&lldev->mtx_drv);
1673                 mask = ifreqp->ifr_reqcap ^ ifnetp->if_capenable;
1674                 if(mask & IFCAP_TXCSUM) {
1675                     if(ifnetp->if_capenable & IFCAP_TXCSUM) {
1676                         ifnetp->if_capenable &= ~(IFCAP_TSO4 | IFCAP_TXCSUM);
1677                         ifnetp->if_hwassist &=
1678                             ~(CSUM_TCP | CSUM_UDP | CSUM_TSO);
1679                     }
1680                     else {
1681                         ifnetp->if_capenable |= IFCAP_TXCSUM;
1682                         ifnetp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1683                     }
1684                 }
1685                 if(mask & IFCAP_TSO4) {
1686                     if(ifnetp->if_capenable & IFCAP_TSO4) {
1687                         ifnetp->if_capenable &= ~IFCAP_TSO4;
1688                         ifnetp->if_hwassist  &= ~CSUM_TSO;
1689
1690                         xge_os_printf("%s: TSO Disabled",
1691                             device_get_nameunit(lldev->device));
1692                     }
1693                     else if(ifnetp->if_capenable & IFCAP_TXCSUM) {
1694                         ifnetp->if_capenable |= IFCAP_TSO4;
1695                         ifnetp->if_hwassist  |= CSUM_TSO;
1696
1697                         xge_os_printf("%s: TSO Enabled",
1698                             device_get_nameunit(lldev->device));
1699                     }
1700                 }
1701
1702                 mtx_unlock(&lldev->mtx_drv);
1703                 break;
1704
1705             /* Custom IOCTL 0 */
1706             case SIOCGPRIVATE_0:
1707                 retValue = xge_ioctl_stats(lldev, ifreqp);
1708                 break;
1709
1710             /* Custom IOCTL 1 */
1711             case SIOCGPRIVATE_1:
1712                 retValue = xge_ioctl_registers(lldev, ifreqp);
1713                 break;
1714
1715             default:
1716                 retValue = EINVAL;
1717                 break;
1718         }
1719         return retValue;
1720 }
1721
1722 /**
1723  * xge_init
1724  * Initialize the interface
1725  *
1726  * @plldev Per-adapter Data
1727  */
1728 void
1729 xge_init(void *plldev)
1730 {
1731         xge_lldev_t *lldev = (xge_lldev_t *)plldev;
1732
1733         mtx_lock(&lldev->mtx_drv);
1734         xge_os_memzero(&lldev->driver_stats, sizeof(xge_driver_stats_t));
1735         xge_device_init(lldev, XGE_HAL_CHANNEL_OC_NORMAL);
1736         mtx_unlock(&lldev->mtx_drv);
1737 }
1738
1739 /**
1740  * xge_device_init
1741  * Initialize the interface (called by holding lock)
1742  *
1743  * @pdevin Per-adapter Data
1744  */
1745 void
1746 xge_device_init(xge_lldev_t *lldev, xge_hal_channel_reopen_e option)
1747 {
1748         struct ifnet     *ifnetp = lldev->ifnetp;
1749         xge_hal_device_t *hldev  = lldev->devh;
1750         struct ifaddr      *ifaddrp;
1751         unsigned char      *macaddr;
1752         struct sockaddr_dl *sockaddrp;
1753         int                 status   = XGE_HAL_OK;
1754
1755         mtx_assert((&lldev->mtx_drv), MA_OWNED);
1756
1757         /* If device is in running state, initializing is not required */
1758         if(ifnetp->if_drv_flags & IFF_DRV_RUNNING)
1759             return;
1760
1761         /* Initializing timer */
1762         callout_init(&lldev->timer, 1);
1763
1764         xge_trace(XGE_TRACE, "Set MTU size");
1765         status = xge_hal_device_mtu_set(hldev, ifnetp->if_mtu);
1766         if(status != XGE_HAL_OK) {
1767             xge_trace(XGE_ERR, "Setting MTU in HAL device failed");
1768             goto _exit;
1769         }
1770
1771         /* Enable HAL device */
1772         xge_hal_device_enable(hldev);
1773
1774         /* Get MAC address and update in HAL */
1775         ifaddrp             = ifnetp->if_addr;
1776         sockaddrp           = (struct sockaddr_dl *)ifaddrp->ifa_addr;
1777         sockaddrp->sdl_type = IFT_ETHER;
1778         sockaddrp->sdl_alen = ifnetp->if_addrlen;
1779         macaddr             = LLADDR(sockaddrp);
1780         xge_trace(XGE_TRACE,
1781             "Setting MAC address: %02x:%02x:%02x:%02x:%02x:%02x\n",
1782             *macaddr, *(macaddr + 1), *(macaddr + 2), *(macaddr + 3),
1783             *(macaddr + 4), *(macaddr + 5));
1784         status = xge_hal_device_macaddr_set(hldev, 0, macaddr);
1785         if(status != XGE_HAL_OK)
1786             xge_trace(XGE_ERR, "Setting MAC address failed (%d)", status);
1787
1788         /* Opening channels */
1789         mtx_unlock(&lldev->mtx_drv);
1790         status = xge_channel_open(lldev, option);
1791         mtx_lock(&lldev->mtx_drv);
1792         if(status != XGE_HAL_OK)
1793             goto _exit;
1794
1795         /* Set appropriate flags */
1796         ifnetp->if_drv_flags  |=  IFF_DRV_RUNNING;
1797         ifnetp->if_flags &= ~IFF_DRV_OACTIVE;
1798
1799         /* Checksum capability */
1800         ifnetp->if_hwassist = (ifnetp->if_capenable & IFCAP_TXCSUM) ?
1801             (CSUM_TCP | CSUM_UDP) : 0;
1802
1803         if((lldev->enabled_tso) && (ifnetp->if_capenable & IFCAP_TSO4))
1804             ifnetp->if_hwassist |= CSUM_TSO;
1805
1806         /* Enable interrupts */
1807         xge_hal_device_intr_enable(hldev);
1808
1809         callout_reset(&lldev->timer, 10*hz, xge_timer, lldev);
1810
1811         /* Disable promiscuous mode */
1812         xge_trace(XGE_TRACE, "If opted, enable promiscuous mode");
1813         xge_enable_promisc(lldev);
1814
1815         /* Device is initialized */
1816         lldev->initialized = 1;
1817         xge_os_mdelay(1000);
1818
1819 _exit:
1820         return;
1821 }
1822
1823 /**
1824  * xge_timer
1825  * Timer timeout function to handle link status
1826  *
1827  * @devp Per-adapter Data
1828  */
1829 void
1830 xge_timer(void *devp)
1831 {
1832         xge_lldev_t      *lldev = (xge_lldev_t *)devp;
1833         xge_hal_device_t *hldev = lldev->devh;
1834
1835         /* Poll for changes */
1836         xge_hal_device_poll(hldev);
1837
1838         /* Reset timer */
1839         callout_reset(&lldev->timer, hz, xge_timer, lldev);
1840
1841         return;
1842 }
1843
1844 /**
1845  * xge_stop
1846  * De-activate the interface
1847  *
1848  * @lldev Per-adater Data
1849  */
1850 void
1851 xge_stop(xge_lldev_t *lldev)
1852 {
1853         mtx_lock(&lldev->mtx_drv);
1854         xge_device_stop(lldev, XGE_HAL_CHANNEL_OC_NORMAL);
1855         mtx_unlock(&lldev->mtx_drv);
1856 }
1857
1858 /**
1859  * xge_isr_filter
1860  * ISR filter function - to filter interrupts from other devices (shared)
1861  *
1862  * @handle Per-adapter Data
1863  *
1864  * Returns
1865  * FILTER_STRAY if interrupt is from other device
1866  * FILTER_SCHEDULE_THREAD if interrupt is from Xframe device
1867  */
1868 int
1869 xge_isr_filter(void *handle)
1870 {
1871         xge_lldev_t *lldev       = (xge_lldev_t *)handle;
1872         xge_hal_pci_bar0_t *bar0 = (xge_hal_pci_bar0_t *)((lldev->devh)->bar0);
1873         u16 retValue = FILTER_STRAY;
1874         u64 val64    = 0;
1875
1876         XGE_DRV_STATS(isr_filter);
1877
1878         val64 = xge_os_pio_mem_read64(lldev->pdev, (lldev->devh)->regh0,
1879             &bar0->general_int_status);
1880         retValue = (!val64) ? FILTER_STRAY : FILTER_SCHEDULE_THREAD;
1881
1882         return retValue;
1883 }
1884
1885 /**
1886  * xge_isr_line
1887  * Interrupt service routine for Line interrupts
1888  *
1889  * @plldev Per-adapter Data
1890  */
1891 void
1892 xge_isr_line(void *plldev)
1893 {
1894         xge_hal_status_e status;
1895         xge_lldev_t      *lldev   = (xge_lldev_t *)plldev;
1896         xge_hal_device_t *hldev   = (xge_hal_device_t *)lldev->devh;
1897         struct ifnet     *ifnetp  = lldev->ifnetp;
1898
1899         XGE_DRV_STATS(isr_line);
1900
1901         if(ifnetp->if_drv_flags & IFF_DRV_RUNNING) {
1902             status = xge_hal_device_handle_irq(hldev);
1903             if(!(IFQ_DRV_IS_EMPTY(&ifnetp->if_snd)))
1904                 xge_send(ifnetp);
1905         }
1906 }
1907
1908 /*
1909  * xge_isr_msi
1910  * ISR for Message signaled interrupts
1911  */
1912 void
1913 xge_isr_msi(void *plldev)
1914 {
1915         xge_lldev_t *lldev = (xge_lldev_t *)plldev;
1916         XGE_DRV_STATS(isr_msi);
1917         xge_hal_device_continue_irq(lldev->devh);
1918 }
1919
1920 /**
1921  * xge_rx_open
1922  * Initiate and open all Rx channels
1923  *
1924  * @qid Ring Index
1925  * @lldev Per-adapter Data
1926  * @rflag Channel open/close/reopen flag
1927  *
1928  * Returns 0 or Error Number
1929  */
1930 int
1931 xge_rx_open(int qid, xge_lldev_t *lldev, xge_hal_channel_reopen_e rflag)
1932 {
1933         u64 adapter_status = 0x0;
1934         xge_hal_status_e status = XGE_HAL_FAIL;
1935
1936         xge_hal_channel_attr_t attr = {
1937             .post_qid      = qid,
1938             .compl_qid     = 0,
1939             .callback      = xge_rx_compl,
1940             .per_dtr_space = sizeof(xge_rx_priv_t),
1941             .flags         = 0,
1942             .type          = XGE_HAL_CHANNEL_TYPE_RING,
1943             .userdata      = lldev,
1944             .dtr_init      = xge_rx_initial_replenish,
1945             .dtr_term      = xge_rx_term
1946         };
1947
1948         /* If device is not ready, return */
1949         status = xge_hal_device_status(lldev->devh, &adapter_status);
1950         if(status != XGE_HAL_OK) {
1951             xge_os_printf("Adapter Status: 0x%llx", (long long) adapter_status);
1952             XGE_EXIT_ON_ERR("Device is not ready", _exit, XGE_HAL_FAIL);
1953         }
1954         else {
1955             status = xge_hal_channel_open(lldev->devh, &attr,
1956                 &lldev->ring_channel[qid], rflag);
1957         }
1958
1959 _exit:
1960         return status;
1961 }
1962
1963 /**
1964  * xge_tx_open
1965  * Initialize and open all Tx channels
1966  *
1967  * @lldev Per-adapter Data
1968  * @tflag Channel open/close/reopen flag
1969  *
1970  * Returns 0 or Error Number
1971  */
1972 int
1973 xge_tx_open(xge_lldev_t *lldev, xge_hal_channel_reopen_e tflag)
1974 {
1975         xge_hal_status_e status = XGE_HAL_FAIL;
1976         u64 adapter_status = 0x0;
1977         int qindex, index;
1978
1979         xge_hal_channel_attr_t attr = {
1980             .compl_qid     = 0,
1981             .callback      = xge_tx_compl,
1982             .per_dtr_space = sizeof(xge_tx_priv_t),
1983             .flags         = 0,
1984             .type          = XGE_HAL_CHANNEL_TYPE_FIFO,
1985             .userdata      = lldev,
1986             .dtr_init      = xge_tx_initial_replenish,
1987             .dtr_term      = xge_tx_term
1988         };
1989
1990         /* If device is not ready, return */
1991         status = xge_hal_device_status(lldev->devh, &adapter_status);
1992         if(status != XGE_HAL_OK) {
1993             xge_os_printf("Adapter Status: 0x%llx", (long long) adapter_status);
1994             XGE_EXIT_ON_ERR("Device is not ready", _exit, XGE_HAL_FAIL);
1995         }
1996
1997         for(qindex = 0; qindex < XGE_FIFO_COUNT; qindex++) {
1998             attr.post_qid = qindex,
1999             status = xge_hal_channel_open(lldev->devh, &attr,
2000                 &lldev->fifo_channel[qindex], tflag);
2001             if(status != XGE_HAL_OK) {
2002                 for(index = 0; index < qindex; index++)
2003                     xge_hal_channel_close(lldev->fifo_channel[index], tflag);
2004             }
2005         }
2006
2007 _exit:
2008         return status;
2009 }
2010
2011 /**
2012  * xge_enable_msi
2013  * Enables MSI
2014  *
2015  * @lldev Per-adapter Data
2016  */
2017 void
2018 xge_enable_msi(xge_lldev_t *lldev)
2019 {
2020         xge_list_t        *item    = NULL;
2021         xge_hal_device_t  *hldev   = lldev->devh;
2022         xge_hal_channel_t *channel = NULL;
2023         u16 offset = 0, val16 = 0;
2024
2025         xge_os_pci_read16(lldev->pdev, NULL,
2026             xge_offsetof(xge_hal_pci_config_le_t, msi_control), &val16);
2027
2028         /* Update msi_data */
2029         offset = (val16 & 0x80) ? 0x4c : 0x48;
2030         xge_os_pci_read16(lldev->pdev, NULL, offset, &val16);
2031         if(val16 & 0x1)
2032             val16 &= 0xfffe;
2033         else
2034             val16 |= 0x1;
2035         xge_os_pci_write16(lldev->pdev, NULL, offset, val16);
2036
2037         /* Update msi_control */
2038         xge_os_pci_read16(lldev->pdev, NULL,
2039             xge_offsetof(xge_hal_pci_config_le_t, msi_control), &val16);
2040         val16 |= 0x10;
2041         xge_os_pci_write16(lldev->pdev, NULL,
2042             xge_offsetof(xge_hal_pci_config_le_t, msi_control), val16);
2043
2044         /* Set TxMAT and RxMAT registers with MSI */
2045         xge_list_for_each(item, &hldev->free_channels) {
2046             channel = xge_container_of(item, xge_hal_channel_t, item);
2047             xge_hal_channel_msi_set(channel, 1, (u32)val16);
2048         }
2049 }
2050
2051 /**
2052  * xge_channel_open
2053  * Open both Tx and Rx channels
2054  *
2055  * @lldev Per-adapter Data
2056  * @option Channel reopen option
2057  */
2058 int
2059 xge_channel_open(xge_lldev_t *lldev, xge_hal_channel_reopen_e option)
2060 {
2061         xge_lro_entry_t *lro_session = NULL;
2062         xge_hal_status_e status   = XGE_HAL_OK;
2063         int index = 0, index2 = 0;
2064
2065         if(lldev->enabled_msi == XGE_HAL_INTR_MODE_MSI) {
2066             xge_msi_info_restore(lldev);
2067             xge_enable_msi(lldev);
2068         }
2069
2070 _exit2:
2071         status = xge_create_dma_tags(lldev->device);
2072         if(status != XGE_HAL_OK)
2073             XGE_EXIT_ON_ERR("DMA tag creation failed", _exit, status);
2074
2075         /* Open ring (Rx) channel */
2076         for(index = 0; index < XGE_RING_COUNT; index++) {
2077             status = xge_rx_open(index, lldev, option);
2078             if(status != XGE_HAL_OK) {
2079                 /*
2080                  * DMA mapping fails in the unpatched Kernel which can't
2081                  * allocate contiguous memory for Jumbo frames.
2082                  * Try using 5 buffer mode.
2083                  */
2084                 if((lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_1) &&
2085                     (((lldev->ifnetp)->if_mtu + XGE_HAL_MAC_HEADER_MAX_SIZE) >
2086                     MJUMPAGESIZE)) {
2087                     /* Close so far opened channels */
2088                     for(index2 = 0; index2 < index; index2++) {
2089                         xge_hal_channel_close(lldev->ring_channel[index2],
2090                             option);
2091                     }
2092
2093                     /* Destroy DMA tags intended to use for 1 buffer mode */
2094                     if(bus_dmamap_destroy(lldev->dma_tag_rx,
2095                         lldev->extra_dma_map)) {
2096                         xge_trace(XGE_ERR, "Rx extra DMA map destroy failed");
2097                     }
2098                     if(bus_dma_tag_destroy(lldev->dma_tag_rx))
2099                         xge_trace(XGE_ERR, "Rx DMA tag destroy failed");
2100                     if(bus_dma_tag_destroy(lldev->dma_tag_tx))
2101                         xge_trace(XGE_ERR, "Tx DMA tag destroy failed");
2102
2103                     /* Switch to 5 buffer mode */
2104                     lldev->buffer_mode = XGE_HAL_RING_QUEUE_BUFFER_MODE_5;
2105                     xge_buffer_mode_init(lldev, (lldev->ifnetp)->if_mtu);
2106
2107                     /* Restart init */
2108                     goto _exit2;
2109                 }
2110                 else {
2111                     XGE_EXIT_ON_ERR("Opening Rx channel failed", _exit1,
2112                         status);
2113                 }
2114             }
2115         }
2116
2117         if(lldev->enabled_lro) {
2118             SLIST_INIT(&lldev->lro_free);
2119             SLIST_INIT(&lldev->lro_active);
2120             lldev->lro_num = XGE_LRO_DEFAULT_ENTRIES;
2121
2122             for(index = 0; index < lldev->lro_num; index++) {
2123                 lro_session = (xge_lro_entry_t *)
2124                     xge_os_malloc(NULL, sizeof(xge_lro_entry_t));
2125                 if(lro_session == NULL) {
2126                     lldev->lro_num = index;
2127                     break;
2128                 }
2129                 SLIST_INSERT_HEAD(&lldev->lro_free, lro_session, next);
2130             }
2131         }
2132
2133         /* Open FIFO (Tx) channel */
2134         status = xge_tx_open(lldev, option);
2135         if(status != XGE_HAL_OK)
2136             XGE_EXIT_ON_ERR("Opening Tx channel failed", _exit1, status);
2137
2138         goto _exit;
2139
2140 _exit1:
2141         /*
2142          * Opening Rx channel(s) failed (index is <last ring index - 1>) or
2143          * Initialization of LRO failed (index is XGE_RING_COUNT)
2144          * Opening Tx channel failed    (index is XGE_RING_COUNT)
2145          */
2146         for(index2 = 0; index2 < index; index2++)
2147             xge_hal_channel_close(lldev->ring_channel[index2], option);
2148
2149 _exit:
2150         return status;
2151 }
2152
2153 /**
2154  * xge_channel_close
2155  * Close both Tx and Rx channels
2156  *
2157  * @lldev Per-adapter Data
2158  * @option Channel reopen option
2159  *
2160  */
2161 void
2162 xge_channel_close(xge_lldev_t *lldev, xge_hal_channel_reopen_e option)
2163 {
2164         int qindex = 0;
2165
2166         DELAY(1000 * 1000);
2167
2168         /* Close FIFO (Tx) channel */
2169         for(qindex = 0; qindex < XGE_FIFO_COUNT; qindex++)
2170             xge_hal_channel_close(lldev->fifo_channel[qindex], option);
2171
2172         /* Close Ring (Rx) channels */
2173         for(qindex = 0; qindex < XGE_RING_COUNT; qindex++)
2174             xge_hal_channel_close(lldev->ring_channel[qindex], option);
2175
2176         if(bus_dmamap_destroy(lldev->dma_tag_rx, lldev->extra_dma_map))
2177             xge_trace(XGE_ERR, "Rx extra map destroy failed");
2178         if(bus_dma_tag_destroy(lldev->dma_tag_rx))
2179             xge_trace(XGE_ERR, "Rx DMA tag destroy failed");
2180         if(bus_dma_tag_destroy(lldev->dma_tag_tx))
2181             xge_trace(XGE_ERR, "Tx DMA tag destroy failed");
2182 }
2183
2184 /**
2185  * dmamap_cb
2186  * DMA map callback
2187  *
2188  * @arg Parameter passed from dmamap
2189  * @segs Segments
2190  * @nseg Number of segments
2191  * @error Error
2192  */
2193 void
2194 dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
2195 {
2196         if(!error) {
2197             *(bus_addr_t *) arg = segs->ds_addr;
2198         }
2199 }
2200
2201 /**
2202  * xge_reset
2203  * Device Reset
2204  *
2205  * @lldev Per-adapter Data
2206  */
2207 void
2208 xge_reset(xge_lldev_t *lldev)
2209 {
2210         xge_trace(XGE_TRACE, "Reseting the chip");
2211
2212         /* If the device is not initialized, return */
2213         if(lldev->initialized) {
2214             mtx_lock(&lldev->mtx_drv);
2215             xge_device_stop(lldev, XGE_HAL_CHANNEL_OC_NORMAL);
2216             xge_device_init(lldev, XGE_HAL_CHANNEL_OC_NORMAL);
2217             mtx_unlock(&lldev->mtx_drv);
2218         }
2219
2220         return;
2221 }
2222
2223 /**
2224  * xge_setmulti
2225  * Set an address as a multicast address
2226  *
2227  * @lldev Per-adapter Data
2228  */
2229 void
2230 xge_setmulti(xge_lldev_t *lldev)
2231 {
2232         struct ifmultiaddr *ifma;
2233         u8                 *lladdr;
2234         xge_hal_device_t   *hldev        = (xge_hal_device_t *)lldev->devh;
2235         struct ifnet       *ifnetp       = lldev->ifnetp;
2236         int                index         = 0;
2237         int                offset        = 1;
2238         int                table_size    = 47;
2239         xge_hal_status_e   status        = XGE_HAL_OK;
2240         u8                 initial_addr[]= {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
2241
2242         if((ifnetp->if_flags & IFF_MULTICAST) && (!lldev->all_multicast)) {
2243             status = xge_hal_device_mcast_enable(hldev);
2244             lldev->all_multicast = 1;
2245         }
2246         else if((ifnetp->if_flags & IFF_MULTICAST) && (lldev->all_multicast)) {
2247             status = xge_hal_device_mcast_disable(hldev);
2248             lldev->all_multicast = 0;
2249         }
2250
2251         if(status != XGE_HAL_OK) {
2252             xge_trace(XGE_ERR, "Enabling/disabling multicast failed");
2253             goto _exit;
2254         }
2255
2256         /* Updating address list */
2257         if_maddr_rlock(ifnetp);
2258         index = 0;
2259         TAILQ_FOREACH(ifma, &ifnetp->if_multiaddrs, ifma_link) {
2260             if(ifma->ifma_addr->sa_family != AF_LINK) {
2261                 continue;
2262             }
2263             lladdr = LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
2264             index += 1;
2265         }
2266         if_maddr_runlock(ifnetp);
2267
2268         if((!lldev->all_multicast) && (index)) {
2269             lldev->macaddr_count = (index + 1);
2270             if(lldev->macaddr_count > table_size) {
2271                 goto _exit;
2272             }
2273
2274             /* Clear old addresses */
2275             for(index = 0; index < 48; index++) {
2276                 xge_hal_device_macaddr_set(hldev, (offset + index),
2277                     initial_addr);
2278             }
2279         }
2280
2281         /* Add new addresses */
2282         if_maddr_rlock(ifnetp);
2283         index = 0;
2284         TAILQ_FOREACH(ifma, &ifnetp->if_multiaddrs, ifma_link) {
2285             if(ifma->ifma_addr->sa_family != AF_LINK) {
2286                 continue;
2287             }
2288             lladdr = LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
2289             xge_hal_device_macaddr_set(hldev, (offset + index), lladdr);
2290             index += 1;
2291         }
2292         if_maddr_runlock(ifnetp);
2293
2294 _exit:
2295         return;
2296 }
2297
2298 /**
2299  * xge_enable_promisc
2300  * Enable Promiscuous Mode
2301  *
2302  * @lldev Per-adapter Data
2303  */
2304 void
2305 xge_enable_promisc(xge_lldev_t *lldev)
2306 {
2307         struct ifnet *ifnetp = lldev->ifnetp;
2308         xge_hal_device_t *hldev = lldev->devh;
2309         xge_hal_pci_bar0_t *bar0 = NULL;
2310         u64 val64 = 0;
2311
2312         bar0 = (xge_hal_pci_bar0_t *) hldev->bar0;
2313
2314         if(ifnetp->if_flags & IFF_PROMISC) {
2315             xge_hal_device_promisc_enable(lldev->devh);
2316
2317             /*
2318              * When operating in promiscuous mode, don't strip the VLAN tag
2319              */
2320             val64 = xge_os_pio_mem_read64(lldev->pdev, hldev->regh0,
2321                 &bar0->rx_pa_cfg);
2322             val64 &= ~XGE_HAL_RX_PA_CFG_STRIP_VLAN_TAG_MODE(1);
2323             val64 |= XGE_HAL_RX_PA_CFG_STRIP_VLAN_TAG_MODE(0);
2324             xge_os_pio_mem_write64(lldev->pdev, hldev->regh0, val64,
2325                 &bar0->rx_pa_cfg);
2326
2327             xge_trace(XGE_TRACE, "Promiscuous mode ON");
2328         }
2329 }
2330
2331 /**
2332  * xge_disable_promisc
2333  * Disable Promiscuous Mode
2334  *
2335  * @lldev Per-adapter Data
2336  */
2337 void
2338 xge_disable_promisc(xge_lldev_t *lldev)
2339 {
2340         xge_hal_device_t *hldev = lldev->devh;
2341         xge_hal_pci_bar0_t *bar0 = NULL;
2342         u64 val64 = 0;
2343
2344         bar0 = (xge_hal_pci_bar0_t *) hldev->bar0;
2345
2346         xge_hal_device_promisc_disable(lldev->devh);
2347
2348         /*
2349          * Strip VLAN tag when operating in non-promiscuous mode
2350          */
2351         val64 = xge_os_pio_mem_read64(lldev->pdev, hldev->regh0,
2352             &bar0->rx_pa_cfg);
2353         val64 &= ~XGE_HAL_RX_PA_CFG_STRIP_VLAN_TAG_MODE(1);
2354         val64 |= XGE_HAL_RX_PA_CFG_STRIP_VLAN_TAG_MODE(1);
2355         xge_os_pio_mem_write64(lldev->pdev, hldev->regh0, val64,
2356             &bar0->rx_pa_cfg);
2357
2358         xge_trace(XGE_TRACE, "Promiscuous mode OFF");
2359 }
2360
2361 /**
2362  * xge_change_mtu
2363  * Change interface MTU to a requested valid size
2364  *
2365  * @lldev Per-adapter Data
2366  * @NewMtu Requested MTU
2367  *
2368  * Returns 0 or Error Number
2369  */
2370 int
2371 xge_change_mtu(xge_lldev_t *lldev, int new_mtu)
2372 {
2373         int status = XGE_HAL_OK;
2374
2375         /* Check requested MTU size for boundary */
2376         if(xge_hal_device_mtu_check(lldev->devh, new_mtu) != XGE_HAL_OK) {
2377             XGE_EXIT_ON_ERR("Invalid MTU", _exit, EINVAL);
2378         }
2379
2380         lldev->mtu = new_mtu;
2381         xge_confirm_changes(lldev, XGE_SET_MTU);
2382
2383 _exit:
2384         return status;
2385 }
2386
2387 /**
2388  * xge_device_stop
2389  *
2390  * Common code for both stop and part of reset. Disables device, interrupts and
2391  * closes channels
2392  *
2393  * @dev Device Handle
2394  * @option Channel normal/reset option
2395  */
2396 void
2397 xge_device_stop(xge_lldev_t *lldev, xge_hal_channel_reopen_e option)
2398 {
2399         xge_hal_device_t *hldev  = lldev->devh;
2400         struct ifnet     *ifnetp = lldev->ifnetp;
2401         u64               val64  = 0;
2402
2403         mtx_assert((&lldev->mtx_drv), MA_OWNED);
2404
2405         /* If device is not in "Running" state, return */
2406         if (!(ifnetp->if_drv_flags & IFF_DRV_RUNNING))
2407             goto _exit;
2408
2409         /* Set appropriate flags */
2410         ifnetp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
2411
2412         /* Stop timer */
2413         callout_stop(&lldev->timer);
2414
2415         /* Disable interrupts */
2416         xge_hal_device_intr_disable(hldev);
2417
2418         mtx_unlock(&lldev->mtx_drv);
2419         xge_queue_flush(xge_hal_device_queue(lldev->devh));
2420         mtx_lock(&lldev->mtx_drv);
2421
2422         /* Disable HAL device */
2423         if(xge_hal_device_disable(hldev) != XGE_HAL_OK) {
2424             xge_trace(XGE_ERR, "Disabling HAL device failed");
2425             xge_hal_device_status(hldev, &val64);
2426             xge_trace(XGE_ERR, "Adapter Status: 0x%llx", (long long)val64);
2427         }
2428
2429         /* Close Tx and Rx channels */
2430         xge_channel_close(lldev, option);
2431
2432         /* Reset HAL device */
2433         xge_hal_device_reset(hldev);
2434
2435         xge_os_mdelay(1000);
2436         lldev->initialized = 0;
2437
2438         if_link_state_change(ifnetp, LINK_STATE_DOWN);
2439
2440 _exit:
2441         return;
2442 }
2443
2444 /**
2445  * xge_set_mbuf_cflags
2446  * set checksum flag for the mbuf
2447  *
2448  * @pkt Packet
2449  */
2450 void
2451 xge_set_mbuf_cflags(mbuf_t pkt)
2452 {
2453         pkt->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
2454         pkt->m_pkthdr.csum_flags |= CSUM_IP_VALID;
2455         pkt->m_pkthdr.csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
2456         pkt->m_pkthdr.csum_data = htons(0xffff);
2457 }
2458
2459 /**
2460  * xge_lro_flush_sessions
2461  * Flush LRO session and send accumulated LRO packet to upper layer
2462  *
2463  * @lldev Per-adapter Data
2464  */
2465 void
2466 xge_lro_flush_sessions(xge_lldev_t *lldev)
2467 {
2468         xge_lro_entry_t *lro_session = NULL;
2469
2470         while(!SLIST_EMPTY(&lldev->lro_active)) {
2471             lro_session = SLIST_FIRST(&lldev->lro_active);
2472             SLIST_REMOVE_HEAD(&lldev->lro_active, next);
2473             xge_lro_flush(lldev, lro_session);
2474         }
2475 }
2476
2477 /**
2478  * xge_lro_flush
2479  * Flush LRO session. Send accumulated LRO packet to upper layer
2480  *
2481  * @lldev Per-adapter Data
2482  * @lro LRO session to be flushed
2483  */
2484 static void
2485 xge_lro_flush(xge_lldev_t *lldev, xge_lro_entry_t *lro_session)
2486 {
2487         struct ip *header_ip;
2488         struct tcphdr *header_tcp;
2489         u32 *ptr;
2490
2491         if(lro_session->append_cnt) {
2492             header_ip = lro_session->lro_header_ip;
2493             header_ip->ip_len = htons(lro_session->len - ETHER_HDR_LEN);
2494             lro_session->m_head->m_pkthdr.len = lro_session->len;
2495             header_tcp = (struct tcphdr *)(header_ip + 1);
2496             header_tcp->th_ack = lro_session->ack_seq;
2497             header_tcp->th_win = lro_session->window;
2498             if(lro_session->timestamp) {
2499                 ptr = (u32 *)(header_tcp + 1);
2500                 ptr[1] = htonl(lro_session->tsval);
2501                 ptr[2] = lro_session->tsecr;
2502             }
2503         }
2504
2505         (*lldev->ifnetp->if_input)(lldev->ifnetp, lro_session->m_head);
2506         lro_session->m_head = NULL;
2507         lro_session->timestamp = 0;
2508         lro_session->append_cnt = 0;
2509         SLIST_INSERT_HEAD(&lldev->lro_free, lro_session, next);
2510 }
2511
2512 /**
2513  * xge_lro_accumulate
2514  * Accumulate packets to form a large LRO packet based on various conditions
2515  *
2516  * @lldev Per-adapter Data
2517  * @m_head Current Packet
2518  *
2519  * Returns XGE_HAL_OK or XGE_HAL_FAIL (failure)
2520  */
2521 static int
2522 xge_lro_accumulate(xge_lldev_t *lldev, struct mbuf *m_head)
2523 {
2524         struct ether_header *header_ethernet;
2525         struct ip *header_ip;
2526         struct tcphdr *header_tcp;
2527         u32 seq, *ptr;
2528         struct mbuf *buffer_next, *buffer_tail;
2529         xge_lro_entry_t *lro_session;
2530         xge_hal_status_e status = XGE_HAL_FAIL;
2531         int hlen, ip_len, tcp_hdr_len, tcp_data_len, tot_len, tcp_options;
2532         int trim;
2533
2534         /* Get Ethernet header */
2535         header_ethernet = mtod(m_head, struct ether_header *);
2536
2537         /* Return if it is not IP packet */
2538         if(header_ethernet->ether_type != htons(ETHERTYPE_IP))
2539             goto _exit;
2540
2541         /* Get IP header */
2542         header_ip = lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_1 ?
2543             (struct ip *)(header_ethernet + 1) :
2544             mtod(m_head->m_next, struct ip *);
2545
2546         /* Return if it is not TCP packet */
2547         if(header_ip->ip_p != IPPROTO_TCP)
2548             goto _exit;
2549
2550         /* Return if packet has options */
2551         if((header_ip->ip_hl << 2) != sizeof(*header_ip))
2552             goto _exit;
2553
2554         /* Return if packet is fragmented */
2555         if(header_ip->ip_off & htons(IP_MF | IP_OFFMASK))
2556             goto _exit;
2557
2558         /* Get TCP header */
2559         header_tcp = (struct tcphdr *)(header_ip + 1);
2560
2561         /* Return if not ACK or PUSH */
2562         if((header_tcp->th_flags & ~(TH_ACK | TH_PUSH)) != 0)
2563             goto _exit;
2564
2565         /* Only timestamp option is handled */
2566         tcp_options = (header_tcp->th_off << 2) - sizeof(*header_tcp);
2567         tcp_hdr_len = sizeof(*header_tcp) + tcp_options;
2568         ptr = (u32 *)(header_tcp + 1);
2569         if(tcp_options != 0) {
2570             if(__predict_false(tcp_options != TCPOLEN_TSTAMP_APPA) ||
2571                 (*ptr != ntohl(TCPOPT_NOP << 24 | TCPOPT_NOP << 16 |
2572                 TCPOPT_TIMESTAMP << 8 | TCPOLEN_TIMESTAMP))) {
2573                 goto _exit;
2574             }
2575         }
2576
2577         /* Total length of packet (IP) */
2578         ip_len = ntohs(header_ip->ip_len);
2579
2580         /* TCP data size */
2581         tcp_data_len = ip_len - (header_tcp->th_off << 2) - sizeof(*header_ip);
2582
2583         /* If the frame is padded, trim it */
2584         tot_len = m_head->m_pkthdr.len;
2585         trim = tot_len - (ip_len + ETHER_HDR_LEN);
2586         if(trim != 0) {
2587             if(trim < 0)
2588                 goto _exit;
2589             m_adj(m_head, -trim);
2590             tot_len = m_head->m_pkthdr.len;
2591         }
2592
2593         buffer_next = m_head;
2594         buffer_tail = NULL;
2595         while(buffer_next != NULL) {
2596             buffer_tail = buffer_next;
2597             buffer_next = buffer_tail->m_next;
2598         }
2599
2600         /* Total size of only headers */
2601         hlen = ip_len + ETHER_HDR_LEN - tcp_data_len;
2602
2603         /* Get sequence number */
2604         seq = ntohl(header_tcp->th_seq);
2605
2606         SLIST_FOREACH(lro_session, &lldev->lro_active, next) {
2607             if(lro_session->source_port == header_tcp->th_sport &&
2608                 lro_session->dest_port == header_tcp->th_dport &&
2609                 lro_session->source_ip == header_ip->ip_src.s_addr &&
2610                 lro_session->dest_ip == header_ip->ip_dst.s_addr) {
2611
2612                 /* Unmatched sequence number, flush LRO session */
2613                 if(__predict_false(seq != lro_session->next_seq)) {
2614                     SLIST_REMOVE(&lldev->lro_active, lro_session,
2615                         xge_lro_entry_t, next);
2616                     xge_lro_flush(lldev, lro_session);
2617                     goto _exit;
2618                 }
2619
2620                 /* Handle timestamp option */
2621                 if(tcp_options) {
2622                     u32 tsval = ntohl(*(ptr + 1));
2623                     if(__predict_false(lro_session->tsval > tsval ||
2624                         *(ptr + 2) == 0)) {
2625                         goto _exit;
2626                     }
2627                     lro_session->tsval = tsval;
2628                     lro_session->tsecr = *(ptr + 2);
2629                 }
2630
2631                 lro_session->next_seq += tcp_data_len;
2632                 lro_session->ack_seq = header_tcp->th_ack;
2633                 lro_session->window = header_tcp->th_win;
2634
2635                 /* If TCP data/payload is of 0 size, free mbuf */
2636                 if(tcp_data_len == 0) {
2637                     m_freem(m_head);
2638                     status = XGE_HAL_OK;
2639                     goto _exit;
2640                 }
2641
2642                 lro_session->append_cnt++;
2643                 lro_session->len += tcp_data_len;
2644
2645                 /* Adjust mbuf so that m_data points to payload than headers */
2646                 m_adj(m_head, hlen);
2647
2648                 /* Append this packet to LRO accumulated packet */
2649                 lro_session->m_tail->m_next = m_head;
2650                 lro_session->m_tail = buffer_tail;
2651
2652                 /* Flush if LRO packet is exceeding maximum size */
2653                 if(lro_session->len >
2654                     (XGE_HAL_LRO_DEFAULT_FRM_LEN - lldev->ifnetp->if_mtu)) {
2655                     SLIST_REMOVE(&lldev->lro_active, lro_session,
2656                         xge_lro_entry_t, next);
2657                     xge_lro_flush(lldev, lro_session);
2658                 }
2659                 status = XGE_HAL_OK;
2660                 goto _exit;
2661             }
2662         }
2663
2664         if(SLIST_EMPTY(&lldev->lro_free))
2665             goto _exit;
2666
2667         /* Start a new LRO session */
2668         lro_session = SLIST_FIRST(&lldev->lro_free);
2669         SLIST_REMOVE_HEAD(&lldev->lro_free, next);
2670         SLIST_INSERT_HEAD(&lldev->lro_active, lro_session, next);
2671         lro_session->source_port = header_tcp->th_sport;
2672         lro_session->dest_port = header_tcp->th_dport;
2673         lro_session->source_ip = header_ip->ip_src.s_addr;
2674         lro_session->dest_ip = header_ip->ip_dst.s_addr;
2675         lro_session->next_seq = seq + tcp_data_len;
2676         lro_session->mss = tcp_data_len;
2677         lro_session->ack_seq = header_tcp->th_ack;
2678         lro_session->window = header_tcp->th_win;
2679
2680         lro_session->lro_header_ip = header_ip;
2681
2682         /* Handle timestamp option */
2683         if(tcp_options) {
2684             lro_session->timestamp = 1;
2685             lro_session->tsval = ntohl(*(ptr + 1));
2686             lro_session->tsecr = *(ptr + 2);
2687         }
2688
2689         lro_session->len = tot_len;
2690         lro_session->m_head = m_head;
2691         lro_session->m_tail = buffer_tail;
2692         status = XGE_HAL_OK;
2693
2694 _exit:
2695         return status;
2696 }
2697
2698 /**
2699  * xge_accumulate_large_rx
2700  * Accumulate packets to form a large LRO packet based on various conditions
2701  *
2702  * @lldev Per-adapter Data
2703  * @pkt Current packet
2704  * @pkt_length Packet Length
2705  * @rxd_priv Rx Descriptor Private Data
2706  */
2707 void
2708 xge_accumulate_large_rx(xge_lldev_t *lldev, struct mbuf *pkt, int pkt_length,
2709         xge_rx_priv_t *rxd_priv)
2710 {
2711         if(xge_lro_accumulate(lldev, pkt) != XGE_HAL_OK) {
2712             bus_dmamap_sync(lldev->dma_tag_rx, rxd_priv->dmainfo[0].dma_map,
2713                 BUS_DMASYNC_POSTREAD);
2714             (*lldev->ifnetp->if_input)(lldev->ifnetp, pkt);
2715         }
2716 }
2717
2718 /**
2719  * xge_rx_compl
2720  * If the interrupt is due to received frame (Rx completion), send it up
2721  *
2722  * @channelh Ring Channel Handle
2723  * @dtr Current Descriptor
2724  * @t_code Transfer Code indicating success or error
2725  * @userdata Per-adapter Data
2726  *
2727  * Returns XGE_HAL_OK or HAL error enums
2728  */
2729 xge_hal_status_e
2730 xge_rx_compl(xge_hal_channel_h channelh, xge_hal_dtr_h dtr, u8 t_code,
2731         void *userdata)
2732 {
2733         struct ifnet       *ifnetp;
2734         xge_rx_priv_t      *rxd_priv = NULL;
2735         mbuf_t              mbuf_up  = NULL;
2736         xge_hal_status_e    status   = XGE_HAL_OK;
2737         xge_hal_dtr_info_t  ext_info;
2738         int                 index;
2739         u16                 vlan_tag;
2740
2741         /*get the user data portion*/
2742         xge_lldev_t *lldev = xge_hal_channel_userdata(channelh);
2743         if(!lldev) {
2744             XGE_EXIT_ON_ERR("Failed to get user data", _exit, XGE_HAL_FAIL);
2745         }
2746
2747         XGE_DRV_STATS(rx_completions);
2748
2749         /* get the interface pointer */
2750         ifnetp = lldev->ifnetp;
2751
2752         do {
2753             XGE_DRV_STATS(rx_desc_compl);
2754
2755             if(!(ifnetp->if_drv_flags & IFF_DRV_RUNNING)) {
2756                 status = XGE_HAL_FAIL;
2757                 goto _exit;
2758             }
2759
2760             if(t_code) {
2761                 xge_trace(XGE_TRACE, "Packet dropped because of %d", t_code);
2762                 XGE_DRV_STATS(rx_tcode);
2763                 xge_hal_device_handle_tcode(channelh, dtr, t_code);
2764                 xge_hal_ring_dtr_post(channelh,dtr);
2765                 continue;
2766             }
2767
2768             /* Get the private data for this descriptor*/
2769             rxd_priv = (xge_rx_priv_t *) xge_hal_ring_dtr_private(channelh,
2770                 dtr);
2771             if(!rxd_priv) {
2772                 XGE_EXIT_ON_ERR("Failed to get descriptor private data", _exit,
2773                     XGE_HAL_FAIL);
2774             }
2775
2776             /*
2777              * Prepare one buffer to send it to upper layer -- since the upper
2778              * layer frees the buffer do not use rxd_priv->buffer. Meanwhile
2779              * prepare a new buffer, do mapping, use it in the current
2780              * descriptor and post descriptor back to ring channel
2781              */
2782             mbuf_up = rxd_priv->bufferArray[0];
2783
2784             /* Gets details of mbuf i.e., packet length */
2785             xge_ring_dtr_get(mbuf_up, channelh, dtr, lldev, rxd_priv);
2786
2787             status =
2788                 (lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_1) ?
2789                 xge_get_buf(dtr, rxd_priv, lldev, 0) :
2790                 xge_get_buf_3b_5b(dtr, rxd_priv, lldev);
2791
2792             if(status != XGE_HAL_OK) {
2793                 xge_trace(XGE_ERR, "No memory");
2794                 XGE_DRV_STATS(rx_no_buf);
2795
2796                 /*
2797                  * Unable to allocate buffer. Instead of discarding, post
2798                  * descriptor back to channel for future processing of same
2799                  * packet.
2800                  */
2801                 xge_hal_ring_dtr_post(channelh, dtr);
2802                 continue;
2803             }
2804
2805             /* Get the extended information */
2806             xge_hal_ring_dtr_info_get(channelh, dtr, &ext_info);
2807
2808             /*
2809              * As we have allocated a new mbuf for this descriptor, post this
2810              * descriptor with new mbuf back to ring channel
2811              */
2812             vlan_tag = ext_info.vlan;
2813             xge_hal_ring_dtr_post(channelh, dtr);
2814             if ((!(ext_info.proto & XGE_HAL_FRAME_PROTO_IP_FRAGMENTED) &&
2815                 (ext_info.proto & XGE_HAL_FRAME_PROTO_TCP_OR_UDP) &&
2816                 (ext_info.l3_cksum == XGE_HAL_L3_CKSUM_OK) &&
2817                 (ext_info.l4_cksum == XGE_HAL_L4_CKSUM_OK))) {
2818
2819                 /* set Checksum Flag */
2820                 xge_set_mbuf_cflags(mbuf_up);
2821
2822                 if(lldev->enabled_lro) {
2823                     xge_accumulate_large_rx(lldev, mbuf_up, mbuf_up->m_len,
2824                         rxd_priv);
2825                 }
2826                 else {
2827                     /* Post-Read sync for buffers*/
2828                     for(index = 0; index < lldev->rxd_mbuf_cnt; index++) {
2829                         bus_dmamap_sync(lldev->dma_tag_rx,
2830                             rxd_priv->dmainfo[0].dma_map, BUS_DMASYNC_POSTREAD);
2831                     }
2832                     (*ifnetp->if_input)(ifnetp, mbuf_up);
2833                 }
2834             }
2835             else {
2836                 /*
2837                  * Packet with erroneous checksum , let the upper layer deal
2838                  * with it
2839                  */
2840
2841                 /* Post-Read sync for buffers*/
2842                 for(index = 0; index < lldev->rxd_mbuf_cnt; index++) {
2843                     bus_dmamap_sync(lldev->dma_tag_rx,
2844                          rxd_priv->dmainfo[0].dma_map, BUS_DMASYNC_POSTREAD);
2845                 }
2846
2847                 if(vlan_tag) {
2848                     mbuf_up->m_pkthdr.ether_vtag = vlan_tag;
2849                     mbuf_up->m_flags |= M_VLANTAG;
2850                 }
2851
2852                 if(lldev->enabled_lro)
2853                     xge_lro_flush_sessions(lldev);
2854
2855                 (*ifnetp->if_input)(ifnetp, mbuf_up);
2856             }
2857         } while(xge_hal_ring_dtr_next_completed(channelh, &dtr, &t_code)
2858             == XGE_HAL_OK);
2859
2860         if(lldev->enabled_lro)
2861             xge_lro_flush_sessions(lldev);
2862
2863 _exit:
2864         return status;
2865 }
2866
2867 /**
2868  * xge_ring_dtr_get
2869  * Get descriptors
2870  *
2871  * @mbuf_up Packet to send up
2872  * @channelh Ring Channel Handle
2873  * @dtr Descriptor
2874  * @lldev Per-adapter Data
2875  * @rxd_priv Rx Descriptor Private Data
2876  *
2877  * Returns XGE_HAL_OK or HAL error enums
2878  */
2879 int
2880 xge_ring_dtr_get(mbuf_t mbuf_up, xge_hal_channel_h channelh, xge_hal_dtr_h dtr,
2881         xge_lldev_t *lldev, xge_rx_priv_t *rxd_priv)
2882 {
2883         mbuf_t           m;
2884         int              pkt_length[5]={0,0}, pkt_len=0;
2885         dma_addr_t       dma_data[5];
2886         int              index;
2887
2888         m = mbuf_up;
2889         pkt_len = 0;
2890
2891         if(lldev->buffer_mode != XGE_HAL_RING_QUEUE_BUFFER_MODE_1) {
2892             xge_os_memzero(pkt_length, sizeof(pkt_length));
2893
2894             /*
2895              * Retrieve data of interest from the completed descriptor -- This
2896              * returns the packet length
2897              */
2898             if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_5) {
2899                 xge_hal_ring_dtr_5b_get(channelh, dtr, dma_data, pkt_length);
2900             }
2901             else {
2902                 xge_hal_ring_dtr_3b_get(channelh, dtr, dma_data, pkt_length);
2903             }
2904
2905             for(index = 0; index < lldev->rxd_mbuf_cnt; index++) {
2906                 m->m_len  = pkt_length[index];
2907
2908                 if(index < (lldev->rxd_mbuf_cnt-1)) {
2909                     m->m_next = rxd_priv->bufferArray[index + 1];
2910                     m = m->m_next;
2911                 }
2912                 else {
2913                     m->m_next = NULL;
2914                 }
2915                 pkt_len+=pkt_length[index];
2916             }
2917
2918             /*
2919              * Since 2 buffer mode is an exceptional case where data is in 3rd
2920              * buffer but not in 2nd buffer
2921              */
2922             if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_2) {
2923                 m->m_len = pkt_length[2];
2924                 pkt_len+=pkt_length[2];
2925             }
2926
2927             /*
2928              * Update length of newly created buffer to be sent up with packet
2929              * length
2930              */
2931             mbuf_up->m_pkthdr.len = pkt_len;
2932         }
2933         else {
2934             /*
2935              * Retrieve data of interest from the completed descriptor -- This
2936              * returns the packet length
2937              */
2938             xge_hal_ring_dtr_1b_get(channelh, dtr,&dma_data[0], &pkt_length[0]);
2939
2940             /*
2941              * Update length of newly created buffer to be sent up with packet
2942              * length
2943              */
2944             mbuf_up->m_len =  mbuf_up->m_pkthdr.len = pkt_length[0];
2945         }
2946
2947         return XGE_HAL_OK;
2948 }
2949
2950 /**
2951  * xge_flush_txds
2952  * Flush Tx descriptors
2953  *
2954  * @channelh Channel handle
2955  */
2956 static void inline
2957 xge_flush_txds(xge_hal_channel_h channelh)
2958 {
2959         xge_lldev_t *lldev = xge_hal_channel_userdata(channelh);
2960         xge_hal_dtr_h tx_dtr;
2961         xge_tx_priv_t *tx_priv;
2962         u8 t_code;
2963
2964         while(xge_hal_fifo_dtr_next_completed(channelh, &tx_dtr, &t_code)
2965             == XGE_HAL_OK) {
2966             XGE_DRV_STATS(tx_desc_compl);
2967             if(t_code) {
2968                 xge_trace(XGE_TRACE, "Tx descriptor with t_code %d", t_code);
2969                 XGE_DRV_STATS(tx_tcode);
2970                 xge_hal_device_handle_tcode(channelh, tx_dtr, t_code);
2971             }
2972
2973             tx_priv = xge_hal_fifo_dtr_private(tx_dtr);
2974             bus_dmamap_unload(lldev->dma_tag_tx, tx_priv->dma_map);
2975             m_freem(tx_priv->buffer);
2976             tx_priv->buffer = NULL;
2977             xge_hal_fifo_dtr_free(channelh, tx_dtr);
2978         }
2979 }
2980
2981 /**
2982  * xge_send
2983  * Transmit function
2984  *
2985  * @ifnetp Interface Handle
2986  */
2987 void
2988 xge_send(struct ifnet *ifnetp)
2989 {
2990         int qindex = 0;
2991         xge_lldev_t *lldev = ifnetp->if_softc;
2992
2993         for(qindex = 0; qindex < XGE_FIFO_COUNT; qindex++) {
2994             if(mtx_trylock(&lldev->mtx_tx[qindex]) == 0) {
2995                 XGE_DRV_STATS(tx_lock_fail);
2996                 break;
2997             }
2998             xge_send_locked(ifnetp, qindex);
2999             mtx_unlock(&lldev->mtx_tx[qindex]);
3000         }
3001 }
3002
3003 static void inline
3004 xge_send_locked(struct ifnet *ifnetp, int qindex)
3005 {
3006         xge_hal_dtr_h            dtr;
3007         static bus_dma_segment_t segs[XGE_MAX_SEGS];
3008         xge_hal_status_e         status;
3009         unsigned int             max_fragments;
3010         xge_lldev_t              *lldev          = ifnetp->if_softc;
3011         xge_hal_channel_h        channelh        = lldev->fifo_channel[qindex];
3012         mbuf_t                   m_head          = NULL;
3013         mbuf_t                   m_buf           = NULL;
3014         xge_tx_priv_t            *ll_tx_priv     = NULL;
3015         register unsigned int    count           = 0;
3016         unsigned int             nsegs           = 0;
3017         u16                      vlan_tag;
3018
3019         max_fragments = ((xge_hal_fifo_t *)channelh)->config->max_frags;
3020
3021         /* If device is not initialized, return */
3022         if((!lldev->initialized) || (!(ifnetp->if_drv_flags & IFF_DRV_RUNNING)))
3023             return;
3024
3025         XGE_DRV_STATS(tx_calls);
3026
3027         /*
3028          * This loop will be executed for each packet in the kernel maintained
3029          * queue -- each packet can be with fragments as an mbuf chain
3030          */
3031         for(;;) {
3032             IF_DEQUEUE(&ifnetp->if_snd, m_head);
3033             if (m_head == NULL) {
3034                 ifnetp->if_drv_flags &= ~(IFF_DRV_OACTIVE);
3035                 return;
3036             }
3037
3038             for(m_buf = m_head; m_buf != NULL; m_buf = m_buf->m_next) {
3039                 if(m_buf->m_len) count += 1;
3040             }
3041
3042             if(count >= max_fragments) {
3043                 m_buf = m_defrag(m_head, M_NOWAIT);
3044                 if(m_buf != NULL) m_head = m_buf;
3045                 XGE_DRV_STATS(tx_defrag);
3046             }
3047
3048             /* Reserve descriptors */
3049             status = xge_hal_fifo_dtr_reserve(channelh, &dtr);
3050             if(status != XGE_HAL_OK) {
3051                 XGE_DRV_STATS(tx_no_txd);
3052                 xge_flush_txds(channelh);
3053                 break;
3054             }
3055
3056             vlan_tag =
3057                 (m_head->m_flags & M_VLANTAG) ? m_head->m_pkthdr.ether_vtag : 0;
3058             xge_hal_fifo_dtr_vlan_set(dtr, vlan_tag);
3059
3060             /* Update Tx private structure for this descriptor */
3061             ll_tx_priv         = xge_hal_fifo_dtr_private(dtr);
3062             ll_tx_priv->buffer = m_head;
3063
3064             /*
3065              * Do mapping -- Required DMA tag has been created in xge_init
3066              * function and DMA maps have already been created in the
3067              * xgell_tx_replenish function.
3068              * Returns number of segments through nsegs
3069              */
3070             if(bus_dmamap_load_mbuf_sg(lldev->dma_tag_tx,
3071                 ll_tx_priv->dma_map, m_head, segs, &nsegs, BUS_DMA_NOWAIT)) {
3072                 xge_trace(XGE_TRACE, "DMA map load failed");
3073                 XGE_DRV_STATS(tx_map_fail);
3074                 break;
3075             }
3076
3077             if(lldev->driver_stats.tx_max_frags < nsegs)
3078                 lldev->driver_stats.tx_max_frags = nsegs;
3079
3080             /* Set descriptor buffer for header and each fragment/segment */
3081             count = 0;
3082             do {
3083                 xge_hal_fifo_dtr_buffer_set(channelh, dtr, count,
3084                     (dma_addr_t)htole64(segs[count].ds_addr),
3085                     segs[count].ds_len);
3086                 count++;
3087             } while(count < nsegs);
3088
3089             /* Pre-write Sync of mapping */
3090             bus_dmamap_sync(lldev->dma_tag_tx, ll_tx_priv->dma_map,
3091                 BUS_DMASYNC_PREWRITE);
3092
3093             if((lldev->enabled_tso) &&
3094                 (m_head->m_pkthdr.csum_flags & CSUM_TSO)) {
3095                 XGE_DRV_STATS(tx_tso);
3096                 xge_hal_fifo_dtr_mss_set(dtr, m_head->m_pkthdr.tso_segsz);
3097             }
3098
3099             /* Checksum */
3100             if(ifnetp->if_hwassist > 0) {
3101                 xge_hal_fifo_dtr_cksum_set_bits(dtr, XGE_HAL_TXD_TX_CKO_IPV4_EN
3102                     | XGE_HAL_TXD_TX_CKO_TCP_EN | XGE_HAL_TXD_TX_CKO_UDP_EN);
3103             }
3104
3105             /* Post descriptor to FIFO channel */
3106             xge_hal_fifo_dtr_post(channelh, dtr);
3107             XGE_DRV_STATS(tx_posted);
3108
3109             /* Send the same copy of mbuf packet to BPF (Berkely Packet Filter)
3110              * listener so that we can use tools like tcpdump */
3111             ETHER_BPF_MTAP(ifnetp, m_head);
3112         }
3113
3114         /* Prepend the packet back to queue */
3115         IF_PREPEND(&ifnetp->if_snd, m_head);
3116         ifnetp->if_drv_flags |= IFF_DRV_OACTIVE;
3117
3118         xge_queue_produce_context(xge_hal_device_queue(lldev->devh),
3119             XGE_LL_EVENT_TRY_XMIT_AGAIN, lldev->devh);
3120         XGE_DRV_STATS(tx_again);
3121 }
3122
3123 /**
3124  * xge_get_buf
3125  * Allocates new mbufs to be placed into descriptors
3126  *
3127  * @dtrh Descriptor Handle
3128  * @rxd_priv Rx Descriptor Private Data
3129  * @lldev Per-adapter Data
3130  * @index Buffer Index (if multi-buffer mode)
3131  *
3132  * Returns XGE_HAL_OK or HAL error enums
3133  */
3134 int
3135 xge_get_buf(xge_hal_dtr_h dtrh, xge_rx_priv_t *rxd_priv,
3136         xge_lldev_t *lldev, int index)
3137 {
3138         register mbuf_t mp            = NULL;
3139         struct          ifnet *ifnetp = lldev->ifnetp;
3140         int             status        = XGE_HAL_OK;
3141         int             buffer_size = 0, cluster_size = 0, count;
3142         bus_dmamap_t    map = rxd_priv->dmainfo[index].dma_map;
3143         bus_dma_segment_t segs[3];
3144
3145         buffer_size = (lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_1) ?
3146             ifnetp->if_mtu + XGE_HAL_MAC_HEADER_MAX_SIZE :
3147             lldev->rxd_mbuf_len[index];
3148
3149         if(buffer_size <= MCLBYTES) {
3150             cluster_size = MCLBYTES;
3151             mp = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
3152         }
3153         else {
3154             cluster_size = MJUMPAGESIZE;
3155             if((lldev->buffer_mode != XGE_HAL_RING_QUEUE_BUFFER_MODE_5) &&
3156                 (buffer_size > MJUMPAGESIZE)) {
3157                 cluster_size = MJUM9BYTES;
3158             }
3159             mp = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, cluster_size);
3160         }
3161         if(!mp) {
3162             xge_trace(XGE_ERR, "Out of memory to allocate mbuf");
3163             status = XGE_HAL_FAIL;
3164             goto getbuf_out;
3165         }
3166
3167         /* Update mbuf's length, packet length and receive interface */
3168         mp->m_len = mp->m_pkthdr.len = buffer_size;
3169         mp->m_pkthdr.rcvif = ifnetp;
3170
3171         /* Load DMA map */
3172         if(bus_dmamap_load_mbuf_sg(lldev->dma_tag_rx, lldev->extra_dma_map,
3173             mp, segs, &count, BUS_DMA_NOWAIT)) {
3174             XGE_DRV_STATS(rx_map_fail);
3175             m_freem(mp);
3176             XGE_EXIT_ON_ERR("DMA map load failed", getbuf_out, XGE_HAL_FAIL);
3177         }
3178
3179         /* Update descriptor private data */
3180         rxd_priv->bufferArray[index]         = mp;
3181         rxd_priv->dmainfo[index].dma_phyaddr = htole64(segs->ds_addr);
3182         rxd_priv->dmainfo[index].dma_map     = lldev->extra_dma_map;
3183         lldev->extra_dma_map = map;
3184
3185         /* Pre-Read/Write sync */
3186         bus_dmamap_sync(lldev->dma_tag_rx, map, BUS_DMASYNC_POSTREAD);
3187
3188         /* Unload DMA map of mbuf in current descriptor */
3189         bus_dmamap_unload(lldev->dma_tag_rx, map);
3190
3191         /* Set descriptor buffer */
3192         if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_1) {
3193             xge_hal_ring_dtr_1b_set(dtrh, rxd_priv->dmainfo[0].dma_phyaddr,
3194                 cluster_size);
3195         }
3196
3197 getbuf_out:
3198         return status;
3199 }
3200
3201 /**
3202  * xge_get_buf_3b_5b
3203  * Allocates new mbufs to be placed into descriptors (in multi-buffer modes)
3204  *
3205  * @dtrh Descriptor Handle
3206  * @rxd_priv Rx Descriptor Private Data
3207  * @lldev Per-adapter Data
3208  *
3209  * Returns XGE_HAL_OK or HAL error enums
3210  */
3211 int
3212 xge_get_buf_3b_5b(xge_hal_dtr_h dtrh, xge_rx_priv_t *rxd_priv,
3213         xge_lldev_t *lldev)
3214 {
3215         bus_addr_t  dma_pointers[5];
3216         int         dma_sizes[5];
3217         int         status = XGE_HAL_OK, index;
3218         int         newindex = 0;
3219
3220         for(index = 0; index < lldev->rxd_mbuf_cnt; index++) {
3221             status = xge_get_buf(dtrh, rxd_priv, lldev, index);
3222             if(status != XGE_HAL_OK) {
3223                 for(newindex = 0; newindex < index; newindex++) {
3224                     m_freem(rxd_priv->bufferArray[newindex]);
3225                 }
3226                 XGE_EXIT_ON_ERR("mbuf allocation failed", _exit, status);
3227             }
3228         }
3229
3230         for(index = 0; index < lldev->buffer_mode; index++) {
3231             if(lldev->rxd_mbuf_len[index] != 0) {
3232                 dma_pointers[index] = rxd_priv->dmainfo[index].dma_phyaddr;
3233                 dma_sizes[index]    = lldev->rxd_mbuf_len[index];
3234             }
3235             else {
3236                 dma_pointers[index] = rxd_priv->dmainfo[index-1].dma_phyaddr;
3237                 dma_sizes[index]    = 1;
3238             }
3239         }
3240
3241         /* Assigning second buffer to third pointer in 2 buffer mode */
3242         if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_2) {
3243             dma_pointers[2] = dma_pointers[1];
3244             dma_sizes[2]    = dma_sizes[1];
3245             dma_sizes[1]    = 1;
3246         }
3247
3248         if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_5) {
3249             xge_hal_ring_dtr_5b_set(dtrh, dma_pointers, dma_sizes);
3250         }
3251         else {
3252             xge_hal_ring_dtr_3b_set(dtrh, dma_pointers, dma_sizes);
3253         }
3254
3255 _exit:
3256         return status;
3257 }
3258
3259 /**
3260  * xge_tx_compl
3261  * If the interrupt is due to Tx completion, free the sent buffer
3262  *
3263  * @channelh Channel Handle
3264  * @dtr Descriptor
3265  * @t_code Transfer Code indicating success or error
3266  * @userdata Per-adapter Data
3267  *
3268  * Returns XGE_HAL_OK or HAL error enum
3269  */
3270 xge_hal_status_e
3271 xge_tx_compl(xge_hal_channel_h channelh,
3272         xge_hal_dtr_h dtr, u8 t_code, void *userdata)
3273 {
3274         xge_tx_priv_t *ll_tx_priv = NULL;
3275         xge_lldev_t   *lldev  = (xge_lldev_t *)userdata;
3276         struct ifnet  *ifnetp = lldev->ifnetp;
3277         mbuf_t         m_buffer = NULL;
3278         int            qindex   = xge_hal_channel_id(channelh);
3279
3280         mtx_lock(&lldev->mtx_tx[qindex]);
3281
3282         XGE_DRV_STATS(tx_completions);
3283
3284         /*
3285          * For each completed descriptor: Get private structure, free buffer,
3286          * do unmapping, and free descriptor
3287          */
3288         do {
3289             XGE_DRV_STATS(tx_desc_compl);
3290
3291             if(t_code) {
3292                 XGE_DRV_STATS(tx_tcode);
3293                 xge_trace(XGE_TRACE, "t_code %d", t_code);
3294                 xge_hal_device_handle_tcode(channelh, dtr, t_code);
3295             }
3296
3297             ll_tx_priv = xge_hal_fifo_dtr_private(dtr);
3298             m_buffer   = ll_tx_priv->buffer;
3299             bus_dmamap_unload(lldev->dma_tag_tx, ll_tx_priv->dma_map);
3300             m_freem(m_buffer);
3301             ll_tx_priv->buffer = NULL;
3302             xge_hal_fifo_dtr_free(channelh, dtr);
3303         } while(xge_hal_fifo_dtr_next_completed(channelh, &dtr, &t_code)
3304             == XGE_HAL_OK);
3305         xge_send_locked(ifnetp, qindex);
3306         ifnetp->if_drv_flags &= ~IFF_DRV_OACTIVE;
3307
3308         mtx_unlock(&lldev->mtx_tx[qindex]);
3309
3310         return XGE_HAL_OK;
3311 }
3312
3313 /**
3314  * xge_tx_initial_replenish
3315  * Initially allocate buffers and set them into descriptors for later use
3316  *
3317  * @channelh Tx Channel Handle
3318  * @dtrh Descriptor Handle
3319  * @index
3320  * @userdata Per-adapter Data
3321  * @reopen Channel open/reopen option
3322  *
3323  * Returns XGE_HAL_OK or HAL error enums
3324  */
3325 xge_hal_status_e
3326 xge_tx_initial_replenish(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh,
3327         int index, void *userdata, xge_hal_channel_reopen_e reopen)
3328 {
3329         xge_tx_priv_t *txd_priv = NULL;
3330         int            status   = XGE_HAL_OK;
3331
3332         /* Get the user data portion from channel handle */
3333         xge_lldev_t *lldev = xge_hal_channel_userdata(channelh);
3334         if(lldev == NULL) {
3335             XGE_EXIT_ON_ERR("Failed to get user data from channel", txinit_out,
3336                 XGE_HAL_FAIL);
3337         }
3338
3339         /* Get the private data */
3340         txd_priv = (xge_tx_priv_t *) xge_hal_fifo_dtr_private(dtrh);
3341         if(txd_priv == NULL) {
3342             XGE_EXIT_ON_ERR("Failed to get descriptor private data", txinit_out,
3343                 XGE_HAL_FAIL);
3344         }
3345
3346         /* Create DMA map for this descriptor */
3347         if(bus_dmamap_create(lldev->dma_tag_tx, BUS_DMA_NOWAIT,
3348             &txd_priv->dma_map)) {
3349             XGE_EXIT_ON_ERR("DMA map creation for Tx descriptor failed",
3350                 txinit_out, XGE_HAL_FAIL);
3351         }
3352
3353 txinit_out:
3354         return status;
3355 }
3356
3357 /**
3358  * xge_rx_initial_replenish
3359  * Initially allocate buffers and set them into descriptors for later use
3360  *
3361  * @channelh Tx Channel Handle
3362  * @dtrh Descriptor Handle
3363  * @index Ring Index
3364  * @userdata Per-adapter Data
3365  * @reopen Channel open/reopen option
3366  *
3367  * Returns XGE_HAL_OK or HAL error enums
3368  */
3369 xge_hal_status_e
3370 xge_rx_initial_replenish(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh,
3371         int index, void *userdata, xge_hal_channel_reopen_e reopen)
3372 {
3373         xge_rx_priv_t  *rxd_priv = NULL;
3374         int             status   = XGE_HAL_OK;
3375         int             index1 = 0, index2 = 0;
3376
3377         /* Get the user data portion from channel handle */
3378         xge_lldev_t *lldev = xge_hal_channel_userdata(channelh);
3379         if(lldev == NULL) {
3380             XGE_EXIT_ON_ERR("Failed to get user data from channel", rxinit_out,
3381                 XGE_HAL_FAIL);
3382         }
3383
3384         /* Get the private data */
3385         rxd_priv = (xge_rx_priv_t *) xge_hal_ring_dtr_private(channelh, dtrh);
3386         if(rxd_priv == NULL) {
3387             XGE_EXIT_ON_ERR("Failed to get descriptor private data", rxinit_out,
3388                 XGE_HAL_FAIL);
3389         }
3390
3391         rxd_priv->bufferArray = xge_os_malloc(NULL,
3392                 (sizeof(rxd_priv->bufferArray) * lldev->rxd_mbuf_cnt));
3393
3394         if(rxd_priv->bufferArray == NULL) {
3395             XGE_EXIT_ON_ERR("Failed to allocate Rxd private", rxinit_out,
3396                 XGE_HAL_FAIL);
3397         }
3398
3399         if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_1) {
3400             /* Create DMA map for these descriptors*/
3401             if(bus_dmamap_create(lldev->dma_tag_rx , BUS_DMA_NOWAIT,
3402                 &rxd_priv->dmainfo[0].dma_map)) {
3403                 XGE_EXIT_ON_ERR("DMA map creation for Rx descriptor failed",
3404                     rxinit_err_out, XGE_HAL_FAIL);
3405             }
3406             /* Get a buffer, attach it to this descriptor */
3407             status = xge_get_buf(dtrh, rxd_priv, lldev, 0);
3408         }
3409         else {
3410             for(index1 = 0; index1 < lldev->rxd_mbuf_cnt; index1++) {
3411                 /* Create DMA map for this descriptor */
3412                 if(bus_dmamap_create(lldev->dma_tag_rx , BUS_DMA_NOWAIT ,
3413                     &rxd_priv->dmainfo[index1].dma_map)) {
3414                     for(index2 = index1 - 1; index2 >= 0; index2--) {
3415                         bus_dmamap_destroy(lldev->dma_tag_rx,
3416                             rxd_priv->dmainfo[index2].dma_map);
3417                     }
3418                     XGE_EXIT_ON_ERR(
3419                         "Jumbo DMA map creation for Rx descriptor failed",
3420                         rxinit_err_out, XGE_HAL_FAIL);
3421                 }
3422             }
3423             status = xge_get_buf_3b_5b(dtrh, rxd_priv, lldev);
3424         }
3425
3426         if(status != XGE_HAL_OK) {
3427             for(index1 = 0; index1 < lldev->rxd_mbuf_cnt; index1++) {
3428                 bus_dmamap_destroy(lldev->dma_tag_rx,
3429                     rxd_priv->dmainfo[index1].dma_map);
3430             }
3431             goto rxinit_err_out;
3432         }
3433         else {
3434             goto rxinit_out;
3435         }
3436
3437 rxinit_err_out:
3438         xge_os_free(NULL, rxd_priv->bufferArray,
3439             (sizeof(rxd_priv->bufferArray) * lldev->rxd_mbuf_cnt));
3440 rxinit_out:
3441         return status;
3442 }
3443
3444 /**
3445  * xge_rx_term
3446  * During unload terminate and free all descriptors
3447  *
3448  * @channelh Rx Channel Handle
3449  * @dtrh Rx Descriptor Handle
3450  * @state Descriptor State
3451  * @userdata Per-adapter Data
3452  * @reopen Channel open/reopen option
3453  */
3454 void
3455 xge_rx_term(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh,
3456         xge_hal_dtr_state_e state, void *userdata,
3457         xge_hal_channel_reopen_e reopen)
3458 {
3459         xge_rx_priv_t *rxd_priv = NULL;
3460         xge_lldev_t   *lldev    = NULL;
3461         int            index = 0;
3462
3463         /* Descriptor state is not "Posted" */
3464         if(state != XGE_HAL_DTR_STATE_POSTED) goto rxterm_out;
3465
3466         /* Get the user data portion */
3467         lldev = xge_hal_channel_userdata(channelh);
3468
3469         /* Get the private data */
3470         rxd_priv = (xge_rx_priv_t *) xge_hal_ring_dtr_private(channelh, dtrh);
3471
3472         for(index = 0; index < lldev->rxd_mbuf_cnt; index++) {
3473             if(rxd_priv->dmainfo[index].dma_map != NULL) {
3474                 bus_dmamap_sync(lldev->dma_tag_rx,
3475                     rxd_priv->dmainfo[index].dma_map, BUS_DMASYNC_POSTREAD);
3476                 bus_dmamap_unload(lldev->dma_tag_rx,
3477                     rxd_priv->dmainfo[index].dma_map);
3478                 if(rxd_priv->bufferArray[index] != NULL)
3479                     m_free(rxd_priv->bufferArray[index]);
3480                 bus_dmamap_destroy(lldev->dma_tag_rx,
3481                     rxd_priv->dmainfo[index].dma_map);
3482             }
3483         }
3484         xge_os_free(NULL, rxd_priv->bufferArray,
3485             (sizeof(rxd_priv->bufferArray) * lldev->rxd_mbuf_cnt));
3486
3487         /* Free the descriptor */
3488         xge_hal_ring_dtr_free(channelh, dtrh);
3489
3490 rxterm_out:
3491         return;
3492 }
3493
3494 /**
3495  * xge_tx_term
3496  * During unload terminate and free all descriptors
3497  *
3498  * @channelh Rx Channel Handle
3499  * @dtrh Rx Descriptor Handle
3500  * @state Descriptor State
3501  * @userdata Per-adapter Data
3502  * @reopen Channel open/reopen option
3503  */
3504 void
3505 xge_tx_term(xge_hal_channel_h channelh, xge_hal_dtr_h dtr,
3506         xge_hal_dtr_state_e state, void *userdata,
3507         xge_hal_channel_reopen_e reopen)
3508 {
3509         xge_tx_priv_t *ll_tx_priv = xge_hal_fifo_dtr_private(dtr);
3510         xge_lldev_t   *lldev      = (xge_lldev_t *)userdata;
3511
3512         /* Destroy DMA map */
3513         bus_dmamap_destroy(lldev->dma_tag_tx, ll_tx_priv->dma_map);
3514 }
3515
3516 /**
3517  * xge_methods
3518  *
3519  * FreeBSD device interface entry points
3520  */
3521 static device_method_t xge_methods[] = {
3522         DEVMETHOD(device_probe,     xge_probe),
3523         DEVMETHOD(device_attach,    xge_attach),
3524         DEVMETHOD(device_detach,    xge_detach),
3525         DEVMETHOD(device_shutdown,  xge_shutdown),
3526
3527         DEVMETHOD_END
3528 };
3529
3530 static driver_t xge_driver = {
3531         "nxge",
3532         xge_methods,
3533         sizeof(xge_lldev_t),
3534 };
3535 static devclass_t xge_devclass;
3536 DRIVER_MODULE(nxge, pci, xge_driver, xge_devclass, 0, 0);
3537